From 92bbfeece3388b123ea3aff0c0802c97fce1b643 Mon Sep 17 00:00:00 2001 From: George Koltsov Date: Tue, 13 Apr 2021 17:42:31 +0100 Subject: [PATCH] Add Group partial export API - Add a set of Group endpoints to initiate relations export - Add a set of Sidekiq workers to perform relations export - Add a set of models to track export & store exported data - Upload exported ndjson.gz to ObjectStorage Changelog: added --- app/models/bulk_imports/export.rb | 68 ++++++++++++ app/models/bulk_imports/export_upload.rb | 18 ++++ app/models/bulk_imports/exports/config.rb | 51 +++++++++ .../bulk_imports/exports/group_config.rb | 19 ++++ .../bulk_imports/exports/project_config.rb | 21 ++++ app/models/group.rb | 2 + app/services/bulk_imports/export_service.rb | 25 +++++ .../bulk_imports/relation_export_service.rb | 74 +++++++++++++ app/uploaders/bulk_imports/export_uploader.rb | 7 ++ app/workers/all_queues.yml | 8 ++ .../bulk_imports/relation_export_worker.rb | 43 ++++++++ ...eorgekoltsov-bulk_import_group_exports.yml | 5 + config/sidekiq_queues.yml | 2 + ...414100914_add_bulk_import_exports_table.rb | 27 +++++ ...n_key_to_bulk_import_exports_on_project.rb | 17 +++ ...ign_key_to_bulk_import_exports_on_group.rb | 17 +++ ...7_add_bulk_import_exports_table_indexes.rb | 29 ++++++ ...10_add_bulk_import_export_uploads_table.rb | 21 ++++ ...to_bulk_import_export_uploads_on_export.rb | 22 ++++ db/schema_migrations/20210414100914 | 1 + db/schema_migrations/20210414130017 | 1 + db/schema_migrations/20210414130526 | 1 + db/schema_migrations/20210414131807 | 1 + db/schema_migrations/20210414133310 | 1 + db/schema_migrations/20210419085714 | 1 + db/structure.sql | 66 ++++++++++++ .../entities/bulk_imports/export_status.rb | 14 +++ lib/api/group_export.rb | 37 +++++++ lib/gitlab/import_export.rb | 17 +++ lib/gitlab/import_export/command_line_util.rb | 13 +++ lib/gitlab/import_export/error.rb | 12 ++- .../json/streaming_serializer.rb | 8 +- spec/factories/bulk_import/export_uploads.rb | 7 ++ spec/factories/bulk_import/exports.rb | 24 +++++ spec/fixtures/bulk_imports/labels.ndjson.gz | Bin 0 -> 202 bytes .../bulk_imports/export_status_spec.rb | 20 ++++ spec/lib/gitlab/import_export/all_models.yml | 2 + .../import_export/import_export_spec.rb | 45 ++++++++ spec/models/bulk_imports/export_spec.rb | 85 +++++++++++++++ .../models/bulk_imports/export_upload_spec.rb | 19 ++++ .../bulk_imports/exports/group_config_spec.rb | 44 ++++++++ .../exports/project_config_spec.rb | 44 ++++++++ spec/requests/api/group_export_spec.rb | 63 ++++++++++++ .../bulk_imports/export_service_spec.rb | 43 ++++++++ .../relation_export_service_spec.rb | 55 ++++++++++ .../relation_export_worker_spec.rb | 97 ++++++++++++++++++ 46 files changed, 1191 insertions(+), 6 deletions(-) create mode 100644 app/models/bulk_imports/export.rb create mode 100644 app/models/bulk_imports/export_upload.rb create mode 100644 app/models/bulk_imports/exports/config.rb create mode 100644 app/models/bulk_imports/exports/group_config.rb create mode 100644 app/models/bulk_imports/exports/project_config.rb create mode 100644 app/services/bulk_imports/export_service.rb create mode 100644 app/services/bulk_imports/relation_export_service.rb create mode 100644 app/uploaders/bulk_imports/export_uploader.rb create mode 100644 app/workers/bulk_imports/relation_export_worker.rb create mode 100644 changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml create mode 100644 db/migrate/20210414100914_add_bulk_import_exports_table.rb create mode 100644 db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb create mode 100644 db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb create mode 100644 db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb create mode 100644 db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb create mode 100644 db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb create mode 100644 db/schema_migrations/20210414100914 create mode 100644 db/schema_migrations/20210414130017 create mode 100644 db/schema_migrations/20210414130526 create mode 100644 db/schema_migrations/20210414131807 create mode 100644 db/schema_migrations/20210414133310 create mode 100644 db/schema_migrations/20210419085714 create mode 100644 lib/api/entities/bulk_imports/export_status.rb create mode 100644 spec/factories/bulk_import/export_uploads.rb create mode 100644 spec/factories/bulk_import/exports.rb create mode 100644 spec/fixtures/bulk_imports/labels.ndjson.gz create mode 100644 spec/lib/api/entities/bulk_imports/export_status_spec.rb create mode 100644 spec/models/bulk_imports/export_spec.rb create mode 100644 spec/models/bulk_imports/export_upload_spec.rb create mode 100644 spec/models/bulk_imports/exports/group_config_spec.rb create mode 100644 spec/models/bulk_imports/exports/project_config_spec.rb create mode 100644 spec/services/bulk_imports/export_service_spec.rb create mode 100644 spec/services/bulk_imports/relation_export_service_spec.rb create mode 100644 spec/workers/bulk_imports/relation_export_worker_spec.rb diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb new file mode 100644 index 00000000000000..f16ef4c0709ce1 --- /dev/null +++ b/app/models/bulk_imports/export.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module BulkImports + class Export < ApplicationRecord + include Gitlab::Utils::StrongMemoize + + self.table_name = 'bulk_import_exports' + + belongs_to :project, optional: true + belongs_to :group, optional: true + + has_one :upload, class_name: 'BulkImports::ExportUpload' + + validates :project, presence: true, unless: :group + validates :group, presence: true, unless: :project + validates :relation, :status, presence: true + + validate :exportable_relation? + + state_machine :status, initial: :started do + state :started, value: 0 + state :finished, value: 1 + state :failed, value: -1 + + event :start do + transition started: :started + transition finished: :started + transition failed: :started + end + + event :finish do + transition started: :finished + transition failed: :failed + end + + event :fail_op do + transition any => :failed + end + end + + def exportable_relation? + return unless exportable + + allowed_relations = ::Gitlab::ImportExport.top_level_relations(exportable.class.name) + + errors.add(:relation, 'Unsupported exportable relation') unless allowed_relations.include?(relation) + end + + def exportable + project || group + end + + def relation_definition + config.exportable_tree[:include].find { |include| include[relation.to_sym] } + end + + def config + strong_memoize(:config) do + case exportable + when ::Project + Exports::ProjectConfig.new(exportable) + when ::Group + Exports::GroupConfig.new(exportable) + end + end + end + end +end diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb new file mode 100644 index 00000000000000..c1884317c41eb7 --- /dev/null +++ b/app/models/bulk_imports/export_upload.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module BulkImports + class ExportUpload < ApplicationRecord + self.table_name = 'bulk_import_export_uploads' + + include WithUploads + include ObjectStorage::BackgroundMove + + belongs_to :export, class_name: 'BulkImports::Export' + + mount_uploader :export_file, ExportUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + end +end diff --git a/app/models/bulk_imports/exports/config.rb b/app/models/bulk_imports/exports/config.rb new file mode 100644 index 00000000000000..b0f7a4c62175b6 --- /dev/null +++ b/app/models/bulk_imports/exports/config.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class Config + include Gitlab::Utils::StrongMemoize + + attr_reader :exportable + + def initialize(exportable) + @exportable = exportable + end + + def exportable_tree + attributes_finder.find_root(exportable_class_sym) + end + + def validate_user_permissions(user) + user.can?(ability, exportable) || (raise ::Gitlab::ImportExport::Error.permission_error(user, exportable)) + end + + private + + def attributes_finder + strong_memoize(:attributes_finder) do + ::Gitlab::ImportExport::AttributesFinder.new(config: import_export_config) + end + end + + def import_export_config + ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h + end + + def exportable_class + @exportable_class ||= exportable.class + end + + def exportable_class_sym + @exportable_class_sym ||= exportable_class.to_s.downcase.to_sym + end + + def import_export_yaml + raise NotImplementedError + end + + def ability + raise NotImplementedError + end + end + end +end diff --git a/app/models/bulk_imports/exports/group_config.rb b/app/models/bulk_imports/exports/group_config.rb new file mode 100644 index 00000000000000..f9f8df171ea9ba --- /dev/null +++ b/app/models/bulk_imports/exports/group_config.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class GroupConfig < Config + def import_export_yaml + ::Gitlab::ImportExport.group_config_file + end + + def export_path + exportable.full_path + end + + def ability + :admin_group + end + end + end +end diff --git a/app/models/bulk_imports/exports/project_config.rb b/app/models/bulk_imports/exports/project_config.rb new file mode 100644 index 00000000000000..253ec5c590031a --- /dev/null +++ b/app/models/bulk_imports/exports/project_config.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class ProjectConfig < Config + def export_path + exportable.disk_path + end + + private + + def import_export_yaml + ::Gitlab::ImportExport.config_file + end + + def ability + :admin_project + end + end + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 60a0e27428d6f3..78d8be7e9c58c4 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -67,6 +67,8 @@ class Group < Namespace has_one :import_state, class_name: 'GroupImportState', inverse_of: :group + has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :group + has_many :group_deploy_keys_groups, inverse_of: :group has_many :group_deploy_keys, through: :group_deploy_keys_groups has_many :group_deploy_tokens diff --git a/app/services/bulk_imports/export_service.rb b/app/services/bulk_imports/export_service.rb new file mode 100644 index 00000000000000..4e67b1fca6e769 --- /dev/null +++ b/app/services/bulk_imports/export_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + class ExportService + attr_reader :exportable, :current_user + + def initialize(exportable:, user:) + @exportable = exportable + @current_user = user + end + + def execute + ::Gitlab::ImportExport.top_level_relations(exportable.class.name).each do |relation| + RelationExportWorker.perform_async(current_user.id, exportable.id, exportable.class.name, relation) + end + + ServiceResponse.success + rescue => e + ServiceResponse.error( + message: e.class, + http_status: :unprocessable_entity + ) + end + end +end diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb new file mode 100644 index 00000000000000..b63ad2256c6cdb --- /dev/null +++ b/app/services/bulk_imports/relation_export_service.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module BulkImports + class RelationExportService + include Gitlab::ImportExport::CommandLineUtil + + def initialize(export) + @export = export + @exportable = export.exportable + @config = export.config + @export_path = @config.export_path + @exportable_tree = @config.exportable_tree + end + + def execute + remove_existing_export_file + serialize_relation + compress_exported_relation + upload_compressed_file + ensure + FileUtils.rm_rf(export_path) if export_path + end + + private + + attr_reader :current_user, :export, :exportable, :export_path, :exportable_tree + + def remove_existing_export_file + upload = export.upload + + return unless upload + return unless upload.export_file&.file + + upload.remove_export_file! + upload.save! + end + + # rubocop: disable CodeReuse/Serializer + def serializer + @serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new( + exportable, + exportable_tree, + json_writer, + exportable_path: '' + ) + end + # rubocop: enable CodeReuse/Serializer + + def json_writer + @json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path) + end + + def serialize_relation + serializer.serialize_relation(export.relation_definition) + end + + def compress_exported_relation + gzip(dir: export_path, filename: ndjson_filename) + end + + def upload_compressed_file + compressed_filename = File.join(export_path, "#{ndjson_filename}.gz") + upload = ExportUpload.find_or_initialize_by(export_id: export.id) # rubocop: disable CodeReuse/ActiveRecord + + File.open(compressed_filename) { |file| upload.export_file = file } + + upload.save! + end + + def ndjson_filename + "#{export.relation}.ndjson" + end + end +end diff --git a/app/uploaders/bulk_imports/export_uploader.rb b/app/uploaders/bulk_imports/export_uploader.rb new file mode 100644 index 00000000000000..356e5ce028e579 --- /dev/null +++ b/app/uploaders/bulk_imports/export_uploader.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module BulkImports + class ExportUploader < ImportExportUploader + EXTENSION_WHITELIST = %w[ndjson.gz].freeze + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index fa6ea54e342ec8..9069dd56656ddb 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1563,6 +1563,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: bulk_imports_relation_export + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: chat_notification :feature_category: :chatops :has_external_dependencies: true diff --git a/app/workers/bulk_imports/relation_export_worker.rb b/app/workers/bulk_imports/relation_export_worker.rb new file mode 100644 index 00000000000000..c7a222bc44c580 --- /dev/null +++ b/app/workers/bulk_imports/relation_export_worker.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module BulkImports + class RelationExportWorker + include ApplicationWorker + include ExceptionBacktrace + + idempotent! + loggable_arguments 2, 3 + feature_category :importers + sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + + def perform(user_id, exportable_id, exportable_class, relation) + user = User.find(user_id) + exportable = exportable(exportable_id, exportable_class) + + export = exportable.bulk_import_exports.safe_find_or_create_by!(relation: relation) + export.config.validate_user_permissions(user) + export.update!(status_event: 'start', jid: jid) + + RelationExportService.new(export).execute + + export.finish! + rescue => e + Gitlab::ErrorTracking.track_exception(e, exportable_id: exportable&.id, exportable_type: exportable&.class&.name) + + export&.update(status_event: 'fail_op', error: e.class) + end + + private + + def exportable(exportable_id, exportable_class) + case exportable_class + when ::Project.name + ::Project.find(exportable_id) + when ::Group.name + ::Group.find(exportable_id) + else + raise ::Gitlab::ImportExport::Error.unsupported_object_type_error + end + end + end +end diff --git a/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml b/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml new file mode 100644 index 00000000000000..12bf337feec9d6 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml @@ -0,0 +1,5 @@ +--- +title: Add Group partial export API +merge_request: 59295 +author: +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index c0aab89fd4639c..b9a73c32b6b999 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -58,6 +58,8 @@ - 1 - - bulk_imports_pipeline - 1 +- - bulk_imports_relation_export + - 1 - - chaos - 2 - - chat_notification diff --git a/db/migrate/20210414100914_add_bulk_import_exports_table.rb b/db/migrate/20210414100914_add_bulk_import_exports_table.rb new file mode 100644 index 00000000000000..b6bcae9725cddd --- /dev/null +++ b/db/migrate/20210414100914_add_bulk_import_exports_table.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddBulkImportExportsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + create_table :bulk_import_exports do |t| + t.bigint :group_id + t.bigint :project_id + t.timestamps_with_timezone null: false + t.integer :status, limit: 2, null: false, default: 0 + t.text :relation, null: false + t.text :jid, unique: true + t.text :error + end + + add_text_limit :bulk_import_exports, :relation, 255 + add_text_limit :bulk_import_exports, :jid, 255 + add_text_limit :bulk_import_exports, :error, 255 + end + + def down + drop_table :bulk_import_exports + end +end diff --git a/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb b/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb new file mode 100644 index 00000000000000..2f7d3713302bd2 --- /dev/null +++ b/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportsOnProject < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_exports, :projects, column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_exports, column: :project_id + end + end +end diff --git a/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb b/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb new file mode 100644 index 00000000000000..b7172c6987e6f5 --- /dev/null +++ b/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportsOnGroup < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_exports, :namespaces, column: :group_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_exports, column: :group_id + end + end +end diff --git a/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb b/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb new file mode 100644 index 00000000000000..1cbd1cadf5eee5 --- /dev/null +++ b/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AddBulkImportExportsTableIndexes < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + GROUP_INDEX_NAME = 'partial_index_bulk_import_exports_on_group_id_and_relation' + PROJECT_INDEX_NAME = 'partial_index_bulk_import_exports_on_project_id_and_relation' + + def up + add_concurrent_index :bulk_import_exports, + [:group_id, :relation], + unique: true, + where: 'group_id IS NOT NULL', + name: GROUP_INDEX_NAME + + add_concurrent_index :bulk_import_exports, + [:project_id, :relation], + unique: true, + where: 'project_id IS NOT NULL', + name: PROJECT_INDEX_NAME + end + + def down + remove_concurrent_index_by_name(:bulk_import_exports, GROUP_INDEX_NAME) + remove_concurrent_index_by_name(:bulk_import_exports, PROJECT_INDEX_NAME) + end +end diff --git a/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb b/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb new file mode 100644 index 00000000000000..6104a2cabf6df1 --- /dev/null +++ b/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddBulkImportExportUploadsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + create_table :bulk_import_export_uploads do |t| + t.bigint :export_id, null: false + t.datetime_with_timezone :updated_at, null: false + t.text :export_file + end + + add_text_limit :bulk_import_export_uploads, :export_file, 255 + end + + def down + drop_table :bulk_import_export_uploads + end +end diff --git a/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb b/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb new file mode 100644 index 00000000000000..373961510f7ad9 --- /dev/null +++ b/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportUploadsOnExport < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_bulk_import_export_uploads_on_export_id' + + def up + add_concurrent_foreign_key :bulk_import_export_uploads, :bulk_import_exports, column: :export_id, on_delete: :cascade + add_concurrent_index :bulk_import_export_uploads, :export_id, name: INDEX_NAME + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_export_uploads, column: :export_id + end + + remove_concurrent_index_by_name :bulk_import_export_uploads, INDEX_NAME + end +end diff --git a/db/schema_migrations/20210414100914 b/db/schema_migrations/20210414100914 new file mode 100644 index 00000000000000..dcbc93d9987425 --- /dev/null +++ b/db/schema_migrations/20210414100914 @@ -0,0 +1 @@ +4950567ba7071183bc008936e4bbe1391dd0100c5caa2a6821be85dc3d2423fc \ No newline at end of file diff --git a/db/schema_migrations/20210414130017 b/db/schema_migrations/20210414130017 new file mode 100644 index 00000000000000..0eaffe4ddd1578 --- /dev/null +++ b/db/schema_migrations/20210414130017 @@ -0,0 +1 @@ +202409998a03fd29c52e3ee9546ab8ec7aa3c56173ee755e9342f1cc6a5f1f6b \ No newline at end of file diff --git a/db/schema_migrations/20210414130526 b/db/schema_migrations/20210414130526 new file mode 100644 index 00000000000000..ebba5c47f22625 --- /dev/null +++ b/db/schema_migrations/20210414130526 @@ -0,0 +1 @@ +2343decc3abb79b38bcde6aba5a8fd208842096d7fb7a4c51872f66f1a125296 \ No newline at end of file diff --git a/db/schema_migrations/20210414131807 b/db/schema_migrations/20210414131807 new file mode 100644 index 00000000000000..9a7800b86f8f51 --- /dev/null +++ b/db/schema_migrations/20210414131807 @@ -0,0 +1 @@ +4db08c0fecd210b329492596cf029518484d256bdb06efff233b3a38677fd6a6 \ No newline at end of file diff --git a/db/schema_migrations/20210414133310 b/db/schema_migrations/20210414133310 new file mode 100644 index 00000000000000..9a0a224e09bb2f --- /dev/null +++ b/db/schema_migrations/20210414133310 @@ -0,0 +1 @@ +f306cf9553e4bd237cfdff31d5432d4ff44302a923e475c477f76d32ccb4d257 \ No newline at end of file diff --git a/db/schema_migrations/20210419085714 b/db/schema_migrations/20210419085714 new file mode 100644 index 00000000000000..2a463b7f931775 --- /dev/null +++ b/db/schema_migrations/20210419085714 @@ -0,0 +1 @@ +3712daae0be577ebd755ad2d692c5996cb99d719c1596cb454f7d79dd5e941d9 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index fcc36aafc7ebed..dab3291b4fa0c6 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10144,6 +10144,47 @@ CREATE SEQUENCE bulk_import_entities_id_seq ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id; +CREATE TABLE bulk_import_export_uploads ( + id bigint NOT NULL, + export_id bigint NOT NULL, + updated_at timestamp with time zone NOT NULL, + export_file text, + CONSTRAINT check_5add76239d CHECK ((char_length(export_file) <= 255)) +); + +CREATE SEQUENCE bulk_import_export_uploads_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE bulk_import_export_uploads_id_seq OWNED BY bulk_import_export_uploads.id; + +CREATE TABLE bulk_import_exports ( + id bigint NOT NULL, + group_id bigint, + project_id bigint, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + status smallint DEFAULT 0 NOT NULL, + relation text NOT NULL, + jid text, + error text, + CONSTRAINT check_24cb010672 CHECK ((char_length(relation) <= 255)), + CONSTRAINT check_8f0f357334 CHECK ((char_length(error) <= 255)), + CONSTRAINT check_9ee6d14d33 CHECK ((char_length(jid) <= 255)) +); + +CREATE SEQUENCE bulk_import_exports_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE bulk_import_exports_id_seq OWNED BY bulk_import_exports.id; + CREATE TABLE bulk_import_failures ( id bigint NOT NULL, bulk_import_entity_id bigint NOT NULL, @@ -19136,6 +19177,10 @@ ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass); +ALTER TABLE ONLY bulk_import_export_uploads ALTER COLUMN id SET DEFAULT nextval('bulk_import_export_uploads_id_seq'::regclass); + +ALTER TABLE ONLY bulk_import_exports ALTER COLUMN id SET DEFAULT nextval('bulk_import_exports_id_seq'::regclass); + ALTER TABLE ONLY bulk_import_failures ALTER COLUMN id SET DEFAULT nextval('bulk_import_failures_id_seq'::regclass); ALTER TABLE ONLY bulk_import_trackers ALTER COLUMN id SET DEFAULT nextval('bulk_import_trackers_id_seq'::regclass); @@ -20249,6 +20294,12 @@ ALTER TABLE ONLY bulk_import_configurations ALTER TABLE ONLY bulk_import_entities ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY bulk_import_export_uploads + ADD CONSTRAINT bulk_import_export_uploads_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT bulk_import_exports_pkey PRIMARY KEY (id); + ALTER TABLE ONLY bulk_import_failures ADD CONSTRAINT bulk_import_failures_pkey PRIMARY KEY (id); @@ -22060,6 +22111,8 @@ CREATE INDEX index_bulk_import_entities_on_parent_id ON bulk_import_entities USI CREATE INDEX index_bulk_import_entities_on_project_id ON bulk_import_entities USING btree (project_id); +CREATE INDEX index_bulk_import_export_uploads_on_export_id ON bulk_import_export_uploads USING btree (export_id); + CREATE INDEX index_bulk_import_failures_on_bulk_import_entity_id ON bulk_import_failures USING btree (bulk_import_entity_id); CREATE INDEX index_bulk_import_failures_on_correlation_id_value ON bulk_import_failures USING btree (correlation_id_value); @@ -24354,6 +24407,10 @@ CREATE INDEX packages_packages_needs_verification ON packages_package_files USIN CREATE INDEX packages_packages_pending_verification ON packages_package_files USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); +CREATE UNIQUE INDEX partial_index_bulk_import_exports_on_group_id_and_relation ON bulk_import_exports USING btree (group_id, relation) WHERE (group_id IS NOT NULL); + +CREATE UNIQUE INDEX partial_index_bulk_import_exports_on_project_id_and_relation ON bulk_import_exports USING btree (project_id, relation) WHERE (project_id IS NOT NULL); + CREATE INDEX partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs ON ci_builds USING btree (scheduled_at) WHERE ((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text)); CREATE INDEX partial_index_deployments_for_legacy_successful_deployments ON deployments USING btree (id) WHERE ((finished_at IS NULL) AND (status = 2)); @@ -24833,6 +24890,9 @@ ALTER TABLE ONLY sprints ALTER TABLE ONLY push_event_payloads ADD CONSTRAINT fk_36c74129da FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT fk_39c726d3b5 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY ci_builds ADD CONSTRAINT fk_3a9eaa254d FOREIGN KEY (stage_id) REFERENCES ci_stages(id) ON DELETE CASCADE; @@ -25028,6 +25088,9 @@ ALTER TABLE ONLY issues ALTER TABLE ONLY protected_branch_merge_access_levels ADD CONSTRAINT fk_8a3072ccb3 FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT fk_8c6f33cebe FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY releases ADD CONSTRAINT fk_8e4456f90f FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; @@ -25283,6 +25346,9 @@ ALTER TABLE ONLY analytics_devops_adoption_segment_selections ALTER TABLE ONLY issues ADD CONSTRAINT fk_df75a7c8b8 FOREIGN KEY (promoted_to_epic_id) REFERENCES epics(id) ON DELETE SET NULL; +ALTER TABLE ONLY bulk_import_export_uploads + ADD CONSTRAINT fk_dfbfb45eca FOREIGN KEY (export_id) REFERENCES bulk_import_exports(id) ON DELETE CASCADE; + ALTER TABLE ONLY experiment_subjects ADD CONSTRAINT fk_dfc3e211d4 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/lib/api/entities/bulk_imports/export_status.rb b/lib/api/entities/bulk_imports/export_status.rb new file mode 100644 index 00000000000000..c9c7f34a16a47b --- /dev/null +++ b/lib/api/entities/bulk_imports/export_status.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class ExportStatus < Grape::Entity + expose :relation + expose :status + expose :error + expose :updated_at + end + end + end +end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 29ffbea687a3ba..9c7f04352f4882 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -43,6 +43,43 @@ class GroupExport < ::API::Base render_api_error!(message: 'Group export could not be started.') end end + + desc 'Start relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + post ':id/export_relations' do + response = ::BulkImports::ExportService.new(exportable: user_group, user: current_user).execute + + if response.success? + accepted! + else + render_api_error!(message: 'Group relations export could not be started.') + end + end + + desc 'Download relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + params do + requires :relation, type: String, desc: 'Group relation name' + end + get ':id/export_relations/download' do + export = user_group.bulk_import_exports.find_by_relation(params[:relation]) + file = export&.upload&.export_file + + if file + present_carrierwave_file!(file) + else + render_api_error!('404 Not found', 404) + end + end + + desc 'Relations export status' do + detail 'This feature was introduced in GitLab 13.12' + end + get ':id/export_relations/status' do + present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index c4867746b0fb13..96915c19cee340 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -99,6 +99,23 @@ def legacy_group_config_file def group_config_file Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end + + def top_level_relations(exportable_class) + config = Gitlab::ImportExport::Config.new(config: import_export_yaml(exportable_class)).to_h + + config.dig(:tree, exportable_class.to_s.downcase.to_sym).keys.map(&:to_s) + end + + def import_export_yaml(exportable_class) + case exportable_class + when ::Project.name + config_file + when ::Group.name + group_config_file + else + raise Gitlab::ImportExport::Error.unsupported_object_type_error + end + end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f8769e261d7bf..ace9d83dc9a084 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,19 @@ def untar_zxf(archive:, dir:) untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def gzip(dir:, filename:) + filepath = File.join(dir, filename) + cmd = %W(gzip #{filepath}) + + _, status = Gitlab::Popen.popen(cmd) + + if status == 0 + status + else + raise Gitlab::ImportExport::Error.file_compression_error + end + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) FileUtils.chmod(DEFAULT_DIR_MODE, path) diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index f11b7a0a298a2f..5da2b10004e2d8 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -3,12 +3,20 @@ module Gitlab module ImportExport class Error < StandardError - def self.permission_error(user, importable) + def self.permission_error(user, object) self.new( "User with ID: %s does not have required permissions for %s: %s with ID: %s" % - [user.id, importable.class.name, importable.name, importable.id] + [user.id, object.class.name, object.name, object.id] ) end + + def self.unsupported_object_type_error + self.new('Unknown object type') + end + + def self.file_compression_error + self.new('File compression failed') + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 05b7679e0ff422..a8b3554ff2c06f 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -38,10 +38,6 @@ def execute end end - private - - attr_reader :json_writer, :relations_schema, :exportable - def serialize_root attributes = exportable.as_json( relations_schema.merge(include: nil, preloads: nil)) @@ -64,6 +60,10 @@ def serialize_relation(definition) end end + private + + attr_reader :json_writer, :relations_schema, :exportable + def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) diff --git a/spec/factories/bulk_import/export_uploads.rb b/spec/factories/bulk_import/export_uploads.rb new file mode 100644 index 00000000000000..9f03498b9d978f --- /dev/null +++ b/spec/factories/bulk_import/export_uploads.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :bulk_import_export_upload, class: 'BulkImports::ExportUpload' do + export { association(:bulk_import_export) } + end +end diff --git a/spec/factories/bulk_import/exports.rb b/spec/factories/bulk_import/exports.rb new file mode 100644 index 00000000000000..dd8831ce33a9ed --- /dev/null +++ b/spec/factories/bulk_import/exports.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :bulk_import_export, class: 'BulkImports::Export', traits: %i[started] do + group + relation { 'labels' } + + trait :started do + status { 0 } + + sequence(:jid) { |n| "bulk_import_export_#{n}" } + end + + trait :finished do + status { 1 } + + sequence(:jid) { |n| "bulk_import_export_#{n}" } + end + + trait :failed do + status { -1 } + end + end +end diff --git a/spec/fixtures/bulk_imports/labels.ndjson.gz b/spec/fixtures/bulk_imports/labels.ndjson.gz new file mode 100644 index 0000000000000000000000000000000000000000..6bb10a533467bda24c18ee1932fbf36d3cd5f319 GIT binary patch literal 202 zcmb2|=HS@IS&_iNoRgT8np3Qomy%VSpU2SKZ^(VfK%lj}zI9Hvw&&KhXSH5Dda=Z# z^QGp*8!cbBx_;dEW#Y@b_v6RC$GgQ}@Uc9%VZM|i!1Csa0CY^S7NI~%QDwJcCe z<=ky)t*qpg^1Ylr?w&WcPGXAK(DXcK=I5gGSqHrjh82hD@-4L7^-jO