diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb new file mode 100644 index 0000000000000000000000000000000000000000..f16ef4c0709ce1f5417e6275071f5cf083652529 --- /dev/null +++ b/app/models/bulk_imports/export.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module BulkImports + class Export < ApplicationRecord + include Gitlab::Utils::StrongMemoize + + self.table_name = 'bulk_import_exports' + + belongs_to :project, optional: true + belongs_to :group, optional: true + + has_one :upload, class_name: 'BulkImports::ExportUpload' + + validates :project, presence: true, unless: :group + validates :group, presence: true, unless: :project + validates :relation, :status, presence: true + + validate :exportable_relation? + + state_machine :status, initial: :started do + state :started, value: 0 + state :finished, value: 1 + state :failed, value: -1 + + event :start do + transition started: :started + transition finished: :started + transition failed: :started + end + + event :finish do + transition started: :finished + transition failed: :failed + end + + event :fail_op do + transition any => :failed + end + end + + def exportable_relation? + return unless exportable + + allowed_relations = ::Gitlab::ImportExport.top_level_relations(exportable.class.name) + + errors.add(:relation, 'Unsupported exportable relation') unless allowed_relations.include?(relation) + end + + def exportable + project || group + end + + def relation_definition + config.exportable_tree[:include].find { |include| include[relation.to_sym] } + end + + def config + strong_memoize(:config) do + case exportable + when ::Project + Exports::ProjectConfig.new(exportable) + when ::Group + Exports::GroupConfig.new(exportable) + end + end + end + end +end diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb new file mode 100644 index 0000000000000000000000000000000000000000..c1884317c41eb72e6a36f23f939ec65fc7e7ba5a --- /dev/null +++ b/app/models/bulk_imports/export_upload.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module BulkImports + class ExportUpload < ApplicationRecord + self.table_name = 'bulk_import_export_uploads' + + include WithUploads + include ObjectStorage::BackgroundMove + + belongs_to :export, class_name: 'BulkImports::Export' + + mount_uploader :export_file, ExportUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + end +end diff --git a/app/models/bulk_imports/exports/config.rb b/app/models/bulk_imports/exports/config.rb new file mode 100644 index 0000000000000000000000000000000000000000..b0f7a4c62175b6560fb367f4747d27e48600eda3 --- /dev/null +++ b/app/models/bulk_imports/exports/config.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class Config + include Gitlab::Utils::StrongMemoize + + attr_reader :exportable + + def initialize(exportable) + @exportable = exportable + end + + def exportable_tree + attributes_finder.find_root(exportable_class_sym) + end + + def validate_user_permissions(user) + user.can?(ability, exportable) || (raise ::Gitlab::ImportExport::Error.permission_error(user, exportable)) + end + + private + + def attributes_finder + strong_memoize(:attributes_finder) do + ::Gitlab::ImportExport::AttributesFinder.new(config: import_export_config) + end + end + + def import_export_config + ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h + end + + def exportable_class + @exportable_class ||= exportable.class + end + + def exportable_class_sym + @exportable_class_sym ||= exportable_class.to_s.downcase.to_sym + end + + def import_export_yaml + raise NotImplementedError + end + + def ability + raise NotImplementedError + end + end + end +end diff --git a/app/models/bulk_imports/exports/group_config.rb b/app/models/bulk_imports/exports/group_config.rb new file mode 100644 index 0000000000000000000000000000000000000000..f9f8df171ea9ba4bb170bfd5aad97bfcc4e70382 --- /dev/null +++ b/app/models/bulk_imports/exports/group_config.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class GroupConfig < Config + def import_export_yaml + ::Gitlab::ImportExport.group_config_file + end + + def export_path + exportable.full_path + end + + def ability + :admin_group + end + end + end +end diff --git a/app/models/bulk_imports/exports/project_config.rb b/app/models/bulk_imports/exports/project_config.rb new file mode 100644 index 0000000000000000000000000000000000000000..253ec5c590031a5e27e8e4a7b03d07c5ce17be82 --- /dev/null +++ b/app/models/bulk_imports/exports/project_config.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module BulkImports + module Exports + class ProjectConfig < Config + def export_path + exportable.disk_path + end + + private + + def import_export_yaml + ::Gitlab::ImportExport.config_file + end + + def ability + :admin_project + end + end + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 60a0e27428d6f35e75d3856e0216c21fde1d878d..78d8be7e9c58c4f1d3ad39c928afe4c54870d676 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -67,6 +67,8 @@ class Group < Namespace has_one :import_state, class_name: 'GroupImportState', inverse_of: :group + has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :group + has_many :group_deploy_keys_groups, inverse_of: :group has_many :group_deploy_keys, through: :group_deploy_keys_groups has_many :group_deploy_tokens diff --git a/app/services/bulk_imports/export_service.rb b/app/services/bulk_imports/export_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..4e67b1fca6e769800587097a62c75d5d39c43d3f --- /dev/null +++ b/app/services/bulk_imports/export_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + class ExportService + attr_reader :exportable, :current_user + + def initialize(exportable:, user:) + @exportable = exportable + @current_user = user + end + + def execute + ::Gitlab::ImportExport.top_level_relations(exportable.class.name).each do |relation| + RelationExportWorker.perform_async(current_user.id, exportable.id, exportable.class.name, relation) + end + + ServiceResponse.success + rescue => e + ServiceResponse.error( + message: e.class, + http_status: :unprocessable_entity + ) + end + end +end diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..b63ad2256c6cdb3de9f6af09c37ca743a4d7442c --- /dev/null +++ b/app/services/bulk_imports/relation_export_service.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module BulkImports + class RelationExportService + include Gitlab::ImportExport::CommandLineUtil + + def initialize(export) + @export = export + @exportable = export.exportable + @config = export.config + @export_path = @config.export_path + @exportable_tree = @config.exportable_tree + end + + def execute + remove_existing_export_file + serialize_relation + compress_exported_relation + upload_compressed_file + ensure + FileUtils.rm_rf(export_path) if export_path + end + + private + + attr_reader :current_user, :export, :exportable, :export_path, :exportable_tree + + def remove_existing_export_file + upload = export.upload + + return unless upload + return unless upload.export_file&.file + + upload.remove_export_file! + upload.save! + end + + # rubocop: disable CodeReuse/Serializer + def serializer + @serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new( + exportable, + exportable_tree, + json_writer, + exportable_path: '' + ) + end + # rubocop: enable CodeReuse/Serializer + + def json_writer + @json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path) + end + + def serialize_relation + serializer.serialize_relation(export.relation_definition) + end + + def compress_exported_relation + gzip(dir: export_path, filename: ndjson_filename) + end + + def upload_compressed_file + compressed_filename = File.join(export_path, "#{ndjson_filename}.gz") + upload = ExportUpload.find_or_initialize_by(export_id: export.id) # rubocop: disable CodeReuse/ActiveRecord + + File.open(compressed_filename) { |file| upload.export_file = file } + + upload.save! + end + + def ndjson_filename + "#{export.relation}.ndjson" + end + end +end diff --git a/app/uploaders/bulk_imports/export_uploader.rb b/app/uploaders/bulk_imports/export_uploader.rb new file mode 100644 index 0000000000000000000000000000000000000000..356e5ce028e5792a45b93c9bc467cddc9a4cd738 --- /dev/null +++ b/app/uploaders/bulk_imports/export_uploader.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module BulkImports + class ExportUploader < ImportExportUploader + EXTENSION_WHITELIST = %w[ndjson.gz].freeze + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index fa6ea54e342ec8c845288e37a9047ed8a8ce0ca5..9069dd56656ddb3754d204c7c26304048fdb7fc4 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1563,6 +1563,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: bulk_imports_relation_export + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: chat_notification :feature_category: :chatops :has_external_dependencies: true diff --git a/app/workers/bulk_imports/relation_export_worker.rb b/app/workers/bulk_imports/relation_export_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..c7a222bc44c5806a068bbbff3ba5f764fcda17d8 --- /dev/null +++ b/app/workers/bulk_imports/relation_export_worker.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module BulkImports + class RelationExportWorker + include ApplicationWorker + include ExceptionBacktrace + + idempotent! + loggable_arguments 2, 3 + feature_category :importers + sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + + def perform(user_id, exportable_id, exportable_class, relation) + user = User.find(user_id) + exportable = exportable(exportable_id, exportable_class) + + export = exportable.bulk_import_exports.safe_find_or_create_by!(relation: relation) + export.config.validate_user_permissions(user) + export.update!(status_event: 'start', jid: jid) + + RelationExportService.new(export).execute + + export.finish! + rescue => e + Gitlab::ErrorTracking.track_exception(e, exportable_id: exportable&.id, exportable_type: exportable&.class&.name) + + export&.update(status_event: 'fail_op', error: e.class) + end + + private + + def exportable(exportable_id, exportable_class) + case exportable_class + when ::Project.name + ::Project.find(exportable_id) + when ::Group.name + ::Group.find(exportable_id) + else + raise ::Gitlab::ImportExport::Error.unsupported_object_type_error + end + end + end +end diff --git a/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml b/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml new file mode 100644 index 0000000000000000000000000000000000000000..12bf337feec9d659cfabe8c5dae1ae38bb199e00 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-bulk_import_group_exports.yml @@ -0,0 +1,5 @@ +--- +title: Add Group partial export API +merge_request: 59295 +author: +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index c0aab89fd4639ce2fc4c7ddae22299784f12bad0..b9a73c32b6b9999ff6abe397b14a3ed7476784d2 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -58,6 +58,8 @@ - 1 - - bulk_imports_pipeline - 1 +- - bulk_imports_relation_export + - 1 - - chaos - 2 - - chat_notification diff --git a/db/migrate/20210414100914_add_bulk_import_exports_table.rb b/db/migrate/20210414100914_add_bulk_import_exports_table.rb new file mode 100644 index 0000000000000000000000000000000000000000..b6bcae9725cddd28a995016a78d13f176a797c1f --- /dev/null +++ b/db/migrate/20210414100914_add_bulk_import_exports_table.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddBulkImportExportsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + create_table :bulk_import_exports do |t| + t.bigint :group_id + t.bigint :project_id + t.timestamps_with_timezone null: false + t.integer :status, limit: 2, null: false, default: 0 + t.text :relation, null: false + t.text :jid, unique: true + t.text :error + end + + add_text_limit :bulk_import_exports, :relation, 255 + add_text_limit :bulk_import_exports, :jid, 255 + add_text_limit :bulk_import_exports, :error, 255 + end + + def down + drop_table :bulk_import_exports + end +end diff --git a/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb b/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb new file mode 100644 index 0000000000000000000000000000000000000000..2f7d3713302bd2dbcabd63cb2880f26a3ce9de0e --- /dev/null +++ b/db/migrate/20210414130017_add_foreign_key_to_bulk_import_exports_on_project.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportsOnProject < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_exports, :projects, column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_exports, column: :project_id + end + end +end diff --git a/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb b/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb new file mode 100644 index 0000000000000000000000000000000000000000..b7172c6987e6f5b139661eaa68ad88bc76aa4c2d --- /dev/null +++ b/db/migrate/20210414130526_add_foreign_key_to_bulk_import_exports_on_group.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportsOnGroup < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_exports, :namespaces, column: :group_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_exports, column: :group_id + end + end +end diff --git a/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb b/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb new file mode 100644 index 0000000000000000000000000000000000000000..1cbd1cadf5eee58ce871df90cea94566dc3a09f5 --- /dev/null +++ b/db/migrate/20210414131807_add_bulk_import_exports_table_indexes.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AddBulkImportExportsTableIndexes < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + GROUP_INDEX_NAME = 'partial_index_bulk_import_exports_on_group_id_and_relation' + PROJECT_INDEX_NAME = 'partial_index_bulk_import_exports_on_project_id_and_relation' + + def up + add_concurrent_index :bulk_import_exports, + [:group_id, :relation], + unique: true, + where: 'group_id IS NOT NULL', + name: GROUP_INDEX_NAME + + add_concurrent_index :bulk_import_exports, + [:project_id, :relation], + unique: true, + where: 'project_id IS NOT NULL', + name: PROJECT_INDEX_NAME + end + + def down + remove_concurrent_index_by_name(:bulk_import_exports, GROUP_INDEX_NAME) + remove_concurrent_index_by_name(:bulk_import_exports, PROJECT_INDEX_NAME) + end +end diff --git a/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb b/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb new file mode 100644 index 0000000000000000000000000000000000000000..6104a2cabf6df16d4c429c663a5cb07d2d62e2b7 --- /dev/null +++ b/db/migrate/20210414133310_add_bulk_import_export_uploads_table.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddBulkImportExportUploadsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + create_table :bulk_import_export_uploads do |t| + t.bigint :export_id, null: false + t.datetime_with_timezone :updated_at, null: false + t.text :export_file + end + + add_text_limit :bulk_import_export_uploads, :export_file, 255 + end + + def down + drop_table :bulk_import_export_uploads + end +end diff --git a/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb b/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb new file mode 100644 index 0000000000000000000000000000000000000000..373961510f7ad96aaf78a04dce36d585d178237e --- /dev/null +++ b/db/migrate/20210419085714_add_foreign_key_to_bulk_import_export_uploads_on_export.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddForeignKeyToBulkImportExportUploadsOnExport < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_bulk_import_export_uploads_on_export_id' + + def up + add_concurrent_foreign_key :bulk_import_export_uploads, :bulk_import_exports, column: :export_id, on_delete: :cascade + add_concurrent_index :bulk_import_export_uploads, :export_id, name: INDEX_NAME + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_export_uploads, column: :export_id + end + + remove_concurrent_index_by_name :bulk_import_export_uploads, INDEX_NAME + end +end diff --git a/db/schema_migrations/20210414100914 b/db/schema_migrations/20210414100914 new file mode 100644 index 0000000000000000000000000000000000000000..dcbc93d9987425928534e2c763c689e845d02388 --- /dev/null +++ b/db/schema_migrations/20210414100914 @@ -0,0 +1 @@ +4950567ba7071183bc008936e4bbe1391dd0100c5caa2a6821be85dc3d2423fc \ No newline at end of file diff --git a/db/schema_migrations/20210414130017 b/db/schema_migrations/20210414130017 new file mode 100644 index 0000000000000000000000000000000000000000..0eaffe4ddd157837619597edf6cb81d9fa0fc761 --- /dev/null +++ b/db/schema_migrations/20210414130017 @@ -0,0 +1 @@ +202409998a03fd29c52e3ee9546ab8ec7aa3c56173ee755e9342f1cc6a5f1f6b \ No newline at end of file diff --git a/db/schema_migrations/20210414130526 b/db/schema_migrations/20210414130526 new file mode 100644 index 0000000000000000000000000000000000000000..ebba5c47f22625b5abe2b29b5e87c767eac08a5d --- /dev/null +++ b/db/schema_migrations/20210414130526 @@ -0,0 +1 @@ +2343decc3abb79b38bcde6aba5a8fd208842096d7fb7a4c51872f66f1a125296 \ No newline at end of file diff --git a/db/schema_migrations/20210414131807 b/db/schema_migrations/20210414131807 new file mode 100644 index 0000000000000000000000000000000000000000..9a7800b86f8f5194cc2cac7e8dad46700bdc5be9 --- /dev/null +++ b/db/schema_migrations/20210414131807 @@ -0,0 +1 @@ +4db08c0fecd210b329492596cf029518484d256bdb06efff233b3a38677fd6a6 \ No newline at end of file diff --git a/db/schema_migrations/20210414133310 b/db/schema_migrations/20210414133310 new file mode 100644 index 0000000000000000000000000000000000000000..9a0a224e09bb2fea00db67faf0143aa0a3fb83ca --- /dev/null +++ b/db/schema_migrations/20210414133310 @@ -0,0 +1 @@ +f306cf9553e4bd237cfdff31d5432d4ff44302a923e475c477f76d32ccb4d257 \ No newline at end of file diff --git a/db/schema_migrations/20210419085714 b/db/schema_migrations/20210419085714 new file mode 100644 index 0000000000000000000000000000000000000000..2a463b7f93177575c6344dce1f6c958b3a88aa4d --- /dev/null +++ b/db/schema_migrations/20210419085714 @@ -0,0 +1 @@ +3712daae0be577ebd755ad2d692c5996cb99d719c1596cb454f7d79dd5e941d9 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index fcc36aafc7ebed2e3b9c5905d7d6c7c285d6d85f..dab3291b4fa0c630e3c2bac29b1001df1b205929 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10144,6 +10144,47 @@ CREATE SEQUENCE bulk_import_entities_id_seq ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id; +CREATE TABLE bulk_import_export_uploads ( + id bigint NOT NULL, + export_id bigint NOT NULL, + updated_at timestamp with time zone NOT NULL, + export_file text, + CONSTRAINT check_5add76239d CHECK ((char_length(export_file) <= 255)) +); + +CREATE SEQUENCE bulk_import_export_uploads_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE bulk_import_export_uploads_id_seq OWNED BY bulk_import_export_uploads.id; + +CREATE TABLE bulk_import_exports ( + id bigint NOT NULL, + group_id bigint, + project_id bigint, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + status smallint DEFAULT 0 NOT NULL, + relation text NOT NULL, + jid text, + error text, + CONSTRAINT check_24cb010672 CHECK ((char_length(relation) <= 255)), + CONSTRAINT check_8f0f357334 CHECK ((char_length(error) <= 255)), + CONSTRAINT check_9ee6d14d33 CHECK ((char_length(jid) <= 255)) +); + +CREATE SEQUENCE bulk_import_exports_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE bulk_import_exports_id_seq OWNED BY bulk_import_exports.id; + CREATE TABLE bulk_import_failures ( id bigint NOT NULL, bulk_import_entity_id bigint NOT NULL, @@ -19136,6 +19177,10 @@ ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass); +ALTER TABLE ONLY bulk_import_export_uploads ALTER COLUMN id SET DEFAULT nextval('bulk_import_export_uploads_id_seq'::regclass); + +ALTER TABLE ONLY bulk_import_exports ALTER COLUMN id SET DEFAULT nextval('bulk_import_exports_id_seq'::regclass); + ALTER TABLE ONLY bulk_import_failures ALTER COLUMN id SET DEFAULT nextval('bulk_import_failures_id_seq'::regclass); ALTER TABLE ONLY bulk_import_trackers ALTER COLUMN id SET DEFAULT nextval('bulk_import_trackers_id_seq'::regclass); @@ -20249,6 +20294,12 @@ ALTER TABLE ONLY bulk_import_configurations ALTER TABLE ONLY bulk_import_entities ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY bulk_import_export_uploads + ADD CONSTRAINT bulk_import_export_uploads_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT bulk_import_exports_pkey PRIMARY KEY (id); + ALTER TABLE ONLY bulk_import_failures ADD CONSTRAINT bulk_import_failures_pkey PRIMARY KEY (id); @@ -22060,6 +22111,8 @@ CREATE INDEX index_bulk_import_entities_on_parent_id ON bulk_import_entities USI CREATE INDEX index_bulk_import_entities_on_project_id ON bulk_import_entities USING btree (project_id); +CREATE INDEX index_bulk_import_export_uploads_on_export_id ON bulk_import_export_uploads USING btree (export_id); + CREATE INDEX index_bulk_import_failures_on_bulk_import_entity_id ON bulk_import_failures USING btree (bulk_import_entity_id); CREATE INDEX index_bulk_import_failures_on_correlation_id_value ON bulk_import_failures USING btree (correlation_id_value); @@ -24354,6 +24407,10 @@ CREATE INDEX packages_packages_needs_verification ON packages_package_files USIN CREATE INDEX packages_packages_pending_verification ON packages_package_files USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); +CREATE UNIQUE INDEX partial_index_bulk_import_exports_on_group_id_and_relation ON bulk_import_exports USING btree (group_id, relation) WHERE (group_id IS NOT NULL); + +CREATE UNIQUE INDEX partial_index_bulk_import_exports_on_project_id_and_relation ON bulk_import_exports USING btree (project_id, relation) WHERE (project_id IS NOT NULL); + CREATE INDEX partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs ON ci_builds USING btree (scheduled_at) WHERE ((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text)); CREATE INDEX partial_index_deployments_for_legacy_successful_deployments ON deployments USING btree (id) WHERE ((finished_at IS NULL) AND (status = 2)); @@ -24833,6 +24890,9 @@ ALTER TABLE ONLY sprints ALTER TABLE ONLY push_event_payloads ADD CONSTRAINT fk_36c74129da FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT fk_39c726d3b5 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY ci_builds ADD CONSTRAINT fk_3a9eaa254d FOREIGN KEY (stage_id) REFERENCES ci_stages(id) ON DELETE CASCADE; @@ -25028,6 +25088,9 @@ ALTER TABLE ONLY issues ALTER TABLE ONLY protected_branch_merge_access_levels ADD CONSTRAINT fk_8a3072ccb3 FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_exports + ADD CONSTRAINT fk_8c6f33cebe FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY releases ADD CONSTRAINT fk_8e4456f90f FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; @@ -25283,6 +25346,9 @@ ALTER TABLE ONLY analytics_devops_adoption_segment_selections ALTER TABLE ONLY issues ADD CONSTRAINT fk_df75a7c8b8 FOREIGN KEY (promoted_to_epic_id) REFERENCES epics(id) ON DELETE SET NULL; +ALTER TABLE ONLY bulk_import_export_uploads + ADD CONSTRAINT fk_dfbfb45eca FOREIGN KEY (export_id) REFERENCES bulk_import_exports(id) ON DELETE CASCADE; + ALTER TABLE ONLY experiment_subjects ADD CONSTRAINT fk_dfc3e211d4 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/lib/api/entities/bulk_imports/export_status.rb b/lib/api/entities/bulk_imports/export_status.rb new file mode 100644 index 0000000000000000000000000000000000000000..c9c7f34a16a47b04533c3bb42b0f358abb086be5 --- /dev/null +++ b/lib/api/entities/bulk_imports/export_status.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class ExportStatus < Grape::Entity + expose :relation + expose :status + expose :error + expose :updated_at + end + end + end +end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 29ffbea687a3ba167f35570303242e4733fd8f8d..9c7f04352f48827ff48de9ab6d5cf3fca29765cc 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -43,6 +43,43 @@ class GroupExport < ::API::Base render_api_error!(message: 'Group export could not be started.') end end + + desc 'Start relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + post ':id/export_relations' do + response = ::BulkImports::ExportService.new(exportable: user_group, user: current_user).execute + + if response.success? + accepted! + else + render_api_error!(message: 'Group relations export could not be started.') + end + end + + desc 'Download relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + params do + requires :relation, type: String, desc: 'Group relation name' + end + get ':id/export_relations/download' do + export = user_group.bulk_import_exports.find_by_relation(params[:relation]) + file = export&.upload&.export_file + + if file + present_carrierwave_file!(file) + else + render_api_error!('404 Not found', 404) + end + end + + desc 'Relations export status' do + detail 'This feature was introduced in GitLab 13.12' + end + get ':id/export_relations/status' do + present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index c4867746b0fb139aa00e4c7ac25b2589bebc20c8..96915c19cee340a10ea314e66ee63df5b6226a53 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -99,6 +99,23 @@ def legacy_group_config_file def group_config_file Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end + + def top_level_relations(exportable_class) + config = Gitlab::ImportExport::Config.new(config: import_export_yaml(exportable_class)).to_h + + config.dig(:tree, exportable_class.to_s.downcase.to_sym).keys.map(&:to_s) + end + + def import_export_yaml(exportable_class) + case exportable_class + when ::Project.name + config_file + when ::Group.name + group_config_file + else + raise Gitlab::ImportExport::Error.unsupported_object_type_error + end + end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f8769e261d7bf1b28c6f74b9d5579a1e4308133..ace9d83dc9a0843d1afcc5bd154bac816bbad806 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,19 @@ def untar_zxf(archive:, dir:) untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def gzip(dir:, filename:) + filepath = File.join(dir, filename) + cmd = %W(gzip #{filepath}) + + _, status = Gitlab::Popen.popen(cmd) + + if status == 0 + status + else + raise Gitlab::ImportExport::Error.file_compression_error + end + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) FileUtils.chmod(DEFAULT_DIR_MODE, path) diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index f11b7a0a298a2f4d32f8a10adb2c09dd7096d08b..5da2b10004e2d8da78637821fe24a4244f8e51e6 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -3,12 +3,20 @@ module Gitlab module ImportExport class Error < StandardError - def self.permission_error(user, importable) + def self.permission_error(user, object) self.new( "User with ID: %s does not have required permissions for %s: %s with ID: %s" % - [user.id, importable.class.name, importable.name, importable.id] + [user.id, object.class.name, object.name, object.id] ) end + + def self.unsupported_object_type_error + self.new('Unknown object type') + end + + def self.file_compression_error + self.new('File compression failed') + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 05b7679e0ff4222afe60c822c115066605ff2c1f..a8b3554ff2c06f698205ef78f4f597df85d460cf 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -38,10 +38,6 @@ def execute end end - private - - attr_reader :json_writer, :relations_schema, :exportable - def serialize_root attributes = exportable.as_json( relations_schema.merge(include: nil, preloads: nil)) @@ -64,6 +60,10 @@ def serialize_relation(definition) end end + private + + attr_reader :json_writer, :relations_schema, :exportable + def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) diff --git a/spec/factories/bulk_import/export_uploads.rb b/spec/factories/bulk_import/export_uploads.rb new file mode 100644 index 0000000000000000000000000000000000000000..9f03498b9d978fd6751aed1835eb5e7cd9a410d9 --- /dev/null +++ b/spec/factories/bulk_import/export_uploads.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :bulk_import_export_upload, class: 'BulkImports::ExportUpload' do + export { association(:bulk_import_export) } + end +end diff --git a/spec/factories/bulk_import/exports.rb b/spec/factories/bulk_import/exports.rb new file mode 100644 index 0000000000000000000000000000000000000000..dd8831ce33a9ed3fad28d23fcb693e0b684fd961 --- /dev/null +++ b/spec/factories/bulk_import/exports.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :bulk_import_export, class: 'BulkImports::Export', traits: %i[started] do + group + relation { 'labels' } + + trait :started do + status { 0 } + + sequence(:jid) { |n| "bulk_import_export_#{n}" } + end + + trait :finished do + status { 1 } + + sequence(:jid) { |n| "bulk_import_export_#{n}" } + end + + trait :failed do + status { -1 } + end + end +end diff --git a/spec/fixtures/bulk_imports/labels.ndjson.gz b/spec/fixtures/bulk_imports/labels.ndjson.gz new file mode 100644 index 0000000000000000000000000000000000000000..6bb10a533467bda24c18ee1932fbf36d3cd5f319 Binary files /dev/null and b/spec/fixtures/bulk_imports/labels.ndjson.gz differ diff --git a/spec/lib/api/entities/bulk_imports/export_status_spec.rb b/spec/lib/api/entities/bulk_imports/export_status_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..7d79e3720275cb69cd5031f6eb51d1d4076a9f2d --- /dev/null +++ b/spec/lib/api/entities/bulk_imports/export_status_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Entities::BulkImports::ExportStatus do + let_it_be(:export) { create(:bulk_import_export) } + + let(:entity) { described_class.new(export, request: double) } + + subject { entity.as_json } + + it 'has the correct attributes' do + expect(subject).to eq({ + relation: export.relation, + status: export.status, + error: export.error, + updated_at: export.updated_at + }) + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 5d1e3c79474efaf2930e3b5318aa4f2275c06f1e..1ed7549f824d67780492d747e7c992136910c517 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -745,3 +745,5 @@ issuable_sla: - issue push_rule: - group +bulk_import_export: + - group diff --git a/spec/lib/gitlab/import_export/import_export_spec.rb b/spec/lib/gitlab/import_export/import_export_spec.rb index 87757b07572e847af94a132350a55c813b59bf94..4fe345bddd9883da887ca2e04b22cb774b81e361 100644 --- a/spec/lib/gitlab/import_export/import_export_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_spec.rb @@ -29,4 +29,49 @@ expect(described_class.snippet_repo_bundle_filename_for(snippet)).to eq "#{snippet.hexdigest}.bundle" end end + + describe '.top_level_relations' do + shared_examples 'returns a list of top level relations' do + it 'returns a list of top level relations' do + config = ::Gitlab::ImportExport::Config.new(config: yaml).to_h + expected = config.dig(:tree, exportable.class.name.downcase.to_sym).keys.map(&:to_s) + + expect(described_class.top_level_relations(exportable.class.name)).to eq(expected) + end + end + + context 'when exportable is group' do + let(:exportable) { create(:group) } + let(:yaml) { ::Gitlab::ImportExport.group_config_file } + + include_examples 'returns a list of top level relations' + end + + context 'when exportable is project' do + let(:exportable) { create(:project) } + let(:yaml) { ::Gitlab::ImportExport.config_file } + + include_examples 'returns a list of top level relations' + end + end + + describe '.import_export_yaml' do + context 'when exportable is group' do + it 'returns group config filepath' do + expect(described_class.import_export_yaml('Group')).to eq(::Gitlab::ImportExport.group_config_file) + end + end + + context 'when exportable is project' do + it 'returns project config filepath' do + expect(described_class.import_export_yaml('Project')).to eq(::Gitlab::ImportExport.config_file) + end + end + + context 'when exportable is unsupported' do + it 'raises unsupported object type error' do + expect { described_class.import_export_yaml('unsupported') }.to raise_error(::Gitlab::ImportExport::Error) + end + end + end end diff --git a/spec/models/bulk_imports/export_spec.rb b/spec/models/bulk_imports/export_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..26d25e6901e00082fe7d2afe6dbcdc413c198691 --- /dev/null +++ b/spec/models/bulk_imports/export_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Export, type: :model do + describe 'associations' do + it { is_expected.to belong_to(:group) } + it { is_expected.to belong_to(:project) } + it { is_expected.to have_one(:upload) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:relation) } + it { is_expected.to validate_presence_of(:status) } + + context 'when not associated with a group or project' do + it 'is invalid' do + export = build(:bulk_import_export, group: nil, project: nil) + + expect(export).not_to be_valid + end + end + + context 'when associated with a group' do + it 'is valid' do + export = build(:bulk_import_export, group: build(:group), project: nil) + + expect(export).to be_valid + end + end + + context 'when associated with a project' do + it 'is valid' do + export = build(:bulk_import_export, group: nil, project: build(:project)) + + expect(export).to be_valid + end + end + + context 'when relation is invalid' do + it 'is invalid' do + export = build(:bulk_import_export, relation: 'unsupported') + + expect(export).not_to be_valid + expect(export.errors).to include(:relation) + end + end + end + + describe '#exportable' do + context 'when associated with project' do + it 'returns project' do + export = create(:bulk_import_export, project: create(:project), group: nil) + + expect(export.exportable).to be_instance_of(Project) + end + end + + context 'when associated with group' do + it 'returns group' do + export = create(:bulk_import_export) + + expect(export.exportable).to be_instance_of(Group) + end + end + end + + describe '#config' do + context 'when associated with project' do + it 'returns project config' do + export = create(:bulk_import_export, project: create(:project), group: nil) + + expect(export.config).to be_instance_of(BulkImports::Exports::ProjectConfig) + end + end + + context 'when associated with group' do + it 'returns group config' do + export = create(:bulk_import_export) + + expect(export.config).to be_instance_of(BulkImports::Exports::GroupConfig) + end + end + end +end diff --git a/spec/models/bulk_imports/export_upload_spec.rb b/spec/models/bulk_imports/export_upload_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..bb659182ab56f0f3a5d65b3b20538ba89296f7a5 --- /dev/null +++ b/spec/models/bulk_imports/export_upload_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::ExportUpload do + subject { described_class.new(export: create(:bulk_import_export)) } + + it 'stores export file' do + method = 'export_file' + filename = 'labels.ndjson.gz' + + subject.public_send("#{method}=", fixture_file_upload("spec/fixtures/bulk_imports/#{filename}")) + subject.save! + + url = "/uploads/-/system/bulk_imports/export_upload/export_file/#{subject.id}/#{filename}" + + expect(subject.public_send(method).url).to eq(url) + end +end diff --git a/spec/models/bulk_imports/exports/group_config_spec.rb b/spec/models/bulk_imports/exports/group_config_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..fc7c3653bd7da52fd27e11c9a9a0a26dee2ba038 --- /dev/null +++ b/spec/models/bulk_imports/exports/group_config_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Exports::GroupConfig do + let_it_be(:exportable) { create(:group) } + + subject { described_class.new(exportable) } + + describe '#exportable_tree' do + it 'returns exportable project structure' do + config = ::Gitlab::ImportExport::Config.new(config: ::Gitlab::ImportExport.group_config_file).to_h + expected = ::Gitlab::ImportExport::AttributesFinder.new(config: config).find_root(:group) + + expect(subject.exportable_tree).to eq(expected) + end + end + + describe '#export_path' do + it 'returns correct export path' do + expect(subject.export_path).to eq(exportable.full_path) + end + end + + describe '#validate_user_permissions' do + let(:user) { create(:user) } + + context 'when user cannot admin project' do + it 'returns false' do + expect { subject.validate_user_permissions(user) }.to raise_error(Gitlab::ImportExport::Error) + end + end + + context 'when user can admin project' do + before do + exportable.add_owner(user) + end + + it 'returns true' do + expect(subject.validate_user_permissions(user)).to eq(true) + end + end + end +end diff --git a/spec/models/bulk_imports/exports/project_config_spec.rb b/spec/models/bulk_imports/exports/project_config_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..fad7e9fc19e9d7c4798398edcd4f5bbff8d90028 --- /dev/null +++ b/spec/models/bulk_imports/exports/project_config_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Exports::ProjectConfig do + let_it_be(:exportable) { create(:project) } + + subject { described_class.new(exportable) } + + describe '#exportable_tree' do + it 'returns exportable project structure' do + config = ::Gitlab::ImportExport::Config.new.to_h + expected = ::Gitlab::ImportExport::AttributesFinder.new(config: config).find_root(:project) + + expect(subject.exportable_tree).to eq(expected) + end + end + + describe '#export_path' do + it 'returns correct export path' do + expect(subject.export_path).to eq(exportable.disk_path) + end + end + + describe '#validate_user_permissions' do + let(:user) { create(:user) } + + context 'when user cannot admin project' do + it 'returns false' do + expect { subject.validate_user_permissions(user) }.to raise_error(Gitlab::ImportExport::Error) + end + end + + context 'when user can admin project' do + before do + exportable.add_maintainer(user) + end + + it 'returns true' do + expect(subject.validate_user_permissions(user)).to eq(true) + end + end + end +end diff --git a/spec/requests/api/group_export_spec.rb b/spec/requests/api/group_export_spec.rb index 50a1e9d0c3df6722fdf42f491ebb5ae0a7219895..38dfc3bbd67fed98e4757ebbe4fb37e04145a894 100644 --- a/spec/requests/api/group_export_spec.rb +++ b/spec/requests/api/group_export_spec.rb @@ -178,4 +178,67 @@ end end end + + describe 'relations export' do + let(:path) { "/groups/#{group.id}/export_relations" } + let(:download_path) { "/groups/#{group.id}/export_relations/download?relation=labels" } + let(:status_path) { "/groups/#{group.id}/export_relations/status" } + + before do + stub_feature_flags(group_import_export: true) + group.add_owner(user) + end + + describe 'POST /groups/:id/export_relations' do + it 'accepts the request' do + post api(path, user) + + expect(response).to have_gitlab_http_status(:accepted) + end + end + + describe 'GET /groups/:id/export_relations/download' do + let(:export) { create(:bulk_import_export, group: group, relation: 'labels') } + let(:upload) { create(:bulk_import_export_upload, export: export) } + + context 'when export file exists' do + before do + upload.export_file = fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz') + upload.save! + end + + it 'downloads exported group archive' do + get api(download_path, user) + + expect(response).to have_gitlab_http_status(:ok) + end + end + + context 'when export_file.file does not exist' do + before do + allow(upload).to receive(:export_file).and_return(nil) + end + + it 'returns 404' do + get api(download_path, user) + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + + describe 'GET /groups/:id/export_relations/status' do + it 'returns a list of relation export statuses' do + create(:bulk_import_export, :started, group: group, relation: 'labels') + create(:bulk_import_export, :finished, group: group, relation: 'milestones') + create(:bulk_import_export, :failed, group: group, relation: 'badges') + + get api(status_path, user) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'badges') + expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1) + end + end + end end diff --git a/spec/services/bulk_imports/export_service_spec.rb b/spec/services/bulk_imports/export_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5404678d118528cf7079b3a17ebe9ab8562e5383 --- /dev/null +++ b/spec/services/bulk_imports/export_service_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::ExportService do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } + + before do + group.add_owner(user) + end + + subject { described_class.new(exportable: group, user: user) } + + describe '#execute' do + it 'schedules RelationExportWorker for each top level relation' do + expect(subject).to receive(:execute).and_return(ServiceResponse.success).and_call_original + top_level_relations = ::Gitlab::ImportExport.top_level_relations(group.class.name) + + top_level_relations.each do |relation| + expect(BulkImports::RelationExportWorker) + .to receive(:perform_async) + .with(user.id, group.id, group.class.name, relation) + end + + subject.execute + end + + context 'when exception occurs' do + it 'does not schedule RelationExportWorker' do + service = described_class.new(exportable: nil, user: user) + + expect(service) + .to receive(:execute) + .and_return(ServiceResponse.error(message: 'Gitlab::ImportExport::Error', http_status: :unprocessible_entity)) + .and_call_original + expect(BulkImports::RelationExportWorker).not_to receive(:perform_async) + + service.execute + end + end + end +end diff --git a/spec/services/bulk_imports/relation_export_service_spec.rb b/spec/services/bulk_imports/relation_export_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1edd8d3ee5da8ec2548e7d1b1abb94ccea5c0d35 --- /dev/null +++ b/spec/services/bulk_imports/relation_export_service_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::RelationExportService do + let_it_be(:relation) { 'labels' } + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:label) { create(:group_label, group: group) } + let_it_be(:export_path) { "#{Dir.tmpdir}/relation_export_service_spec/tree" } + let_it_be_with_reload(:export) { create(:bulk_import_export, group: group, relation: relation) } + + before do + group.add_owner(user) + + allow(export).to receive(:export_path).and_return(export_path) + end + + after :all do + FileUtils.rm_rf(export_path) + end + + subject { described_class.new(export) } + + describe '#perform' do + it 'exports specified relation' do + subject.execute + + expect(export.reload.upload.export_file).to be_present + end + + it 'removes temp export files' do + subject.execute + + expect(Dir.exist?(export_path)).to eq(false) + end + + context 'when there is existing export present' do + let(:upload) { create(:bulk_import_export_upload, export: export) } + + before do + upload.export_file = fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz') + upload.save! + end + + it 'removes existing export before exporting' do + expect_any_instance_of(BulkImports::ExportUpload) do |upload| + expect(upload).to receive(:remove_export_file!) + end + + subject.execute + end + end + end +end diff --git a/spec/workers/bulk_imports/relation_export_worker_spec.rb b/spec/workers/bulk_imports/relation_export_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..adcb650041a9c70eb654c47bb7687779f13a2961 --- /dev/null +++ b/spec/workers/bulk_imports/relation_export_worker_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::RelationExportWorker do + let_it_be(:relation) { 'labels' } + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be_with_reload(:export) { create(:bulk_import_export, group: group) } + + let(:job_args) { [user.id, group.id, group.class.name, relation] } + + before do + group.add_owner(user) + end + + describe '#perform' do + include_examples 'an idempotent worker' do + context 'when export record does not exist' do + let(:another_group) { create(:group) } + let(:job_args) { [user.id, another_group.id, another_group.class.name, relation] } + + it 'creates export record' do + another_group.add_owner(user) + + expect { perform_multiple(job_args) } + .to change { another_group.bulk_import_exports.count } + .from(0) + .to(1) + end + end + + it 'executes RelationExportService' do + service = instance_double(BulkImports::RelationExportService) + + expect(BulkImports::RelationExportService) + .to receive(:new) + .with(export) + .twice + .and_return(service) + expect(service) + .to receive(:execute) + .twice + + perform_multiple(job_args) + end + + it 'exports specified relation and marks export as finished' do + perform_multiple(job_args) + + expect(export.upload.export_file).to be_present + expect(export.finished?).to eq(true) + end + + context 'when exception occurs during export' do + shared_examples 'tracks exception' do |exception_class| + it 'tracks exception' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with(exception_class, exportable_id: group.id, exportable_type: group.class.name) + .twice + .and_call_original + + perform_multiple(job_args) + end + end + + before do + allow_next_instance_of(BulkImports::ExportUpload) do |upload| + allow(upload).to receive(:save!).and_raise(StandardError) + end + end + + it 'marks export as failed' do + perform_multiple(job_args) + + expect(export.failed?).to eq(true) + end + + include_examples 'tracks exception', StandardError + + context 'when passed relation is not supported' do + let(:relation) { 'unsupported' } + + include_examples 'tracks exception', ActiveRecord::RecordInvalid + end + + context 'when user is not allowed to perform export' do + let(:another_user) { create(:user) } + let(:job_args) { [another_user.id, group.id, group.class.name, relation] } + + include_examples 'tracks exception', Gitlab::ImportExport::Error + end + end + end + end +end