diff --git a/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb index e3c8508815f8525ade9d81ac5c566e1a985a88c8..e769d05653b8338f3d1a33002bfdd2a2ab7834b9 100644 --- a/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb @@ -6,14 +6,40 @@ module Stage class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker include StageMethods + PLACEHOLDER_WAIT_INTERVAL = 30.seconds + private - # project - An instance of Project. + # @param project [Project] def import(project) + placeholder_reference_store = project.placeholder_reference_store + + if placeholder_reference_store&.any? + info( + project.id, + message: 'Delaying finalization as placeholder references are pending', + placeholder_store_count: placeholder_reference_store.count + ) + + reschedule(project) + + return + end + project.after_import Gitlab::Import::Metrics.new(:bitbucket_server_importer, project).track_finished_import end + + def reschedule(project) + ::Import::LoadPlaceholderReferencesWorker.perform_async( + project.import_type, + project.import_state.id, + { current_user_id: project.creator_id } + ) + + self.class.perform_in(PLACEHOLDER_WAIT_INTERVAL.seconds, project.id) + end end end end diff --git a/app/workers/import/load_placeholder_references_worker.rb b/app/workers/import/load_placeholder_references_worker.rb index e846bbb4bebf6c7e1d0c5cee955bb28179113f52..cb15e72a8212d084943d4219730d067ecca75551 100644 --- a/app/workers/import/load_placeholder_references_worker.rb +++ b/app/workers/import/load_placeholder_references_worker.rb @@ -25,8 +25,11 @@ def perform(import_source, import_uid, params = {}) end def perform_failure(exception, import_source, import_uid) - fail_import(import_source, import_uid) + log_failure(exception, import_source, import_uid) + clear_placeholder_reference_store(import_source, import_uid) + end + def log_failure(exception, import_source, import_uid) ::Import::Framework::Logger.error( message: 'Failed to load all references to placeholder user contributions', error: exception.message, @@ -35,21 +38,9 @@ def perform_failure(exception, import_source, import_uid) ) end - private - - def fail_import(import_source, import_uid) - import_state = nil - - case import_source - when 'gitlab' - import_state = BulkImport.find_by_id(import_uid) - when 'github', 'bitbucket', 'bitbucket_server', 'fogbugz', 'gitea', 'gitlab_project' - import_state = ProjectImportState.find_by_id(import_uid) - when 'import_group_from_file' - import_state = GroupImportState.find_by_group_id(import_uid) - end - - import_state&.fail_op + def clear_placeholder_reference_store(import_source, import_uid) + store = PlaceholderReferences::Store.new(import_source: import_source, import_uid: import_uid) + store.clear! end end end diff --git a/lib/import/placeholder_references/store.rb b/lib/import/placeholder_references/store.rb index b3d07c736b44e81dd2b2985ab3004daeccc850e7..c7822f30ddacda7877bfcf6183ad0fb9c28628e4 100644 --- a/lib/import/placeholder_references/store.rb +++ b/lib/import/placeholder_references/store.rb @@ -34,6 +34,10 @@ def any? !empty? end + def clear! + cache.del(cache_key) + end + private attr_reader :import_source, :import_uid diff --git a/spec/lib/import/placeholder_references/store_spec.rb b/spec/lib/import/placeholder_references/store_spec.rb index 5fed3c2dbcae6edb0f70ff9d988f049ab3f64804..a7f7afc81d173af367d396215ec993578c954bd7 100644 --- a/spec/lib/import/placeholder_references/store_spec.rb +++ b/spec/lib/import/placeholder_references/store_spec.rb @@ -72,6 +72,14 @@ end end + describe '#clear!' do + it 'removes the values from the cache' do + store.add('foo') + + expect { store.clear! }.to change { store.count }.from(1).to(0) + end + end + def cache Gitlab::Cache::Import::Caching end diff --git a/spec/workers/gitlab/bitbucket_server_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/bitbucket_server_import/stage/finish_import_worker_spec.rb index cb61ee13dd1d23e4e105238a3114c08bdb191070..2001b39ad635d13db99f81e032fed950e97b026b 100644 --- a/spec/workers/gitlab/bitbucket_server_import/stage/finish_import_worker_spec.rb +++ b/spec/workers/gitlab/bitbucket_server_import/stage/finish_import_worker_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BitbucketServerImport::Stage::FinishImportWorker, feature_category: :importers do - let_it_be(:project) { create(:project, :import_started) } + let_it_be(:project) { create(:project, :import_started, import_type: :bitbucket_server) } subject(:worker) { described_class.new } @@ -19,5 +19,66 @@ expect(project.import_state.reload).to be_finished end + + context 'when there are no unloaded placeholder references' do + before do + allow_next_instance_of(::Import::PlaceholderReferences::Store) do |store| + allow(store).to receive(:any?).and_return(false) + end + end + + it 'does not queue LoadPlaceholderReferencesWorker' do + expect(Import::LoadPlaceholderReferencesWorker).not_to receive(:perform_async) + + worker.perform(project.id) + end + + it 'does not re-enqueue itself' do + expect(described_class).not_to receive(:perform_in) + + worker.perform(project.id) + end + end + + context 'when there are unloaded placeholder references' do + before do + allow_next_instance_of(::Import::PlaceholderReferences::Store) do |store| + allow(store).to receive_messages(any?: true, count: 5) + end + end + + it 'queues LoadPlaceholderReferencesWorker' do + expect(Import::LoadPlaceholderReferencesWorker).to receive(:perform_async).with( + project.import_type, + project.import_state.id, + { current_user_id: project.creator.id } + ) + + worker.perform(project.id) + end + + it 'schedules itself to run again after 30 seconds' do + expect(described_class).to receive(:perform_in).with(30.seconds, project.id) + + worker.perform(project.id) + end + + it 'writes a log entry' do + allow(Gitlab::BitbucketServerImport::Logger).to receive(:info) + + expect(Gitlab::BitbucketServerImport::Logger) + .to receive(:info) + .with( + a_hash_including( + message: 'Delaying finalization as placeholder references are pending', + import_stage: 'Gitlab::BitbucketServerImport::Stage::FinishImportWorker', + placeholder_store_count: 5, + project_id: project.id + ) + ).once + + worker.perform(project.id) + end + end end end diff --git a/spec/workers/import/load_placeholder_references_worker_spec.rb b/spec/workers/import/load_placeholder_references_worker_spec.rb index b6c07940a61ddf6b8e7ea61a9513e9a741dc830c..80e35d210acedfdeb782233a99002f6f2e7103fd 100644 --- a/spec/workers/import/load_placeholder_references_worker_spec.rb +++ b/spec/workers/import/load_placeholder_references_worker_spec.rb @@ -54,7 +54,7 @@ let_it_be(:project) { create(:project) } shared_examples 'failed user contribution mapping' do - it 'logs the failure and fails the import status record', :aggregate_failures do + it 'logs the failure and clears the placeholder cache', :aggregate_failures do exception = StandardError.new('Some error') expect(::Import::Framework::Logger).to receive(:error).with({ @@ -64,9 +64,11 @@ import_uid: uid }) - described_class.sidekiq_retries_exhausted_block.call({ 'args' => [import_source, uid] }, exception) + expect_next_instance_of(Import::PlaceholderReferences::Store) do |store| + expect(store).to receive(:clear!) + end - expect(import_status_record.reload).to be_failed + described_class.sidekiq_retries_exhausted_block.call({ 'args' => [import_source, uid] }, exception) end # This case should not happen, but in case it does, there should still be a relevant error log anyway