From 2f7171054d6615cfa23ff099299d3959d41ac71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20W=C3=A4lter?= Date: Thu, 9 Sep 2021 22:46:13 +0200 Subject: [PATCH 1/3] Reschedule 'ExtractProjectTopicsIntoSeparateTable' background migration Changelog: fixed --- ...tract_project_topics_into_separate_table_2.rb | 16 ++++++++++++++++ db/schema_migrations/20210909104800 | 1 + ...extract_project_topics_into_separate_table.rb | 12 ++++++++---- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 db/post_migrate/20210909104800_reschedule_extract_project_topics_into_separate_table_2.rb create mode 100644 db/schema_migrations/20210909104800 diff --git a/db/post_migrate/20210909104800_reschedule_extract_project_topics_into_separate_table_2.rb b/db/post_migrate/20210909104800_reschedule_extract_project_topics_into_separate_table_2.rb new file mode 100644 index 00000000000000..ad31a40f324848 --- /dev/null +++ b/db/post_migrate/20210909104800_reschedule_extract_project_topics_into_separate_table_2.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class RescheduleExtractProjectTopicsIntoSeparateTable2 < Gitlab::Database::Migration[1.0] + MIGRATION = 'ExtractProjectTopicsIntoSeparateTable' + DELAY_INTERVAL = 4.minutes + + disable_ddl_transaction! + + def up + requeue_background_migration_jobs_by_range_at_intervals(MIGRATION, DELAY_INTERVAL) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20210909104800 b/db/schema_migrations/20210909104800 new file mode 100644 index 00000000000000..70ad28d72b85b1 --- /dev/null +++ b/db/schema_migrations/20210909104800 @@ -0,0 +1 @@ +834825de758314db01a3daa2d2b943c11751553705c4dd078f2087b9ec6ff7f7 \ No newline at end of file diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb index c5beb5a686537e..4af99480da5e03 100644 --- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb +++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb @@ -34,10 +34,14 @@ class Project < ActiveRecord::Base def perform(start_id, stop_id) Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| if Project.exists?(id: tagging.taggable_id) - topic = Topic.find_or_create_by(name: tagging.tag.name) - project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) - - tagging.delete if project_topic.persisted? + begin + topic = Topic.find_or_create_by(name: tagging.tag.name) + project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) + + tagging.delete if project_topic.persisted? + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) + end else tagging.delete end -- GitLab From 3b3cd6df03d8edc9e2a64ce7782b6f981ad6db3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20W=C3=A4lter?= Date: Tue, 14 Sep 2021 15:13:29 +0200 Subject: [PATCH 2/3] Extend specs for missing tag --- .../extract_project_topics_into_separate_table.rb | 2 ++ .../extract_project_topics_into_separate_table_spec.rb | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb index 4af99480da5e03..0f1b1ec8cb2d42 100644 --- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb +++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb @@ -41,6 +41,8 @@ def perform(start_id, stop_id) tagging.delete if project_topic.persisted? rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) + + tagging.delete end else tagging.delete diff --git a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb index 05f40ee2501d81..a111007a984e79 100644 --- a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb +++ b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb @@ -22,8 +22,9 @@ other_tagging = taggings.create!(taggable_type: 'Other', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) tagging_3 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_3.id) tagging_4 = taggings.create!(taggable_type: 'Project', taggable_id: -1, context: 'topics', tag_id: tag_1.id) + tagging_5 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: -1) - subject.perform(tagging_1.id, tagging_4.id) + subject.perform(tagging_1.id, tagging_5.id) # Tagging records expect { tagging_1.reload }.to raise_error(ActiveRecord::RecordNotFound) @@ -31,6 +32,7 @@ expect { other_tagging.reload }.not_to raise_error(ActiveRecord::RecordNotFound) expect { tagging_3.reload }.to raise_error(ActiveRecord::RecordNotFound) expect { tagging_4.reload }.to raise_error(ActiveRecord::RecordNotFound) + expect { tagging_5.reload }.to raise_error(ActiveRecord::RecordNotFound) # Topic records topic_1 = topics.find_by(name: 'Topic1') -- GitLab From 5092d6cd5375c218451259e2056658805603b9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20W=C3=A4lter?= Date: Thu, 16 Sep 2021 10:34:33 +0200 Subject: [PATCH 3/3] Do not delete tagging on every StandardError --- .../extract_project_topics_into_separate_table.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb index 0f1b1ec8cb2d42..31b5b5cdb7324d 100644 --- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb +++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb @@ -33,7 +33,7 @@ class Project < ActiveRecord::Base def perform(start_id, stop_id) Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| - if Project.exists?(id: tagging.taggable_id) + if Project.exists?(id: tagging.taggable_id) && tagging.tag begin topic = Topic.find_or_create_by(name: tagging.tag.name) project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) @@ -41,8 +41,6 @@ def perform(start_id, stop_id) tagging.delete if project_topic.persisted? rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) - - tagging.delete end else tagging.delete -- GitLab