From 59d428480920172adc478cc7b3c773d053dac99b Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Fri, 7 May 2021 13:34:26 +0200 Subject: [PATCH 1/7] Migrate 'tags' to 'topics' for projects --- app/models/project.rb | 6 +++++- ...20210505093539_migrate_project_tags_to_topics.rb | 13 +++++++++++++ db/schema_migrations/20210505093539 | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20210505093539_migrate_project_tags_to_topics.rb create mode 100644 db/schema_migrations/20210505093539 diff --git a/app/models/project.rb b/app/models/project.rb index 759633a70d2d1e..b7ea87f7d95054 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -128,7 +128,11 @@ class Project < ApplicationRecord after_initialize :use_hashed_storage after_create :check_repository_absence! - acts_as_ordered_taggable + acts_as_ordered_taggable_on :topics + # TODO: eliminate aliases in the further process of the 'tags -> topics' migration + # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 + alias_method :tag_list, :topic_list + alias_method :tags, :topics alias_method :topics, :tag_list attr_accessor :old_path_with_namespace diff --git a/db/migrate/20210505093539_migrate_project_tags_to_topics.rb b/db/migrate/20210505093539_migrate_project_tags_to_topics.rb new file mode 100644 index 00000000000000..964c0e18ada9fa --- /dev/null +++ b/db/migrate/20210505093539_migrate_project_tags_to_topics.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class MigrateProjectTagsToTopics < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute("UPDATE taggings SET context = 'topics' WHERE taggable_type = 'Project' AND context = 'tags'") + end + + def down + execute("UPDATE taggings SET context = 'tags' WHERE taggable_type = 'Project' AND context = 'topics'") + end +end diff --git a/db/schema_migrations/20210505093539 b/db/schema_migrations/20210505093539 new file mode 100644 index 00000000000000..cbfb1f4a75e427 --- /dev/null +++ b/db/schema_migrations/20210505093539 @@ -0,0 +1 @@ +4fa374e6565d6bc18f5779b12e54e7f1c47e441f0a95267e3c476789302cee27 \ No newline at end of file -- GitLab From 8413a7c5f1a566af62fddf70d9076a4543d91ea0 Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Fri, 7 May 2021 14:22:30 +0200 Subject: [PATCH 2/7] Add changelog entry --- app/models/project.rb | 1 + changelogs/unreleased/project-topics-data-migration.yml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelogs/unreleased/project-topics-data-migration.yml diff --git a/app/models/project.rb b/app/models/project.rb index b7ea87f7d95054..e54230d5f76c8b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -132,6 +132,7 @@ class Project < ApplicationRecord # TODO: eliminate aliases in the further process of the 'tags -> topics' migration # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 alias_method :tag_list, :topic_list + alias_method :tag_list=, :topic_list= alias_method :tags, :topics alias_method :topics, :tag_list diff --git a/changelogs/unreleased/project-topics-data-migration.yml b/changelogs/unreleased/project-topics-data-migration.yml new file mode 100644 index 00000000000000..4e77d4ea2470c4 --- /dev/null +++ b/changelogs/unreleased/project-topics-data-migration.yml @@ -0,0 +1,5 @@ +--- +title: Migrate 'tags' to 'topics' for project in the database context +merge_request: 61237 +author: Jonas Wälter @wwwjon +type: changed -- GitLab From 00fd295e1b307a938d994e35e7194d60df796086 Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Mon, 10 May 2021 17:06:58 +0200 Subject: [PATCH 3/7] Fix topics tags alias --- app/models/project.rb | 12 +++++++----- app/presenters/project_presenter.rb | 8 ++++---- lib/gitlab/import_export/project/import_export.yml | 1 + spec/lib/gitlab/import_export/all_models.yml | 3 ++- spec/requests/api/project_attributes.yml | 2 ++ 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index e54230d5f76c8b..c8d6a0dc49dde8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -129,12 +129,14 @@ class Project < ApplicationRecord after_create :check_repository_absence! acts_as_ordered_taggable_on :topics - # TODO: eliminate aliases in the further process of the 'tags -> topics' migration + # The following 'alias_attribute' and 'has_many' definitions are required during the 'tags -> topics' migration + # TODO: eliminate 'alias_attribute' and 'has_many' definitions in the further process of the migration # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 - alias_method :tag_list, :topic_list - alias_method :tag_list=, :topic_list= - alias_method :tags, :topics - alias_method :topics, :tag_list + alias_attribute :tag_list, :topic_list + has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + class_name: 'ActsAsTaggableOn::Tag', + through: :topic_taggings, + source: :tag attr_accessor :old_path_with_namespace attr_accessor :template_name diff --git a/app/presenters/project_presenter.rb b/app/presenters/project_presenter.rb index 3726238474f296..4f803ba34f4b2f 100644 --- a/app/presenters/project_presenter.rb +++ b/app/presenters/project_presenter.rb @@ -401,16 +401,16 @@ def gitlab_ci_anchor_data end def topics_to_show - project.topics.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord + project.topic_list.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord end def topics_not_shown - project.topics - topics_to_show + project.topic_list - topics_to_show end def count_of_extra_topics_not_shown - if project.topics.count > MAX_TOPICS_TO_SHOW - project.topics.count - MAX_TOPICS_TO_SHOW + if project.topic_list.count > MAX_TOPICS_TO_SHOW + project.topic_list.count - MAX_TOPICS_TO_SHOW else 0 end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 447d4db865e3ba..d000c331b6d96a 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -153,6 +153,7 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :topic_list - :mirror_user_id - :mirror_trigger_builds - :only_mirror_protected_branches diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 8253cb90ce55ab..84743e5759416c 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -343,8 +343,9 @@ project: - external_approval_rules - taggings - base_tags -- tag_taggings - tags +- topic_taggings +- topics - chat_services - cluster - clusters diff --git a/spec/requests/api/project_attributes.yml b/spec/requests/api/project_attributes.yml index f9eb9de94dbb2b..d28442bd692c3c 100644 --- a/spec/requests/api/project_attributes.yml +++ b/spec/requests/api/project_attributes.yml @@ -41,6 +41,7 @@ itself: # project - reset_approvals_on_push - runners_token_encrypted - storage_version + - topic_list - updated_at remapped_attributes: avatar: avatar_url @@ -67,6 +68,7 @@ itself: # project - readme_url - shared_with_groups - ssh_url_to_repo + - tag_list - web_url build_auto_devops: # auto_devops -- GitLab From ed0553f445994b40b5e64a07ff99c272af52d0c0 Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Tue, 11 May 2021 13:06:21 +0200 Subject: [PATCH 4/7] Add background migration for project topics --- app/models/project.rb | 22 +++++++-- ...05093539_migrate_project_tags_to_topics.rb | 13 ----- ...ct_taggings_context_from_tags_to_topics.rb | 30 ++++++++++++ db/schema_migrations/20210505093539 | 1 - db/schema_migrations/20210511095658 | 1 + ...ct_taggings_context_from_tags_to_topics.rb | 17 +++++++ ...ggings_context_from_tags_to_topics_spec.rb | 29 +++++++++++ spec/models/project_spec.rb | 49 +++++++++++++++++++ 8 files changed, 145 insertions(+), 17 deletions(-) delete mode 100644 db/migrate/20210505093539_migrate_project_tags_to_topics.rb create mode 100644 db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb delete mode 100644 db/schema_migrations/20210505093539 create mode 100644 db/schema_migrations/20210511095658 create mode 100644 lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb create mode 100644 spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb diff --git a/app/models/project.rb b/app/models/project.rb index c8d6a0dc49dde8..037f4271c21e34 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -129,15 +129,31 @@ class Project < ApplicationRecord after_create :check_repository_absence! acts_as_ordered_taggable_on :topics - # The following 'alias_attribute' and 'has_many' definitions are required during the 'tags -> topics' migration - # TODO: eliminate 'alias_attribute' and 'has_many' definitions in the further process of the migration + # The 'tag_list' alias and the 'tags' relation are required during the 'tags -> topics' migration + # TODO: eliminate 'tag_list' and 'tags' in the further process of the migration # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 alias_attribute :tag_list, :topic_list has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, class_name: 'ActsAsTaggableOn::Tag', - through: :topic_taggings, + through: :taggings, source: :tag + # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1]. + # [1] https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61237 + # TODO: remove 'topic_list' and 'topic_list=' once the background migration is complete + def topic_list + # Return both old topics (context 'tags') and new topics (context 'topics') + tag_list_on('tags') + tag_list_on('topics') + end + + def topic_list=(new_tags) + # Old topics with context 'tags' are added as new topics with context 'topics' + super(new_tags) + + # Remove old topics with context 'tags' + set_tag_list_on('tags', '') + end + attr_accessor :old_path_with_namespace attr_accessor :template_name attr_writer :pipeline_status diff --git a/db/migrate/20210505093539_migrate_project_tags_to_topics.rb b/db/migrate/20210505093539_migrate_project_tags_to_topics.rb deleted file mode 100644 index 964c0e18ada9fa..00000000000000 --- a/db/migrate/20210505093539_migrate_project_tags_to_topics.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -class MigrateProjectTagsToTopics < ActiveRecord::Migration[6.0] - DOWNTIME = false - - def up - execute("UPDATE taggings SET context = 'topics' WHERE taggable_type = 'Project' AND context = 'tags'") - end - - def down - execute("UPDATE taggings SET context = 'tags' WHERE taggable_type = 'Project' AND context = 'topics'") - end -end diff --git a/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 00000000000000..5ffc40fe653162 --- /dev/null +++ b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ScheduleMigrateProjectTaggingsContextFromTagsToTopics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 50_000 + DELAY_INTERVAL = 2.minutes + MIGRATION = 'MigrateProjectTaggingsContextFromTagsToTopics' + + disable_ddl_transaction! + + class Taggings < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'taggings' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + Taggings.all, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + end +end diff --git a/db/schema_migrations/20210505093539 b/db/schema_migrations/20210505093539 deleted file mode 100644 index cbfb1f4a75e427..00000000000000 --- a/db/schema_migrations/20210505093539 +++ /dev/null @@ -1 +0,0 @@ -4fa374e6565d6bc18f5779b12e54e7f1c47e441f0a95267e3c476789302cee27 \ No newline at end of file diff --git a/db/schema_migrations/20210511095658 b/db/schema_migrations/20210511095658 new file mode 100644 index 00000000000000..d3fa692768db43 --- /dev/null +++ b/db/schema_migrations/20210511095658 @@ -0,0 +1 @@ +7387c23bbbc376e26c057179ebe2796be183462acb1fc509d451f0fede13ed93 \ No newline at end of file diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 00000000000000..5aaebf41d23fa2 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the context of project taggings from `tags` to `topics` + class MigrateProjectTaggingsContextFromTagsToTopics + # Temporary AR table for taggings + class Taggings < ActiveRecord::Base + self.table_name = 'taggings' + end + + def perform(start_id, stop_id) + Taggings.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).update_all(context: 'topics') + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb new file mode 100644 index 00000000000000..8ac44c8cf9232f --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, schema: 20210511095658 do + let(:taggings) { table(:taggings) } + + it 'correctly migrates project taggings context from tags to topics' do + project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') + project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') + project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') + project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') + project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') + + described_class.new.perform(project_old_tagging_1.id, project_old_tagging_2.id) + + project_old_tagging_1.reload + project_new_tagging_1.reload + project_other_context_tagging_1.reload + project_old_tagging_2.reload + project_old_tagging_3.reload + + expect(project_old_tagging_1.context).to eq('topics') + expect(project_new_tagging_1.context).to eq('topics') + expect(project_other_context_tagging_1.context).to eq('other') + expect(project_old_tagging_2.context).to eq('topics') + expect(project_old_tagging_3.context).to eq('tags') + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 43cea0ddcd1068..c4522e6692ea1b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6963,6 +6963,55 @@ def enable_lfs end end + describe 'topics' do + let_it_be(:project) { create(:project, tag_list: 'topic1, topic2, topic3') } + + it 'topic_list returns correct string array' do + expect(project.topic_list).to match(%w[topic1 topic2 topic3]) + end + + it 'topics returns correct tag records' do + expect(project.topics.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.topics.map(&:name)).to match(%w[topic1 topic2 topic3]) + end + + context 'aliases' do + it 'tag_list returns correct string array' do + expect(project.tag_list).to match(%w[topic1 topic2 topic3]) + end + + it 'tags returns correct tag records' do + expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3]) + end + end + + context 'intermediate state during background migration' do + before do + project.taggings.first.update!(context: 'tags') + project.instance_variable_set("@tag_list", nil) + project.reload + end + + it 'tag_list returns string array including old and new topics' do + expect(project.tag_list).to match(%w[topic1 topic2 topic3]) + end + + it 'tags returns old and new tag records' do + expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3]) + expect(project.taggings.map(&:context)).to match(%w[tags topics topics]) + end + + it 'update tag_list adds new topics and removes old topics' do + project.update!(tag_list: 'topic1, topic2, topic3, topic4') + + expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3 topic4]) + expect(project.taggings.map(&:context)).to match(%w[topics topics topics topics]) + end + end + end + def finish_job(export_job) export_job.start export_job.finish -- GitLab From 89a564a888ae29195fa7940a12adac04ec2e4473 Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Wed, 12 May 2021 16:36:00 +0200 Subject: [PATCH 5/7] Fix project topics specs --- app/models/project.rb | 15 ++++++++++++--- spec/finders/projects_finder_spec.rb | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 037f4271c21e34..9db81c168c73d9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -129,13 +129,22 @@ class Project < ApplicationRecord after_create :check_repository_absence! acts_as_ordered_taggable_on :topics - # The 'tag_list' alias and the 'tags' relation are required during the 'tags -> topics' migration - # TODO: eliminate 'tag_list' and 'tags' in the further process of the migration + # The 'tag_list' alias and the 'has_many' associations are required during the 'tags -> topics' migration + # TODO: eliminate 'tag_list', 'topic_taggings' and 'tags' in the further process of the migration # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 alias_attribute :tag_list, :topic_list + has_many :topic_taggings, -> { includes(:tag).order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + as: :taggable, + class_name: 'ActsAsTaggableOn::Tagging', + after_add: :dirtify_tag_list, + after_remove: :dirtify_tag_list + has_many :topics, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + class_name: 'ActsAsTaggableOn::Tag', + through: :topic_taggings, + source: :tag has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, class_name: 'ActsAsTaggableOn::Tag', - through: :taggings, + through: :topic_taggings, source: :tag # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1]. diff --git a/spec/finders/projects_finder_spec.rb b/spec/finders/projects_finder_spec.rb index a178261e899321..364e5de4ece441 100644 --- a/spec/finders/projects_finder_spec.rb +++ b/spec/finders/projects_finder_spec.rb @@ -139,7 +139,7 @@ describe 'filter by tags' do before do - public_project.tag_list.add('foo') + public_project.tag_list = 'foo' public_project.save! end -- GitLab From fa776a18e2acd4222d888287366474529a19d583 Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Mon, 17 May 2021 10:06:49 +0200 Subject: [PATCH 6/7] Optimize background migration using temp index --- app/models/project.rb | 3 ++- ...ry_index_for_project_topics_to_taggings.rb | 19 +++++++++++++++++++ ...ry_index_for_project_topics_to_taggings.rb | 19 +++++++++++++++++++ ...ct_taggings_context_from_tags_to_topics.rb | 6 +++--- db/schema_migrations/20210511095657 | 1 + db/schema_migrations/20210517075444 | 1 + ...ct_taggings_context_from_tags_to_topics.rb | 8 ++++++-- ...ggings_context_from_tags_to_topics_spec.rb | 6 +++--- spec/models/project_spec.rb | 18 +++++++++--------- 9 files changed, 63 insertions(+), 18 deletions(-) create mode 100644 db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb create mode 100644 db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb create mode 100644 db/schema_migrations/20210511095657 create mode 100644 db/schema_migrations/20210517075444 diff --git a/app/models/project.rb b/app/models/project.rb index 9db81c168c73d9..b4375442152a76 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -131,7 +131,7 @@ class Project < ApplicationRecord acts_as_ordered_taggable_on :topics # The 'tag_list' alias and the 'has_many' associations are required during the 'tags -> topics' migration # TODO: eliminate 'tag_list', 'topic_taggings' and 'tags' in the further process of the migration - # https://gitlab.com/gitlab-org/gitlab/-/issues/328226 + # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 alias_attribute :tag_list, :topic_list has_many :topic_taggings, -> { includes(:tag).order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, as: :taggable, @@ -150,6 +150,7 @@ class Project < ApplicationRecord # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1]. # [1] https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61237 # TODO: remove 'topic_list' and 'topic_list=' once the background migration is complete + # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 def topic_list # Return both old topics (context 'tags') and new topics (context 'topics') tag_list_on('tags') + tag_list_on('topics') diff --git a/db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb b/db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb new file mode 100644 index 00000000000000..bfd09653695dff --- /dev/null +++ b/db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags' + INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'" + + disable_ddl_transaction! + + def up + # this index was used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics + remove_concurrent_index_by_name :taggings, INDEX_NAME + end + + def down + add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb b/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb new file mode 100644 index 00000000000000..0d807df614cc4a --- /dev/null +++ b/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags' + INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'" + + disable_ddl_transaction! + + def up + # this index is used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics + add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :taggings, INDEX_NAME + end +end diff --git a/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb index 5ffc40fe653162..25d23b771d5c89 100644 --- a/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb +++ b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb @@ -4,13 +4,13 @@ class ScheduleMigrateProjectTaggingsContextFromTagsToTopics < ActiveRecord::Migr include Gitlab::Database::MigrationHelpers DOWNTIME = false - BATCH_SIZE = 50_000 + BATCH_SIZE = 30_000 DELAY_INTERVAL = 2.minutes MIGRATION = 'MigrateProjectTaggingsContextFromTagsToTopics' disable_ddl_transaction! - class Taggings < ActiveRecord::Base + class Tagging < ActiveRecord::Base include ::EachBatch self.table_name = 'taggings' @@ -18,7 +18,7 @@ class Taggings < ActiveRecord::Base def up queue_background_migration_jobs_by_range_at_intervals( - Taggings.all, + Tagging.where(taggable_type: 'Project', context: 'tags'), MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE diff --git a/db/schema_migrations/20210511095657 b/db/schema_migrations/20210511095657 new file mode 100644 index 00000000000000..503ccfb459aa92 --- /dev/null +++ b/db/schema_migrations/20210511095657 @@ -0,0 +1 @@ +4d11cdf876786db5e827ea1a50b70e2d5b3814fd7c0b0c083ab61adad9685364 \ No newline at end of file diff --git a/db/schema_migrations/20210517075444 b/db/schema_migrations/20210517075444 new file mode 100644 index 00000000000000..4b4aee8710cc47 --- /dev/null +++ b/db/schema_migrations/20210517075444 @@ -0,0 +1 @@ +ec08c18ac37f2ae7298650df58345755eada20aaa5b7ed3dfd54ee5cea88ebdd \ No newline at end of file diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb index 5aaebf41d23fa2..68bbd3cfebb523 100644 --- a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -5,12 +5,16 @@ module BackgroundMigration # The class to migrate the context of project taggings from `tags` to `topics` class MigrateProjectTaggingsContextFromTagsToTopics # Temporary AR table for taggings - class Taggings < ActiveRecord::Base + class Tagging < ActiveRecord::Base + include EachBatch + self.table_name = 'taggings' end def perform(start_id, stop_id) - Taggings.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).update_all(context: 'topics') + Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| + relation.update_all(context: 'topics') + end end end end diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb index 8ac44c8cf9232f..5e2f32c54be52a 100644 --- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb @@ -3,16 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, schema: 20210511095658 do - let(:taggings) { table(:taggings) } - it 'correctly migrates project taggings context from tags to topics' do + taggings = table(:taggings) + project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') - described_class.new.perform(project_old_tagging_1.id, project_old_tagging_2.id) + subject.perform(project_old_tagging_1.id, project_old_tagging_2.id) project_old_tagging_1.reload project_new_tagging_1.reload diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index c4522e6692ea1b..f60b807bc344d3 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6967,22 +6967,22 @@ def enable_lfs let_it_be(:project) { create(:project, tag_list: 'topic1, topic2, topic3') } it 'topic_list returns correct string array' do - expect(project.topic_list).to match(%w[topic1 topic2 topic3]) + expect(project.topic_list).to match_array(%w[topic1 topic2 topic3]) end it 'topics returns correct tag records' do expect(project.topics.first.class.name).to eq('ActsAsTaggableOn::Tag') - expect(project.topics.map(&:name)).to match(%w[topic1 topic2 topic3]) + expect(project.topics.map(&:name)).to match_array(%w[topic1 topic2 topic3]) end context 'aliases' do it 'tag_list returns correct string array' do - expect(project.tag_list).to match(%w[topic1 topic2 topic3]) + expect(project.tag_list).to match_array(%w[topic1 topic2 topic3]) end it 'tags returns correct tag records' do expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') - expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3]) + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3]) end end @@ -6994,20 +6994,20 @@ def enable_lfs end it 'tag_list returns string array including old and new topics' do - expect(project.tag_list).to match(%w[topic1 topic2 topic3]) + expect(project.tag_list).to match_array(%w[topic1 topic2 topic3]) end it 'tags returns old and new tag records' do expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') - expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3]) - expect(project.taggings.map(&:context)).to match(%w[tags topics topics]) + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3]) + expect(project.taggings.map(&:context)).to match_array(%w[tags topics topics]) end it 'update tag_list adds new topics and removes old topics' do project.update!(tag_list: 'topic1, topic2, topic3, topic4') - expect(project.tags.map(&:name)).to match(%w[topic1 topic2 topic3 topic4]) - expect(project.taggings.map(&:context)).to match(%w[topics topics topics topics]) + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3 topic4]) + expect(project.taggings.map(&:context)).to match_array(%w[topics topics topics topics]) end end end -- GitLab From bcc9ad77d915b3a2b295208712bc5b0d261ac02d Mon Sep 17 00:00:00 2001 From: Jonas Waelter Date: Tue, 18 May 2021 16:34:49 +0200 Subject: [PATCH 7/7] Move migration to db/post_migrate --- ...75444_remove_temporary_index_for_project_topics_to_taggings.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/{migrate => post_migrate}/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb (100%) diff --git a/db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb b/db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb similarity index 100% rename from db/migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb rename to db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb -- GitLab