diff --git a/app/models/project.rb b/app/models/project.rb index 759633a70d2d1e6c2bc559b46aa6f832c4e93f2f..b4375442152a76f7fcc60c70a083b3fc19370ba0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -128,8 +128,41 @@ class Project < ApplicationRecord after_initialize :use_hashed_storage after_create :check_repository_absence! - acts_as_ordered_taggable - alias_method :topics, :tag_list + acts_as_ordered_taggable_on :topics + # The 'tag_list' alias and the 'has_many' associations are required during the 'tags -> topics' migration + # TODO: eliminate 'tag_list', 'topic_taggings' and 'tags' in the further process of the migration + # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 + alias_attribute :tag_list, :topic_list + has_many :topic_taggings, -> { includes(:tag).order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + as: :taggable, + class_name: 'ActsAsTaggableOn::Tagging', + after_add: :dirtify_tag_list, + after_remove: :dirtify_tag_list + has_many :topics, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + class_name: 'ActsAsTaggableOn::Tag', + through: :topic_taggings, + source: :tag + has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + class_name: 'ActsAsTaggableOn::Tag', + through: :topic_taggings, + source: :tag + + # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1]. + # [1] https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61237 + # TODO: remove 'topic_list' and 'topic_list=' once the background migration is complete + # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 + def topic_list + # Return both old topics (context 'tags') and new topics (context 'topics') + tag_list_on('tags') + tag_list_on('topics') + end + + def topic_list=(new_tags) + # Old topics with context 'tags' are added as new topics with context 'topics' + super(new_tags) + + # Remove old topics with context 'tags' + set_tag_list_on('tags', '') + end attr_accessor :old_path_with_namespace attr_accessor :template_name diff --git a/app/presenters/project_presenter.rb b/app/presenters/project_presenter.rb index 3726238474f2962805245a0f280da621bd2641d3..4f803ba34f4b2f28e845b452afe536e213677f5b 100644 --- a/app/presenters/project_presenter.rb +++ b/app/presenters/project_presenter.rb @@ -401,16 +401,16 @@ def gitlab_ci_anchor_data end def topics_to_show - project.topics.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord + project.topic_list.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord end def topics_not_shown - project.topics - topics_to_show + project.topic_list - topics_to_show end def count_of_extra_topics_not_shown - if project.topics.count > MAX_TOPICS_TO_SHOW - project.topics.count - MAX_TOPICS_TO_SHOW + if project.topic_list.count > MAX_TOPICS_TO_SHOW + project.topic_list.count - MAX_TOPICS_TO_SHOW else 0 end diff --git a/changelogs/unreleased/project-topics-data-migration.yml b/changelogs/unreleased/project-topics-data-migration.yml new file mode 100644 index 0000000000000000000000000000000000000000..4e77d4ea2470c4d2995a801663e2a2dc4c770a36 --- /dev/null +++ b/changelogs/unreleased/project-topics-data-migration.yml @@ -0,0 +1,5 @@ +--- +title: Migrate 'tags' to 'topics' for project in the database context +merge_request: 61237 +author: Jonas Wälter @wwwjon +type: changed diff --git a/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb b/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb new file mode 100644 index 0000000000000000000000000000000000000000..0d807df614cc4ae3e291159a1c3bb287977fdc43 --- /dev/null +++ b/db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags' + INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'" + + disable_ddl_transaction! + + def up + # this index is used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics + add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :taggings, INDEX_NAME + end +end diff --git a/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 0000000000000000000000000000000000000000..25d23b771d5c89ce975b43a17a7d084166399fc5 --- /dev/null +++ b/db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ScheduleMigrateProjectTaggingsContextFromTagsToTopics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 30_000 + DELAY_INTERVAL = 2.minutes + MIGRATION = 'MigrateProjectTaggingsContextFromTagsToTopics' + + disable_ddl_transaction! + + class Tagging < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'taggings' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + Tagging.where(taggable_type: 'Project', context: 'tags'), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + end +end diff --git a/db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb b/db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb new file mode 100644 index 0000000000000000000000000000000000000000..bfd09653695dff21595dd12492842efe2ac0131e --- /dev/null +++ b/db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags' + INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'" + + disable_ddl_transaction! + + def up + # this index was used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics + remove_concurrent_index_by_name :taggings, INDEX_NAME + end + + def down + add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20210511095657 b/db/schema_migrations/20210511095657 new file mode 100644 index 0000000000000000000000000000000000000000..503ccfb459aa9250e582f74c935038c1de19f65e --- /dev/null +++ b/db/schema_migrations/20210511095657 @@ -0,0 +1 @@ +4d11cdf876786db5e827ea1a50b70e2d5b3814fd7c0b0c083ab61adad9685364 \ No newline at end of file diff --git a/db/schema_migrations/20210511095658 b/db/schema_migrations/20210511095658 new file mode 100644 index 0000000000000000000000000000000000000000..d3fa692768db43719085c1f2c665f40ff10134a2 --- /dev/null +++ b/db/schema_migrations/20210511095658 @@ -0,0 +1 @@ +7387c23bbbc376e26c057179ebe2796be183462acb1fc509d451f0fede13ed93 \ No newline at end of file diff --git a/db/schema_migrations/20210517075444 b/db/schema_migrations/20210517075444 new file mode 100644 index 0000000000000000000000000000000000000000..4b4aee8710cc47057093d9dd151c6439d42fc021 --- /dev/null +++ b/db/schema_migrations/20210517075444 @@ -0,0 +1 @@ +ec08c18ac37f2ae7298650df58345755eada20aaa5b7ed3dfd54ee5cea88ebdd \ No newline at end of file diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 0000000000000000000000000000000000000000..68bbd3cfebb5237fd9e08c16452d3bee8bab4949 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the context of project taggings from `tags` to `topics` + class MigrateProjectTaggingsContextFromTagsToTopics + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + include EachBatch + + self.table_name = 'taggings' + end + + def perform(start_id, stop_id) + Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| + relation.update_all(context: 'topics') + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 447d4db865e3ba405265bd517a09d14acad716fe..d000c331b6d96a696985387c4959422a8dce5513 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -153,6 +153,7 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :topic_list - :mirror_user_id - :mirror_trigger_builds - :only_mirror_protected_branches diff --git a/spec/finders/projects_finder_spec.rb b/spec/finders/projects_finder_spec.rb index a178261e8993213621e2b30f20d6f53f59102b7f..364e5de4ece441357d0b66bc740ca0c8e13e0d9c 100644 --- a/spec/finders/projects_finder_spec.rb +++ b/spec/finders/projects_finder_spec.rb @@ -139,7 +139,7 @@ describe 'filter by tags' do before do - public_project.tag_list.add('foo') + public_project.tag_list = 'foo' public_project.save! end diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5e2f32c54be52ae952be497018a054ea14db4a85 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, schema: 20210511095658 do + it 'correctly migrates project taggings context from tags to topics' do + taggings = table(:taggings) + + project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') + project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') + project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') + project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') + project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') + + subject.perform(project_old_tagging_1.id, project_old_tagging_2.id) + + project_old_tagging_1.reload + project_new_tagging_1.reload + project_other_context_tagging_1.reload + project_old_tagging_2.reload + project_old_tagging_3.reload + + expect(project_old_tagging_1.context).to eq('topics') + expect(project_new_tagging_1.context).to eq('topics') + expect(project_other_context_tagging_1.context).to eq('other') + expect(project_old_tagging_2.context).to eq('topics') + expect(project_old_tagging_3.context).to eq('tags') + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 8253cb90ce55ab5d3eb852f832bee55e1ce72ee2..84743e5759416c4a8804f1c76bbaf6660f68cced 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -343,8 +343,9 @@ project: - external_approval_rules - taggings - base_tags -- tag_taggings - tags +- topic_taggings +- topics - chat_services - cluster - clusters diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 43cea0ddcd106850a735a6d869179f8657c83cec..f60b807bc344d31c1d85b1fb3ca4cd9763bb09e5 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6963,6 +6963,55 @@ def enable_lfs end end + describe 'topics' do + let_it_be(:project) { create(:project, tag_list: 'topic1, topic2, topic3') } + + it 'topic_list returns correct string array' do + expect(project.topic_list).to match_array(%w[topic1 topic2 topic3]) + end + + it 'topics returns correct tag records' do + expect(project.topics.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.topics.map(&:name)).to match_array(%w[topic1 topic2 topic3]) + end + + context 'aliases' do + it 'tag_list returns correct string array' do + expect(project.tag_list).to match_array(%w[topic1 topic2 topic3]) + end + + it 'tags returns correct tag records' do + expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3]) + end + end + + context 'intermediate state during background migration' do + before do + project.taggings.first.update!(context: 'tags') + project.instance_variable_set("@tag_list", nil) + project.reload + end + + it 'tag_list returns string array including old and new topics' do + expect(project.tag_list).to match_array(%w[topic1 topic2 topic3]) + end + + it 'tags returns old and new tag records' do + expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag') + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3]) + expect(project.taggings.map(&:context)).to match_array(%w[tags topics topics]) + end + + it 'update tag_list adds new topics and removes old topics' do + project.update!(tag_list: 'topic1, topic2, topic3, topic4') + + expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3 topic4]) + expect(project.taggings.map(&:context)).to match_array(%w[topics topics topics topics]) + end + end + end + def finish_job(export_job) export_job.start export_job.finish diff --git a/spec/requests/api/project_attributes.yml b/spec/requests/api/project_attributes.yml index f9eb9de94dbb2b07a89489ae2cfe89865449f092..d28442bd692c3c25d9dada9b2cad145e0198e80f 100644 --- a/spec/requests/api/project_attributes.yml +++ b/spec/requests/api/project_attributes.yml @@ -41,6 +41,7 @@ itself: # project - reset_approvals_on_push - runners_token_encrypted - storage_version + - topic_list - updated_at remapped_attributes: avatar: avatar_url @@ -67,6 +68,7 @@ itself: # project - readme_url - shared_with_groups - ssh_url_to_repo + - tag_list - web_url build_auto_devops: # auto_devops