diff --git a/db/docs/batched_background_migrations/delete_orphaned_groups.yml b/db/docs/batched_background_migrations/delete_orphaned_groups.yml new file mode 100644 index 0000000000000000000000000000000000000000..c8d84dc1c66b7b5fdca38b7eeb6184eac5480522 --- /dev/null +++ b/db/docs/batched_background_migrations/delete_orphaned_groups.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: DeleteOrphanedGroups +description: Deletes orhpaned groups whose parent's does not exist +feature_category: groups_and_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/172420 +milestone: '17.7' +queued_migration_version: 20241112163029 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20241112163029_queue_delete_orphaned_groups.rb b/db/post_migrate/20241112163029_queue_delete_orphaned_groups.rb new file mode 100644 index 0000000000000000000000000000000000000000..4e2d7d702b40ecc9a9fa9662d21de8b5a37ea5c6 --- /dev/null +++ b/db/post_migrate/20241112163029_queue_delete_orphaned_groups.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueDeleteOrphanedGroups < Gitlab::Database::Migration[2.2] + milestone '17.7' + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "DeleteOrphanedGroups" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + return unless Gitlab.com_except_jh? && !Gitlab.staging? + + queue_batched_background_migration( + MIGRATION, + :namespaces, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) + end +end diff --git a/db/schema_migrations/20241112163029 b/db/schema_migrations/20241112163029 new file mode 100644 index 0000000000000000000000000000000000000000..f95805a988d7f624d24e40cc3337eac4128b842b --- /dev/null +++ b/db/schema_migrations/20241112163029 @@ -0,0 +1 @@ +ce1cea13a912cd65923aea1757bc0c073e425d20ac6e7bae620f66cee518bdb7 \ No newline at end of file diff --git a/lib/gitlab/background_migration/delete_orphaned_groups.rb b/lib/gitlab/background_migration/delete_orphaned_groups.rb new file mode 100644 index 0000000000000000000000000000000000000000..a032ad40180b684cc93867480887a760a26df342 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_groups.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DeleteOrphanedGroups < BatchedMigrationJob + operation_name :delete_orphaned_group_records + feature_category :groups_and_projects + + scope_to ->(relation) { relation.where(type: 'Group').where.not(parent_id: nil) } + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins("LEFT JOIN namespaces AS parent ON namespaces.parent_id = parent.id") + .where(parent: { id: nil }) + .pluck(:id).each do |orphaned_group_id| + ::GroupDestroyWorker.perform(orphaned_group_id, admin_bot.id) + end + end + end + + private + + def admin_bot + @_admin_bot ||= Users::Internal.admin_bot + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_groups_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_groups_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..ea6536b1959fadb2fe8ffa070777909873feddb6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_groups_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedGroups, feature_category: :groups_and_projects do + let(:namespaces) { table(:namespaces) } + let!(:parent) { namespaces.create!(name: 'Group', type: 'Group', path: 'space1') } + let!(:group) { namespaces.create!(name: 'GitLab', type: 'Group', path: 'group1') } + + subject(:background_migration) do + described_class.new( + start_id: namespaces.without(parent).minimum(:id), + end_id: namespaces.maximum(:id), + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + describe '#perform' do + before do + # Remove constraint so we can create invalid records + ApplicationRecord.connection.execute("ALTER TABLE namespaces DROP CONSTRAINT fk_7f813d8c90;") + end + + after do + # Re-create constraint after the test + ApplicationRecord.connection.execute(<<~SQL) + ALTER TABLE ONLY namespaces ADD CONSTRAINT fk_7f813d8c90 + FOREIGN KEY (parent_id) REFERENCES namespaces(id) ON DELETE RESTRICT NOT VALID; + SQL + end + + it 'enqueues ::GroupDestroyWorker for each group whose parent\'s do not exist' do + orphaned_groups = (1..4).map do |i| + namespaces.create!(name: "Group #{i}", path: "orphaned_group_#{i}", type: 'Group', parent_id: parent.id) + end + groups = (1..4).map do |i| + namespaces.create!(name: "Group #{i}", path: "group_#{i}", type: 'Group', parent_id: group.id) + end + parent.destroy! + + orphaned_groups.each do |group| + expect(::GroupDestroyWorker).to receive(:perform).with(group.id, ::Users::Internal.admin_bot.id) + end + + groups.each do |group| + expect(::GroupDestroyWorker).not_to receive(:perform).with(group.id, ::Users::Internal.admin_bot.id) + end + + background_migration + end + end +end diff --git a/spec/migrations/20241112163029_queue_delete_orphaned_groups_spec.rb b/spec/migrations/20241112163029_queue_delete_orphaned_groups_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..d4e514b53dd2d9683d73ac793e2de02688c692de --- /dev/null +++ b/spec/migrations/20241112163029_queue_delete_orphaned_groups_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueDeleteOrphanedGroups, migration: :gitlab_main, feature_category: :groups_and_projects do + let!(:batched_migration) { described_class::MIGRATION } + + it 'does not schedule a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + end + end + + context 'when executed on .com' do + before do + allow(Gitlab).to receive(:com_except_jh?).and_return(true) + end + + describe '#up' do + it 'schedules background migration' do + migrate! + + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :namespaces, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + end + end + + describe '#down' do + it 'removes scheduled background migrations' do + migrate! + schema_migrate_down! + + expect(batched_migration).not_to have_scheduled_batched_migration + end + end + end +end