From 8c6593bc2ff0faca482db7aa1ca71c4c6813a9b7 Mon Sep 17 00:00:00 2001 From: mo khan Date: Thu, 23 Oct 2025 17:05:48 -0600 Subject: [PATCH 1/4] Add backfill migration job to fill spam_logs.organization_id --- .../backfill_spam_logs_organization_id.yml | 8 ++++ db/docs/spam_logs.yml | 1 + ...ueue_backfill_spam_logs_organization_id.rb | 40 +++++++++++++++++++ db/schema_migrations/20251023230133 | 1 + .../backfill_spam_logs_organization_id.rb | 10 +++++ ...backfill_spam_logs_organization_id_spec.rb | 15 +++++++ ...backfill_spam_logs_organization_id_spec.rb | 33 +++++++++++++++ 7 files changed, 108 insertions(+) create mode 100644 db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml create mode 100644 db/post_migrate/20251023230133_queue_backfill_spam_logs_organization_id.rb create mode 100644 db/schema_migrations/20251023230133 create mode 100644 lib/gitlab/background_migration/backfill_spam_logs_organization_id.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb create mode 100644 spec/migrations/20251023230133_queue_backfill_spam_logs_organization_id_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml b/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml new file mode 100644 index 00000000000000..5f8e28110505b4 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillSpamLogsOrganizationId +description: Backfills sharding key `spam_logs.organization_id` from `users`. +feature_category: instance_resiliency +introduced_by_url: +milestone: '18.6' +queued_migration_version: 20251023230133 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/spam_logs.yml b/db/docs/spam_logs.yml index 6738fe6f613bd6..7b01568c966411 100644 --- a/db/docs/spam_logs.yml +++ b/db/docs/spam_logs.yml @@ -19,3 +19,4 @@ desired_sharding_key: sharding_key: organization_id belongs_to: user table_size: small +desired_sharding_key_migration_job_name: BackfillSpamLogsOrganizationId diff --git a/db/post_migrate/20251023230133_queue_backfill_spam_logs_organization_id.rb b/db/post_migrate/20251023230133_queue_backfill_spam_logs_organization_id.rb new file mode 100644 index 00000000000000..80355947681420 --- /dev/null +++ b/db/post_migrate/20251023230133_queue_backfill_spam_logs_organization_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillSpamLogsOrganizationId < Gitlab::Database::Migration[2.3] + milestone '18.6' + restrict_gitlab_migration gitlab_schema: :gitlab_main_org + + MIGRATION = "BackfillSpamLogsOrganizationId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :spam_logs, + :id, + :organization_id, + :users, + :organization_id, + :user_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :spam_logs, + :id, + [ + :organization_id, + :users, + :organization_id, + :user_id + ] + ) + end +end diff --git a/db/schema_migrations/20251023230133 b/db/schema_migrations/20251023230133 new file mode 100644 index 00000000000000..e2c20d596669e2 --- /dev/null +++ b/db/schema_migrations/20251023230133 @@ -0,0 +1 @@ +fda696fcb5d092e2f6a8e3310af7ed0075f4a32978ff3559a5491e4fa2214dcf \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_spam_logs_organization_id.rb b/lib/gitlab/background_migration/backfill_spam_logs_organization_id.rb new file mode 100644 index 00000000000000..81557171dab7e6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_spam_logs_organization_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillSpamLogsOrganizationId < BackfillDesiredShardingKeyJob + operation_name :backfill_spam_logs_organization_id + feature_category :instance_resiliency + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb new file mode 100644 index 00000000000000..4fc1f5df74e2df --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe(Gitlab::BackgroundMigration::BackfillSpamLogsOrganizationId, + feature_category: :instance_resiliency, + schema: schema) do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :spam_logs } + let(:backfill_column) { :organization_id } + let(:backfill_via_table) { :users } + let(:backfill_via_column) { :organization_id } + let(:backfill_via_foreign_key) { :user_id } + end +end diff --git a/spec/migrations/20251023230133_queue_backfill_spam_logs_organization_id_spec.rb b/spec/migrations/20251023230133_queue_backfill_spam_logs_organization_id_spec.rb new file mode 100644 index 00000000000000..20226a2dca8908 --- /dev/null +++ b/spec/migrations/20251023230133_queue_backfill_spam_logs_organization_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillSpamLogsOrganizationId, feature_category: :instance_resiliency do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :spam_logs, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_org, + job_arguments: [ + :organization_id, + :users, + :organization_id, + :user_id + ] + ) + } + end + end +end -- GitLab From 2aed0f5e622ce3cf9cad55b93386620d8aa5b1a8 Mon Sep 17 00:00:00 2001 From: mo khan Date: Thu, 23 Oct 2025 17:08:17 -0600 Subject: [PATCH 2/4] Add MR url --- .../backfill_spam_logs_organization_id.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml b/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml index 5f8e28110505b4..751afb89a3ac94 100644 --- a/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml +++ b/db/docs/batched_background_migrations/backfill_spam_logs_organization_id.yml @@ -2,7 +2,7 @@ migration_job_name: BackfillSpamLogsOrganizationId description: Backfills sharding key `spam_logs.organization_id` from `users`. feature_category: instance_resiliency -introduced_by_url: +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/210030 milestone: '18.6' queued_migration_version: 20251023230133 finalized_by: # version of the migration that finalized this BBM -- GitLab From f9d711c4e4433c14d0c79a9b56634276151e68ac Mon Sep 17 00:00:00 2001 From: mo khan Date: Thu, 23 Oct 2025 17:26:33 -0600 Subject: [PATCH 3/4] Add missing schema to test --- .../backfill_spam_logs_organization_id_spec.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb index 4fc1f5df74e2df..d542b2a4131478 100644 --- a/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb @@ -2,9 +2,7 @@ require 'spec_helper' -RSpec.describe(Gitlab::BackgroundMigration::BackfillSpamLogsOrganizationId, - feature_category: :instance_resiliency, - schema: schema) do +RSpec.describe Gitlab::BackgroundMigration::BackfillSpamLogsOrganizationId, feature_category: :instance_resiliency, schema: 20240611142348 do include_examples 'desired sharding key backfill job' do let(:batch_table) { :spam_logs } let(:backfill_column) { :organization_id } -- GitLab From 934334c4e983d9d9521ebb2a7330a227abb1a9c5 Mon Sep 17 00:00:00 2001 From: mo khan Date: Thu, 23 Oct 2025 17:28:21 -0600 Subject: [PATCH 4/4] Remove schema from spec --- .../backfill_spam_logs_organization_id_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb index d542b2a4131478..33e47552613b11 100644 --- a/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_spam_logs_organization_id_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSpamLogsOrganizationId, feature_category: :instance_resiliency, schema: 20240611142348 do +RSpec.describe Gitlab::BackgroundMigration::BackfillSpamLogsOrganizationId, feature_category: :instance_resiliency do include_examples 'desired sharding key backfill job' do let(:batch_table) { :spam_logs } let(:backfill_column) { :organization_id } -- GitLab