From cebfc715d0a2dd7c84e6ec99a3d2f6da7cd0f9e0 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 22 Aug 2024 14:12:58 -0800 Subject: [PATCH 01/45] Initial output of batched_background_migration generator --- ...backfill_missing_namespace_id_on_notes.yml | 9 +++++ ..._backfill_missing_namespace_id_on_notes.rb | 36 +++++++++++++++++++ .../backfill_missing_namespace_id_on_notes.rb | 22 ++++++++++++ ...fill_missing_namespace_id_on_notes_spec.rb | 9 +++++ ...fill_missing_namespace_id_on_notes_spec.rb | 31 ++++++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml create mode 100644 db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb create mode 100644 lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb create mode 100644 spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml new file mode 100644 index 00000000000000..9a73e73804fdc5 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillMissingNamespaceIdOnNotes +description: Backfills missing namespace_id values to support sharding +feature_category: code_review_workflow +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/479909 +milestone: '17.4' +queued_migration_version: 20240822220027 +finalize_after: 2024-11-31 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb new file mode 100644 index 00000000000000..f7b80cf482356f --- /dev/null +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html +# for more information on when/how to queue batched background migrations + +# Update below commented lines with appropriate values. + +class QueueBackfillMissingNamespaceIdOnNotes < Gitlab::Database::Migration[2.2] + milestone '17.4' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillMissingNamespaceIdOnNotes" + # DELAY_INTERVAL = 2.minutes + # BATCH_SIZE = 1000 + # SUB_BATCH_SIZE = 100 + + def up + # If you are requeueing an already executed migration, you need to delete the prior batched migration record + # for the new enqueue to be executed, else, you can delete this line. + # delete_batched_background_migration(MIGRATION, :notes, :namespace_id, []) + + queue_batched_background_migration( + MIGRATION, + :notes, + :namespace_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :notes, :namespace_id, []) + end +end diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb new file mode 100644 index 00000000000000..8c7f13e2b58f52 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html +# for more information on how to use batched background migrations + +# Update below commented lines with appropriate values. + +module Gitlab + module BackgroundMigration + class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob + # operation_name :my_operation # This is used as the key on collecting metrics + # scope_to ->(relation) { relation.where(column: "value") } + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + # Your action on each sub_batch + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb new file mode 100644 index 00000000000000..69b11bfcd4529d --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillMissingNamespaceIdOnNotes, feature_category: :code_review_workflow do + xit "example" do + expect(true). be_true + end +end diff --git a/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb new file mode 100644 index 00000000000000..d955daf4e7285b --- /dev/null +++ b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillMissingNamespaceIdOnNotes, feature_category: :code_review_workflow do + # This is an auto-generated example test, which we're uncommenting for now in + # order to commit the code (thanks, Rubocop...) + # + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :notes, + column_name: :namespace_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end + # END GENERATED EXAMPLE + # +end -- GitLab From 7b5f4c6b2c5917be0b4e5286fc89cbb345ff2f2d Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:01:33 -0700 Subject: [PATCH 02/45] Remove unused code --- ...0822220027_queue_backfill_missing_namespace_id_on_notes.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index f7b80cf482356f..b9b7adce06a82e 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -16,10 +16,6 @@ class QueueBackfillMissingNamespaceIdOnNotes < Gitlab::Database::Migration[2.2] # SUB_BATCH_SIZE = 100 def up - # If you are requeueing an already executed migration, you need to delete the prior batched migration record - # for the new enqueue to be executed, else, you can delete this line. - # delete_batched_background_migration(MIGRATION, :notes, :namespace_id, []) - queue_batched_background_migration( MIGRATION, :notes, -- GitLab From 19eb5373574e69f8c591ac967db60a7b7b90b7ce Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:01:55 -0700 Subject: [PATCH 03/45] Start with generic default values --- ...22220027_queue_backfill_missing_namespace_id_on_notes.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index b9b7adce06a82e..b7a0741984c8b8 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -11,9 +11,9 @@ class QueueBackfillMissingNamespaceIdOnNotes < Gitlab::Database::Migration[2.2] restrict_gitlab_migration gitlab_schema: :gitlab_main MIGRATION = "BackfillMissingNamespaceIdOnNotes" - # DELAY_INTERVAL = 2.minutes - # BATCH_SIZE = 1000 - # SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 def up queue_batched_background_migration( -- GitLab From 8315cf2c53efb53c7cad25608d2bfa49d18168f9 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:08:54 -0700 Subject: [PATCH 04/45] Remove extraneous comment --- .../backfill_missing_namespace_id_on_notes.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 8c7f13e2b58f52..474df7134c0c46 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -1,10 +1,5 @@ # frozen_string_literal: true -# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html -# for more information on how to use batched background migrations - -# Update below commented lines with appropriate values. - module Gitlab module BackgroundMigration class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob -- GitLab From 7e40ad51fa0557f3aa81d1f671159121bca007dd Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:09:27 -0700 Subject: [PATCH 05/45] Define operation_name in migration --- .../backfill_missing_namespace_id_on_notes.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 474df7134c0c46..3481cbaed63d19 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -3,8 +3,7 @@ module Gitlab module BackgroundMigration class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob - # operation_name :my_operation # This is used as the key on collecting metrics - # scope_to ->(relation) { relation.where(column: "value") } + operation_name :backfill_missing_namespace_id_on_notes # This is used as the key on collecting metrics feature_category :code_review_workflow def perform -- GitLab From 7da76692c8905c547af64ae0948cd6907538ef17 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:28:44 -0700 Subject: [PATCH 06/45] Scope batches to namespace_id = nil --- .../backfill_missing_namespace_id_on_notes.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 3481cbaed63d19..cadd602959660b 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -3,7 +3,8 @@ module Gitlab module BackgroundMigration class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob - operation_name :backfill_missing_namespace_id_on_notes # This is used as the key on collecting metrics + operation_name :backfill_missing_namespace_id_on_notes + scope_to ->(relation) { relation.where(namespace_id: nil) } feature_category :code_review_workflow def perform -- GitLab From f29a039ff900551153a4baf72bbee17f856c4820 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 11:29:11 -0700 Subject: [PATCH 07/45] Build the basic case/when structure for finding namespace_id --- .../backfill_missing_namespace_id_on_notes.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index cadd602959660b..0d9732098ee76f 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -9,7 +9,24 @@ class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob def perform each_sub_batch do |sub_batch| - # Your action on each sub_batch + sub_batch.each do |note| + note.update!(namespace_id: extract_namespace_id(note.noteable_type)) + end + end + end + + private + + def extract_namespace_id(note) + case note.noteable_type + when "AlertManagement::Alert", "Commit", "MergeRequest", "Vulnerability" + note.noteable.project.namespace_id + when "DesignManagement::Design", "Epic", "Issue" + note.noteable.namespace_id + when "Snippet" + note.noteable.author.namespace_id + else + raise "Unknown noteable type: #{noteable_type}" end end end -- GitLab From 73539db34f3a7e742c57cc69e5c0c948843772d8 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 12:10:04 -0700 Subject: [PATCH 08/45] Add schema migration record --- db/schema_migrations/20240822220027 | 1 + 1 file changed, 1 insertion(+) create mode 100644 db/schema_migrations/20240822220027 diff --git a/db/schema_migrations/20240822220027 b/db/schema_migrations/20240822220027 new file mode 100644 index 00000000000000..d078b0a5895168 --- /dev/null +++ b/db/schema_migrations/20240822220027 @@ -0,0 +1 @@ +dff289034006ffbf12d45275cf91d3dc3843c80894cea79738ffec3149edbf76 \ No newline at end of file -- GitLab From 43b7f5432a3d8376575069b915cae402aef30ef3 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 13:55:41 -0700 Subject: [PATCH 09/45] Wrap date in quotes It appears that YAML.load parses unquoted dates using Date.parse instead of treating them as Strings, so force the casting via quotes. --- .../backfill_missing_namespace_id_on_notes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml index 9a73e73804fdc5..fb071487873a81 100644 --- a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -5,5 +5,5 @@ feature_category: code_review_workflow introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/479909 milestone: '17.4' queued_migration_version: 20240822220027 -finalize_after: 2024-11-31 +finalize_after: "2024-11-31" finalized_by: # version of the migration that finalized this BBM -- GitLab From c6ee9095daacb9c9b9095d4dba91b854f5c11500 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 30 Aug 2024 15:05:29 -0700 Subject: [PATCH 10/45] Prefer MRs over Issues, I guess --- .../backfill_missing_namespace_id_on_notes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml index fb071487873a81..3d5c59050f89c1 100644 --- a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -2,7 +2,7 @@ migration_job_name: BackfillMissingNamespaceIdOnNotes description: Backfills missing namespace_id values to support sharding feature_category: code_review_workflow -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/479909 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/163687 milestone: '17.4' queued_migration_version: 20240822220027 finalize_after: "2024-11-31" -- GitLab From 11e0032f06cf7fbb5d5db14e63ea6ca511f5b2e9 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Sat, 31 Aug 2024 06:33:12 -0700 Subject: [PATCH 11/45] Tidy up sample test for rubocop --- .../backfill_missing_namespace_id_on_notes_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 69b11bfcd4529d..d66bd7c1925b8f 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -4,6 +4,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillMissingNamespaceIdOnNotes, feature_category: :code_review_workflow do xit "example" do - expect(true). be_true + expect(true).be_true end end -- GitLab From 35e76d175396d5b4e74a6bcb3aa23cb9b14ba013 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 4 Sep 2024 18:38:24 -0700 Subject: [PATCH 12/45] Use gitlab-optimized batch sizing This approach is recommended by https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#for-gitlab-saas --- ..._backfill_missing_namespace_id_on_notes.rb | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index b7a0741984c8b8..c87871879bcade 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -14,6 +14,8 @@ class QueueBackfillMissingNamespaceIdOnNotes < Gitlab::Database::Migration[2.2] DELAY_INTERVAL = 2.minutes BATCH_SIZE = 1000 SUB_BATCH_SIZE = 100 + GITLAB_OPTIMIZED_BATCH_SIZE = 75_000 + GITLAB_OPTIMIZED_SUB_BATCH_SIZE = 250 def up queue_batched_background_migration( @@ -21,12 +23,27 @@ def up :notes, :namespace_id, job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE + **batch_sizes ) end def down delete_batched_background_migration(MIGRATION, :notes, :namespace_id, []) end + + private + + def batch_sizes + if Gitlab.com_except_jh? + { + batch_size: GITLAB_OPTIMIZED_BATCH_SIZE, + sub_batch_size: GITLAB_OPTIMIZED_SUB_BATCH_SIZE + } + else + { + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + } + end + end end -- GitLab From 604d9c13e56d8a80fa2162a8701dc3ccf3b2e5f8 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 30 Sep 2024 10:22:08 -0700 Subject: [PATCH 13/45] Iterate over notes.id --- ...240822220027_queue_backfill_missing_namespace_id_on_notes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index c87871879bcade..2647eb2fa3f959 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -21,7 +21,7 @@ def up queue_batched_background_migration( MIGRATION, :notes, - :namespace_id, + :id, job_interval: DELAY_INTERVAL, **batch_sizes ) -- GitLab From f60e9b4407c25e20ba9ceba1c7d484d33079b830 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 30 Sep 2024 10:24:41 -0700 Subject: [PATCH 14/45] Remove extraneous scope --- .../backfill_missing_namespace_id_on_notes.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 0d9732098ee76f..caa9f1fbb0f2a6 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -4,7 +4,6 @@ module Gitlab module BackgroundMigration class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob operation_name :backfill_missing_namespace_id_on_notes - scope_to ->(relation) { relation.where(namespace_id: nil) } feature_category :code_review_workflow def perform -- GitLab From da5ddaf2f2f1adb4ff8760e90ee8c3ef986fbaf8 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 30 Sep 2024 10:27:11 -0700 Subject: [PATCH 15/45] Skip #update! if namespace_id.present? --- .../backfill_missing_namespace_id_on_notes.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index caa9f1fbb0f2a6..08cadd43eb086b 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -9,6 +9,8 @@ class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob def perform each_sub_batch do |sub_batch| sub_batch.each do |note| + next if note.namespace_id.present? + note.update!(namespace_id: extract_namespace_id(note.noteable_type)) end end -- GitLab From 326c31ba987c6dd1b37cd29be65ec5dec9ca7d48 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 30 Sep 2024 13:38:14 -0700 Subject: [PATCH 16/45] Update test expecations --- ...2220027_queue_backfill_missing_namespace_id_on_notes_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb index d955daf4e7285b..e7b8b4841a3fba 100644 --- a/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb @@ -18,7 +18,7 @@ migration.after -> { expect(batched_migration).to have_scheduled_batched_migration( table_name: :notes, - column_name: :namespace_id, + column_name: :id, interval: described_class::DELAY_INTERVAL, batch_size: described_class::BATCH_SIZE, sub_batch_size: described_class::SUB_BATCH_SIZE -- GitLab From 8da7617288822355d1ad52a025d4c8d39b660bd9 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 30 Sep 2024 13:59:50 -0700 Subject: [PATCH 17/45] Additional update of spec --- ...240822220027_queue_backfill_missing_namespace_id_on_notes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index 2647eb2fa3f959..8b309adff2efbb 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -28,7 +28,7 @@ def up end def down - delete_batched_background_migration(MIGRATION, :notes, :namespace_id, []) + delete_batched_background_migration(MIGRATION, :notes, :id, []) end private -- GitLab From 8090e12def27dd62a2659ebf4178eb9506ff29ca Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 4 Oct 2024 11:09:45 -0700 Subject: [PATCH 18/45] Update milestone to 17.5 --- .../backfill_missing_namespace_id_on_notes.yml | 2 +- ...240822220027_queue_backfill_missing_namespace_id_on_notes.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml index 3d5c59050f89c1..94fb747d0a4207 100644 --- a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -3,7 +3,7 @@ migration_job_name: BackfillMissingNamespaceIdOnNotes description: Backfills missing namespace_id values to support sharding feature_category: code_review_workflow introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/163687 -milestone: '17.4' +milestone: '17.5' queued_migration_version: 20240822220027 finalize_after: "2024-11-31" finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb index 8b309adff2efbb..c83f1a97a46b16 100644 --- a/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb +++ b/db/post_migrate/20240822220027_queue_backfill_missing_namespace_id_on_notes.rb @@ -6,7 +6,7 @@ # Update below commented lines with appropriate values. class QueueBackfillMissingNamespaceIdOnNotes < Gitlab::Database::Migration[2.2] - milestone '17.4' + milestone '17.5' restrict_gitlab_migration gitlab_schema: :gitlab_main -- GitLab From d52a27ca9c207d7225e2ee57bbf37f4fe9e91d67 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 4 Oct 2024 11:11:30 -0700 Subject: [PATCH 19/45] Return when noteable_type doesn't match --- .../backfill_missing_namespace_id_on_notes.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 08cadd43eb086b..2dec850e4e64b5 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -26,8 +26,6 @@ def extract_namespace_id(note) note.noteable.namespace_id when "Snippet" note.noteable.author.namespace_id - else - raise "Unknown noteable type: #{noteable_type}" end end end -- GitLab From 5dfbcdb7d1521804738e73798e7f744571b5bf59 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 10 Oct 2024 13:46:37 -0700 Subject: [PATCH 20/45] WIP adding tests and improving record loading/processing --- .../backfill_missing_namespace_id_on_notes.rb | 25 +++- ...fill_missing_namespace_id_on_notes_spec.rb | 111 +++++++++++++++++- 2 files changed, 130 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 2dec850e4e64b5..d1ae46b9d0a57f 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -11,7 +11,11 @@ def perform sub_batch.each do |note| next if note.namespace_id.present? - note.update!(namespace_id: extract_namespace_id(note.noteable_type)) + if note.noteable_type.present? + note.update!(namespace_id: extract_namespace_id(note)) + else + note.destroy + end end end end @@ -19,13 +23,26 @@ def perform private def extract_namespace_id(note) + # Attempt to find namespace_id from the project first. + # + if note.project_id + project = Project.find_by_id(note.project_id) + + return project.namespace_id if project + end + + # We have to load the noteable here because we don't have access to the + # usual ActiveRecord relationships to do it for us. + # + noteable = note.noteable_type.constantize.find(note.noteable_id) + case note.noteable_type when "AlertManagement::Alert", "Commit", "MergeRequest", "Vulnerability" - note.noteable.project.namespace_id + noteable.project.namespace_id when "DesignManagement::Design", "Epic", "Issue" - note.noteable.namespace_id + noteable.namespace_id when "Snippet" - note.noteable.author.namespace_id + noteable.author.namespace_id end end end diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index d66bd7c1925b8f..875571c5389fc6 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -3,7 +3,114 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillMissingNamespaceIdOnNotes, feature_category: :code_review_workflow do - xit "example" do - expect(true).be_true + let(:namespaces_table) { table(:namespaces) } + let(:notes_table) { table(:notes) } + let(:projects_table) { table(:projects) } + let(:snippets_table) { table(:snippets) } + let(:users_table) { table(:users) } + + let(:namespace_1) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-1') } + let(:project_namespace_2) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-2', type: 'Project') } + let!(:user_1) { users_table.create!(name: 'bob', email: 'bob@example.com', projects_limit: 1) } + + let!(:project_1) do + projects_table + .create!( + name: 'project1', + path: 'path1', + namespace_id: namespace_1.id, + project_namespace_id: project_namespace_2.id, + visibility_level: 0 + ) + end + + context "when namespace_id is derived from note.project_id" do + let(:alert_management_alert_note) do + notes_table.create!(project_id: project_1.id, noteable_type: "AlertManagement::Alert") + end + + let(:commit_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Commit") } + let(:merge_request_note) { notes_table.create!(project_id: project_1.id, noteable_type: "MergeRequest") } + let(:vulnerability_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Vulnerability") } + let(:design_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Design") } + let(:work_item_note) { notes_table.create!(project_id: project_1.id, noteable_type: "WorkItem") } + let(:issue_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Issue") } + let(:epic_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Epic") } + + it "updates the namespace_id" do + [ + alert_management_alert_note, + commit_note, + merge_request_note, + vulnerability_note, + design_note, + work_item_note, + issue_note, + epic_note + ].each do |test_note| + expect(test_note.project_id).not_to be_nil + + test_note.update_columns(namespace_id: nil) + test_note.reload + + expect(test_note.namespace_id).to be_nil + + described_class.new( + start_id: test_note.id, + end_id: test_note.id, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + + test_note.reload + + expect(test_note.namespace_id).not_to be_nil + expect(test_note.namespace_id).to eq(Project.find(test_note.project_id).namespace_id) + end + end + end + + context "when namespace_id is derived from noteable.author.namespace_id" do + let!(:snippet) { snippets_table.create!(author_id: user_1.id) } + + let(:personal_snippet_note) do + notes_table.create!(author_id: user_1.id, noteable_type: "Snippet", noteable_id: snippet.id) + end + + let(:project_snippet_note) do + notes_table.create!(author_id: user_1.id, noteable_type: "Snippet", noteable_id: snippet.id) + end + + before do + user_1.attributes["namespace_id"] = namespace_1.id + user_1.save! + end + + it "updates the namespace_id" do + [project_snippet_note, personal_snippet_note].each do |test_note| + test_note.update_columns(namespace_id: nil) + test_note.reload + + expect(test_note.namespace_id).to be_nil + + described_class.new( + start_id: test_note.id, + end_id: test_note.id, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + + test_note.reload + + expect(test_note.namespace_id).not_to be_nil + expect(test_note.namespace_id).to eq(test_note.noteable.author.namespace_id) + end + end end end -- GitLab From 11f54ef32b2d41e6a8f87765578051a64968cf66 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 10 Oct 2024 14:13:26 -0700 Subject: [PATCH 21/45] Update test to include better namespace support for snippets --- .../backfill_missing_namespace_id_on_notes_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 875571c5389fc6..a78cfb16e7f5c5 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -11,7 +11,6 @@ let(:namespace_1) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-1') } let(:project_namespace_2) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-2', type: 'Project') } - let!(:user_1) { users_table.create!(name: 'bob', email: 'bob@example.com', projects_limit: 1) } let!(:project_1) do projects_table @@ -24,6 +23,8 @@ ) end + let!(:user_1) { users_table.create!(name: 'bob', email: 'bob@example.com', projects_limit: 1) } + context "when namespace_id is derived from note.project_id" do let(:alert_management_alert_note) do notes_table.create!(project_id: project_1.id, noteable_type: "AlertManagement::Alert") @@ -84,9 +85,8 @@ notes_table.create!(author_id: user_1.id, noteable_type: "Snippet", noteable_id: snippet.id) end - before do - user_1.attributes["namespace_id"] = namespace_1.id - user_1.save! + let!(:user_namespace) do + namespaces_table.create!(name: 'namespace', path: 'user-namespace-path', type: 'User', owner_id: user_1.id) end it "updates the namespace_id" do @@ -109,7 +109,7 @@ test_note.reload expect(test_note.namespace_id).not_to be_nil - expect(test_note.namespace_id).to eq(test_note.noteable.author.namespace_id) + expect(test_note.namespace_id).to eq(user_namespace.id) end end end -- GitLab From eb3f78c0509bb55f59ac9e2fdc1a47d4b7b6c1c5 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 14 Oct 2024 11:00:22 -0700 Subject: [PATCH 22/45] Delete extraneous comments --- ...0027_queue_backfill_missing_namespace_id_on_notes_spec.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb index e7b8b4841a3fba..01c61445fe0ca4 100644 --- a/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/migrations/20240822220027_queue_backfill_missing_namespace_id_on_notes_spec.rb @@ -4,9 +4,6 @@ require_migration! RSpec.describe QueueBackfillMissingNamespaceIdOnNotes, feature_category: :code_review_workflow do - # This is an auto-generated example test, which we're uncommenting for now in - # order to commit the code (thanks, Rubocop...) - # let!(:batched_migration) { described_class::MIGRATION } it 'schedules a new batched migration' do @@ -26,6 +23,4 @@ } end end - # END GENERATED EXAMPLE - # end -- GitLab From 07bd9903519f6e38d6e3f9ea1b0109a386473680 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 16 Oct 2024 11:54:11 -0700 Subject: [PATCH 23/45] Test for nil note.noteable_type --- .../backfill_missing_namespace_id_on_notes.rb | 6 ++++- ...fill_missing_namespace_id_on_notes_spec.rb | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index d1ae46b9d0a57f..a3055c07e9090e 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -11,7 +11,7 @@ def perform sub_batch.each do |note| next if note.namespace_id.present? - if note.noteable_type.present? + if backfillable?(note) note.update!(namespace_id: extract_namespace_id(note)) else note.destroy @@ -22,6 +22,10 @@ def perform private + def backfillable?(note) + note.noteable_type.present? + end + def extract_namespace_id(note) # Attempt to find namespace_id from the project first. # diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index a78cfb16e7f5c5..5dcee463577db2 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -113,4 +113,28 @@ end end end + + context "when noteable_type is nil" do + let(:merge_request_note) { notes_table.create!(project_id: project_1.id, noteable_type: "MergeRequest") } + + it "deletes the note" do + expect_next_instance_of(described_class) do |migration| + expect(migration).to receive(:backfillable?).and_return(false) + end + + test_note = merge_request_note + + migration = described_class.new( + start_id: test_note.id, + end_id: test_note.id, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + + expect { migration.perform }.to change { Note.count }.by(-1) + end + end end -- GitLab From 7cfb524a98845153793cf86cd2ad3e6e4123c10b Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Tue, 5 Nov 2024 13:18:22 -0800 Subject: [PATCH 24/45] Initial commit of CTE approach to improve performance --- .../backfill_missing_namespace_id_on_notes.rb | 68 ++++++++++++++++--- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index a3055c07e9090e..af993cf6aa9d05 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -6,22 +6,72 @@ class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob operation_name :backfill_missing_namespace_id_on_notes feature_category :code_review_workflow + # def perform + # each_sub_batch do |sub_batch| + # sub_batch.each do |note| + # next if note.namespace_id.present? + + # if backfillable?(note) + # note.update!(namespace_id: extract_namespace_id(note)) + # else + # note.destroy + # end + # end + # end + # end + def perform each_sub_batch do |sub_batch| - sub_batch.each do |note| - next if note.namespace_id.present? - - if backfillable?(note) - note.update!(namespace_id: extract_namespace_id(note)) - else - note.destroy - end - end + connection.execute(build_query(sub_batch)) end end private + # rubocop:disable Layout/LineLength -- SQL! + # rubocop:disable Metrics/MethodLength -- I do what I want + def build_query(scope) + records_query = scope.where(namespace_id: nil).select(" + id, + ( + -- This is the same logic that you have in ruby, we need to implement all cases + case + when exists (select 1 from projects where id = notes.project_id) then (select namespace_id from projects where id = notes.project_id) + when noteable_type = 'AlertManagement::Alert' then (select namespace_id from projects where id = (select project_id from alert_management_alerts where noteable_id = notes.id limit 1) limit 1) + -- when noteable_type = 'Commit' then (select namespace_id from projects where id = (select project_id from commits where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'Vulnerability' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) + -- These 3 need to pull namespace_id from the noteable + when noteable_type = 'DesignManagement::Design' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'Epic' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'Issue' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) + -- Snippets pull from author + when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) + + else + -1 + end + ) as namespace_id_to_set + ") + + <<~SQL + with records AS (select * from + ( + #{records_query.to_sql} + ) foo + ), updated_rows as ( + -- updating records with the located namespace_id_to_set value + update notes set namespace_id = namespace_id_to_set from records where records.id=notes.id and namespace_id_to_set <> -1 + ), deleted_rows as ( + -- deleting the records where we couldn't find the namespace id + delete from notes where id IN (select id from records where namespace_id_to_set = -1) + ) + select 1 + SQL + end + # rubocop:enable Layout/LineLength + # rubocop:enable Metrics/MethodLength + def backfillable?(note) note.noteable_type.present? end -- GitLab From 996f5da49bf1c866161ee26b346ab06b50381be7 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Tue, 5 Nov 2024 13:22:12 -0800 Subject: [PATCH 25/45] Fix stupid rubocop error --- .../backfill_missing_namespace_id_on_notes_spec.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 5dcee463577db2..7286a40de8afa9 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -75,7 +75,12 @@ end context "when namespace_id is derived from noteable.author.namespace_id" do - let!(:snippet) { snippets_table.create!(author_id: user_1.id) } + let!(:snippet) do + snippets_table.create!( + author_id: user_1.id, + project_id: project_1.id + ) + end let(:personal_snippet_note) do notes_table.create!(author_id: user_1.id, noteable_type: "Snippet", noteable_id: snippet.id) -- GitLab From 3db059914641f7891fe03626bec04ac575af5a28 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Tue, 5 Nov 2024 15:07:39 -0800 Subject: [PATCH 26/45] Pull namespace_id from project of note --- .../backfill_missing_namespace_id_on_notes.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index af993cf6aa9d05..62021f48496b7c 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -38,7 +38,6 @@ def build_query(scope) case when exists (select 1 from projects where id = notes.project_id) then (select namespace_id from projects where id = notes.project_id) when noteable_type = 'AlertManagement::Alert' then (select namespace_id from projects where id = (select project_id from alert_management_alerts where noteable_id = notes.id limit 1) limit 1) - -- when noteable_type = 'Commit' then (select namespace_id from projects where id = (select project_id from commits where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'Vulnerability' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) -- These 3 need to pull namespace_id from the noteable @@ -47,7 +46,8 @@ def build_query(scope) when noteable_type = 'Issue' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) -- Snippets pull from author when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) - + -- Commits pull namespace_id from the project of the note + when noteable_type = 'Commit' then (select namespace_id from projects where id = (select namespace_id from projects where id = notes.project_id limit 1) limit 1) else -1 end -- GitLab From a1258ab46d64b7313a0b6195adf75361f592ea70 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Tue, 5 Nov 2024 15:07:55 -0800 Subject: [PATCH 27/45] Remove extraneous space --- .../backfill_missing_namespace_id_on_notes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 62021f48496b7c..68f98b13bf740a 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -55,7 +55,7 @@ def build_query(scope) ") <<~SQL - with records AS (select * from + with records AS (select * from ( #{records_query.to_sql} ) foo -- GitLab From 073851bd4178d7890eb293bf2c3720222fc278ae Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 21 Nov 2024 17:31:43 -0800 Subject: [PATCH 28/45] Use correct tables (cut-n-pasta error) --- .../backfill_missing_namespace_id_on_notes.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 68f98b13bf740a..55166515090ce6 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -41,9 +41,9 @@ def build_query(scope) when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'Vulnerability' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) -- These 3 need to pull namespace_id from the noteable - when noteable_type = 'DesignManagement::Design' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) - when noteable_type = 'Epic' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) - when noteable_type = 'Issue' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'DesignManagement::Design' then (select namespace_id from projects where id = (select project_id from design_management_designs where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'Epic' then (select namespace_id from projects where id = (select project_id from epics where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'Issue' then (select namespace_id from projects where id = (select project_id from issues where noteable_id = notes.id limit 1) limit 1) -- Snippets pull from author when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) -- Commits pull namespace_id from the project of the note -- GitLab From d26754400f9f66e91cb3c1387e648336359148f5 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 21 Nov 2024 17:35:21 -0800 Subject: [PATCH 29/45] Exempt from cross-join restriction --- .../backfill_missing_namespace_id_on_notes.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 55166515090ce6..2cbd806eccc0d6 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -22,7 +22,11 @@ class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob def perform each_sub_batch do |sub_batch| - connection.execute(build_query(sub_batch)) + Gitlab::Database.allow_cross_joins_across_databases( + url: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/163687' + ) do + connection.execute(build_query(sub_batch)) + end end end -- GitLab From 172efcfb7c70e02f889b370ae4b5866ee6453c43 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 27 Nov 2024 13:48:56 -0800 Subject: [PATCH 30/45] Update SQL queries --- .../backfill_missing_namespace_id_on_notes.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 2cbd806eccc0d6..d26cf1c84aab52 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -45,9 +45,9 @@ def build_query(scope) when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'Vulnerability' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) -- These 3 need to pull namespace_id from the noteable - when noteable_type = 'DesignManagement::Design' then (select namespace_id from projects where id = (select project_id from design_management_designs where noteable_id = notes.id limit 1) limit 1) - when noteable_type = 'Epic' then (select namespace_id from projects where id = (select project_id from epics where noteable_id = notes.id limit 1) limit 1) - when noteable_type = 'Issue' then (select namespace_id from projects where id = (select project_id from issues where noteable_id = notes.id limit 1) limit 1) + when noteable_type = 'DesignManagement::Design' then (select namespace_id from design_management_designs where id = notes.noteable_id limit 1) + when noteable_type = 'Epic' then (select namespace_id from epics where id = notes.noteable_id limit 1) + when noteable_type = 'Issue' then (select namespace_id from issues where id = notes.noteable_id limit 1) -- Snippets pull from author when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) -- Commits pull namespace_id from the project of the note -- GitLab From 56afdec84ba2fd79e207e42ce4e5324ae3824f5f Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 27 Nov 2024 13:50:33 -0800 Subject: [PATCH 31/45] Update SQL for commit notes --- .../backfill_missing_namespace_id_on_notes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index d26cf1c84aab52..e12123690c6d84 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -51,7 +51,7 @@ def build_query(scope) -- Snippets pull from author when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) -- Commits pull namespace_id from the project of the note - when noteable_type = 'Commit' then (select namespace_id from projects where id = (select namespace_id from projects where id = notes.project_id limit 1) limit 1) + when noteable_type = 'Commit' then (select namespace_id from projects where id = notes.project_id limit 1) else -1 end -- GitLab From 70b6bdb6fff738089efa840b982dd0537d54e6a8 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 19 Dec 2024 12:28:03 -0800 Subject: [PATCH 32/45] Declare organization_id when creating test records --- ...fill_missing_namespace_id_on_notes_spec.rb | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 7286a40de8afa9..8208db5cbf353d 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -8,9 +8,26 @@ let(:projects_table) { table(:projects) } let(:snippets_table) { table(:snippets) } let(:users_table) { table(:users) } + let(:organizations_table) { table(:organizations) } - let(:namespace_1) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-1') } - let(:project_namespace_2) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-2', type: 'Project') } + let!(:organization) { organizations_table.create!(name: 'organization', path: 'organization') } + + let(:namespace_1) do + namespaces_table.create!( + name: 'namespace', + path: 'namespace-path-1', + organization_id: organization.id + ) + end + + let(:project_namespace_2) do + namespaces_table.create!( + name: 'namespace', + path: 'namespace-path-2', + type: 'Project', + organization_id: organization.id + ) + end let!(:project_1) do projects_table @@ -19,7 +36,8 @@ path: 'path1', namespace_id: namespace_1.id, project_namespace_id: project_namespace_2.id, - visibility_level: 0 + visibility_level: 0, + organization_id: organization.id ) end -- GitLab From 497b9a4f5b17394de333dbd14a3f902429b4650f Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 19 Dec 2024 12:37:10 -0800 Subject: [PATCH 33/45] Add organization_id to additional namespace creation --- .../backfill_missing_namespace_id_on_notes_spec.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 8208db5cbf353d..349e109fbccc11 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -109,7 +109,13 @@ end let!(:user_namespace) do - namespaces_table.create!(name: 'namespace', path: 'user-namespace-path', type: 'User', owner_id: user_1.id) + namespaces_table.create!( + name: 'namespace', + path: 'user-namespace-path', + type: 'User', + owner_id: user_1.id, + organization_id: organization.id + ) end it "updates the namespace_id" do -- GitLab From 3fe3d8313a565d2a3d7a0d64b935bf9de3704b10 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 19 Dec 2024 12:37:45 -0800 Subject: [PATCH 34/45] Cleanup fetching of namespace_/group_id from notes --- .../backfill_missing_namespace_id_on_notes.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index e12123690c6d84..e781778dd32af3 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -44,12 +44,13 @@ def build_query(scope) when noteable_type = 'AlertManagement::Alert' then (select namespace_id from projects where id = (select project_id from alert_management_alerts where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'Vulnerability' then (select namespace_id from projects where id = (select project_id from vulnerabilities where noteable_id = notes.id limit 1) limit 1) - -- These 3 need to pull namespace_id from the noteable + -- These 2 need to pull namespace_id from the noteable when noteable_type = 'DesignManagement::Design' then (select namespace_id from design_management_designs where id = notes.noteable_id limit 1) - when noteable_type = 'Epic' then (select namespace_id from epics where id = notes.noteable_id limit 1) when noteable_type = 'Issue' then (select namespace_id from issues where id = notes.noteable_id limit 1) + -- Epics pull in group_id + when noteable_type = 'Epic' then (select group_id from epics where id = notes.noteable_id limit 1) -- Snippets pull from author - when noteable_type = 'Snippet' then (select id from namespaces where id = (select author_id from notes where id = notes.id limit 1) limit 1) + when noteable_type = 'Snippet' then (select id from namespaces where owner_id = (select author_id from notes where id = notes.id limit 1) limit 1) -- Commits pull namespace_id from the project of the note when noteable_type = 'Commit' then (select namespace_id from projects where id = notes.project_id limit 1) else -- GitLab From 6f47588888d9c98bfb0c068270837dac95c68334 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 19 Dec 2024 12:38:56 -0800 Subject: [PATCH 35/45] Remove extraneous comment --- .../backfill_missing_namespace_id_on_notes.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index e781778dd32af3..99a9116aa259f1 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -38,7 +38,6 @@ def build_query(scope) records_query = scope.where(namespace_id: nil).select(" id, ( - -- This is the same logic that you have in ruby, we need to implement all cases case when exists (select 1 from projects where id = notes.project_id) then (select namespace_id from projects where id = notes.project_id) when noteable_type = 'AlertManagement::Alert' then (select namespace_id from projects where id = (select project_id from alert_management_alerts where noteable_id = notes.id limit 1) limit 1) -- GitLab From 9ba3a6c33781965857c8c5e69b949b12a4cc73c8 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 19 Dec 2024 12:58:01 -0800 Subject: [PATCH 36/45] Remove commented out code --- .../backfill_missing_namespace_id_on_notes.rb | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 99a9116aa259f1..9248c250d9ac1b 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -6,20 +6,6 @@ class BackfillMissingNamespaceIdOnNotes < BatchedMigrationJob operation_name :backfill_missing_namespace_id_on_notes feature_category :code_review_workflow - # def perform - # each_sub_batch do |sub_batch| - # sub_batch.each do |note| - # next if note.namespace_id.present? - - # if backfillable?(note) - # note.update!(namespace_id: extract_namespace_id(note)) - # else - # note.destroy - # end - # end - # end - # end - def perform each_sub_batch do |sub_batch| Gitlab::Database.allow_cross_joins_across_databases( -- GitLab From b44a209a7b54c620de0514bf673b0105c0ee2886 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 30 Jan 2025 14:41:06 -0600 Subject: [PATCH 37/45] Update milestone --- .../backfill_missing_namespace_id_on_notes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml index 94fb747d0a4207..c56c48f6c0b1f0 100644 --- a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -3,7 +3,7 @@ migration_job_name: BackfillMissingNamespaceIdOnNotes description: Backfills missing namespace_id values to support sharding feature_category: code_review_workflow introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/163687 -milestone: '17.5' +milestone: '17.9' queued_migration_version: 20240822220027 finalize_after: "2024-11-31" finalized_by: # version of the migration that finalized this BBM -- GitLab From a05c1c2f16baa2317e8db292ba89b5f2fda16190 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 30 Jan 2025 14:41:43 -0600 Subject: [PATCH 38/45] Update to an optimistic finalize_after date --- .../backfill_missing_namespace_id_on_notes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml index c56c48f6c0b1f0..c15162fbf4a839 100644 --- a/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml +++ b/db/docs/batched_background_migrations/backfill_missing_namespace_id_on_notes.yml @@ -5,5 +5,5 @@ feature_category: code_review_workflow introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/163687 milestone: '17.9' queued_migration_version: 20240822220027 -finalize_after: "2024-11-31" +finalize_after: "2025-03-31" finalized_by: # version of the migration that finalized this BBM -- GitLab From 896e35f33991ca448f9561f238a8c132b8b204f3 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 30 Jan 2025 15:38:23 -0600 Subject: [PATCH 39/45] Remove outdated test This test is _potentially_ outdated, from when we were attempting to run this migration in Ruby. We _may_ not need to test for this case going forwards, so I'm removing it but isolating the removal to this single commit so we can restore it easily. --- ...fill_missing_namespace_id_on_notes_spec.rb | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 349e109fbccc11..0e80eead314347 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -142,28 +142,4 @@ end end end - - context "when noteable_type is nil" do - let(:merge_request_note) { notes_table.create!(project_id: project_1.id, noteable_type: "MergeRequest") } - - it "deletes the note" do - expect_next_instance_of(described_class) do |migration| - expect(migration).to receive(:backfillable?).and_return(false) - end - - test_note = merge_request_note - - migration = described_class.new( - start_id: test_note.id, - end_id: test_note.id, - batch_table: :notes, - batch_column: :id, - sub_batch_size: 1, - pause_ms: 0, - connection: ActiveRecord::Base.connection - ) - - expect { migration.perform }.to change { Note.count }.by(-1) - end - end end -- GitLab From fb42df3d506712bef4085d2838263ef5105afabf Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 30 Jan 2025 18:54:26 -0600 Subject: [PATCH 40/45] Improve testing of Epic Notes --- ...fill_missing_namespace_id_on_notes_spec.rb | 81 ++++++++++++++++++- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 0e80eead314347..1c6cdabf6283a9 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -8,6 +8,9 @@ let(:projects_table) { table(:projects) } let(:snippets_table) { table(:snippets) } let(:users_table) { table(:users) } + let(:epics_table) { table(:epics) } + let(:issues_table) { table(:issues) } + let(:work_item_types_table) { table(:work_item_types) } let(:organizations_table) { table(:organizations) } let!(:organization) { organizations_table.create!(name: 'organization', path: 'organization') } @@ -54,7 +57,6 @@ let(:design_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Design") } let(:work_item_note) { notes_table.create!(project_id: project_1.id, noteable_type: "WorkItem") } let(:issue_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Issue") } - let(:epic_note) { notes_table.create!(project_id: project_1.id, noteable_type: "Epic") } it "updates the namespace_id" do [ @@ -64,8 +66,7 @@ vulnerability_note, design_note, work_item_note, - issue_note, - epic_note + issue_note ].each do |test_note| expect(test_note.project_id).not_to be_nil @@ -142,4 +143,78 @@ end end end + + context "when namespace_id is derived from noteable.id" do + let!(:user_namespace) do + namespaces_table.create!( + name: 'namespace', + path: 'user-namespace-path', + type: 'User', + owner_id: user_1.id, + organization_id: organization.id + ) + end + + let!(:work_items_type) do + work_item_types_table.create!( + id: Random.random_number(4000), + name: 'User Story', + correct_id: Random.random_number(4000) + ) + end + + let!(:issue) do + issues_table.create!( + title: "Are you kidding me", + project_id: project_1.id, + author_id: user_1.id, + namespace_id: user_namespace.id, + correct_work_item_type_id: work_items_type.correct_id, + work_item_type_id: work_items_type.id + ) + end + + let!(:epic) do + epics_table.create!( + title: "Example Epic", + group_id: user_namespace.id, + author_id: user_1.id, + iid: Random.random_number(4000), + title_html: "Example", + issue_id: issue.id + ) + end + + let(:epic_note) do + notes_table.create!( + namespace_id: user_namespace.id, + noteable_type: "Epic", + noteable_id: epic.id + ) + end + + it "updates the namespace_id" do + [epic_note].each do |test_note| + test_note.update_columns(namespace_id: nil) + test_note.reload + + expect(test_note.namespace_id).to be_nil + + described_class.new( + start_id: test_note.id, + end_id: test_note.id, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + + test_note.reload + + expect(test_note.namespace_id).not_to be_nil + expect(test_note.namespace_id).to eq(user_namespace.id) + end + end + end end -- GitLab From 8f1270b4ad22ee6a138d464e1b0551f6ed937be0 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 31 Jan 2025 05:49:05 -0600 Subject: [PATCH 41/45] Update query with COALESCE --- .../backfill_missing_namespace_id_on_notes.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 9248c250d9ac1b..71647a929f95fe 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -24,7 +24,8 @@ def build_query(scope) records_query = scope.where(namespace_id: nil).select(" id, ( - case + coalesce( + (case when exists (select 1 from projects where id = notes.project_id) then (select namespace_id from projects where id = notes.project_id) when noteable_type = 'AlertManagement::Alert' then (select namespace_id from projects where id = (select project_id from alert_management_alerts where noteable_id = notes.id limit 1) limit 1) when noteable_type = 'MergeRequest' then (select namespace_id from projects where id = (select project_id from merge_requests where noteable_id = notes.id limit 1) limit 1) @@ -41,7 +42,7 @@ def build_query(scope) else -1 end - ) as namespace_id_to_set + ), -1)) as namespace_id_to_set ") <<~SQL -- GitLab From 1bf9daf58290b144f08e0c598be511d286082815 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 31 Jan 2025 13:57:02 +0000 Subject: [PATCH 42/45] Epics use group namespaces, not user namespaces --- .../backfill_missing_namespace_id_on_notes_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 1c6cdabf6283a9..423ee9f786df47 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -145,11 +145,11 @@ end context "when namespace_id is derived from noteable.id" do - let!(:user_namespace) do + let!(:group_namespace) do namespaces_table.create!( name: 'namespace', - path: 'user-namespace-path', - type: 'User', + path: 'group-namespace-path', + type: 'Group', owner_id: user_1.id, organization_id: organization.id ) @@ -187,7 +187,7 @@ let(:epic_note) do notes_table.create!( - namespace_id: user_namespace.id, + namespace_id: group_namespace.id, noteable_type: "Epic", noteable_id: epic.id ) @@ -213,7 +213,7 @@ test_note.reload expect(test_note.namespace_id).not_to be_nil - expect(test_note.namespace_id).to eq(user_namespace.id) + expect(test_note.namespace_id).to eq(group_namespace.id) end end end -- GitLab From 15fc9d9e243c9b4298c53774e4a4f9ac3d577aae Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 31 Jan 2025 08:21:26 -0600 Subject: [PATCH 43/45] Additional group_id changes --- .../backfill_missing_namespace_id_on_notes_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 423ee9f786df47..09c910f560e46f 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -168,7 +168,7 @@ title: "Are you kidding me", project_id: project_1.id, author_id: user_1.id, - namespace_id: user_namespace.id, + namespace_id: group_namespace.id, correct_work_item_type_id: work_items_type.correct_id, work_item_type_id: work_items_type.id ) @@ -177,7 +177,7 @@ let!(:epic) do epics_table.create!( title: "Example Epic", - group_id: user_namespace.id, + group_id: group_namespace.id, author_id: user_1.id, iid: Random.random_number(4000), title_html: "Example", -- GitLab From 5c9665a4d45711039f94e6cc7d3ff1cd65d017cc Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 31 Jan 2025 10:10:04 -0600 Subject: [PATCH 44/45] Refactor 'foo' alias out of code --- .../backfill_missing_namespace_id_on_notes.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb index 71647a929f95fe..58e29346a3a8d7 100644 --- a/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes.rb @@ -46,10 +46,8 @@ def build_query(scope) ") <<~SQL - with records AS (select * from - ( - #{records_query.to_sql} - ) foo + with records AS ( + #{records_query.to_sql} ), updated_rows as ( -- updating records with the located namespace_id_to_set value update notes set namespace_id = namespace_id_to_set from records where records.id=notes.id and namespace_id_to_set <> -1 -- GitLab From e797f21413116ced1067b783ded36fe68563b021 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 31 Jan 2025 10:14:06 -0600 Subject: [PATCH 45/45] Cleanup development sass --- .../backfill_missing_namespace_id_on_notes_spec.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb index 09c910f560e46f..11f0aa31197fa7 100644 --- a/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_missing_namespace_id_on_notes_spec.rb @@ -165,8 +165,7 @@ let!(:issue) do issues_table.create!( - title: "Are you kidding me", - project_id: project_1.id, + title: "Example Epic", author_id: user_1.id, namespace_id: group_namespace.id, correct_work_item_type_id: work_items_type.correct_id, -- GitLab