From 61a927218182ad7a2f0814524f1b51fb0b598762 Mon Sep 17 00:00:00 2001 From: GitLab Duo Date: Thu, 4 Dec 2025 14:22:22 +0000 Subject: [PATCH 1/4] feat: Add sharding keys to web_hook_logs_daily table Implement organization_id, project_id, and group_id sharding keys for the web_hook_logs_daily partitioned table to support multi-tenancy. - Add three sharding key columns with indexes and foreign keys - Create triggers to auto-populate keys from parent web_hooks table - Implement batched background migration for backfilling existing data - Add multi-column not null constraint for new partitions - Update table documentation with sharding key configuration This enables proper data isolation and supports future sharding requirements for webhook logs across organizations, projects, and groups. --- ...fill_web_hook_logs_daily_sharding_keys.yml | 7 + db/docs/web_hook_logs_daily.yml | 5 +- ..._organization_id_to_web_hook_logs_daily.rb | 11 + ...2_add_project_id_to_web_hook_logs_daily.rb | 11 + ...003_add_group_id_to_web_hook_logs_daily.rb | 11 + ..._web_hook_logs_daily_on_organization_id.rb | 20 ++ ...index_web_hook_logs_daily_on_project_id.rb | 20 ++ ...d_index_web_hook_logs_daily_on_group_id.rb | 20 ++ ..._fk_web_hook_logs_daily_organization_id.rb | 23 ++ ...8_add_fk_web_hook_logs_daily_project_id.rb | 23 ++ ...009_add_fk_web_hook_logs_daily_group_id.rb | 23 ++ ...ding_key_trigger_to_web_hook_logs_daily.rb | 34 +++ ...nt_to_web_hook_logs_daily_sharding_keys.rb | 25 +++ ...kfill_web_hook_logs_daily_sharding_keys.rb | 27 +++ db/schema_migrations/20251202100002 | 1 + db/schema_migrations/20251202100003 | 1 + db/schema_migrations/20251202100004 | 1 + db/schema_migrations/20251202100005 | 1 + db/schema_migrations/20251202100006 | 1 + db/schema_migrations/20251202100007 | 1 + db/schema_migrations/20251202100008 | 1 + db/schema_migrations/20251202100009 | 1 + db/schema_migrations/20251202100010 | 1 + db/schema_migrations/20251202100011 | 1 + db/schema_migrations/20251202100012 | 1 + db/structure.sql | 30 +++ ...kfill_web_hook_logs_daily_sharding_keys.rb | 41 ++++ spec/db/schema_spec.rb | 2 +- ..._web_hook_logs_daily_sharding_keys_spec.rb | 196 ++++++++++++++++++ .../gitlab/organizations/sharding_key_spec.rb | 13 +- ...key_trigger_to_web_hook_logs_daily_spec.rb | 124 +++++++++++ ..._web_hook_logs_daily_sharding_keys_spec.rb | 50 +++++ 32 files changed, 723 insertions(+), 4 deletions(-) create mode 100644 db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml create mode 100644 db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb create mode 100644 db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb create mode 100644 db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb create mode 100644 db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb create mode 100644 db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb create mode 100644 db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb create mode 100644 db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb create mode 100644 db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb create mode 100644 db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb create mode 100644 db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb create mode 100644 db/post_migrate/20251202100011_add_not_null_constraint_to_web_hook_logs_daily_sharding_keys.rb create mode 100644 db/post_migrate/20251202100012_queue_backfill_web_hook_logs_daily_sharding_keys.rb create mode 100644 db/schema_migrations/20251202100002 create mode 100644 db/schema_migrations/20251202100003 create mode 100644 db/schema_migrations/20251202100004 create mode 100644 db/schema_migrations/20251202100005 create mode 100644 db/schema_migrations/20251202100006 create mode 100644 db/schema_migrations/20251202100007 create mode 100644 db/schema_migrations/20251202100008 create mode 100644 db/schema_migrations/20251202100009 create mode 100644 db/schema_migrations/20251202100010 create mode 100644 db/schema_migrations/20251202100011 create mode 100644 db/schema_migrations/20251202100012 create mode 100644 lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys_spec.rb create mode 100644 spec/migrations/add_sharding_key_trigger_to_web_hook_logs_daily_spec.rb create mode 100644 spec/migrations/queue_backfill_web_hook_logs_daily_sharding_keys_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml b/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml new file mode 100644 index 00000000000000..2b4246ab1c435f --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml @@ -0,0 +1,7 @@ +--- +migration_job_name: BackfillWebHookLogsDailyShardingKeys +description: Backfill web_hooks_daily sharding key +feature_category: webhooks +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215236 +milestone: '18.7' +queued_migration_version: 20251202100012 diff --git a/db/docs/web_hook_logs_daily.yml b/db/docs/web_hook_logs_daily.yml index 1f152804d1bd5f..ab6045654b6cab 100644 --- a/db/docs/web_hook_logs_daily.yml +++ b/db/docs/web_hook_logs_daily.yml @@ -9,5 +9,8 @@ description: Webhooks logs data partitioned by day. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175379 milestone: '17.8' gitlab_schema: gitlab_main_org -sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/524820 +sharding_key: + project_id: projects + group_id: namespaces + organization_id: organizations table_size: small diff --git a/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb new file mode 100644 index 00000000000000..184278eda29045 --- /dev/null +++ b/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddOrganizationIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] + milestone '18.7' + + def change + # rubocop:disable Migration/PreventAddingColumns -- required for sharding + add_column :web_hook_logs_daily, :organization_id, :bigint, if_not_exists: true + # rubocop:enable Migration/PreventAddingColumns + end +end diff --git a/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb new file mode 100644 index 00000000000000..205ba7e4f5563b --- /dev/null +++ b/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddProjectIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] + milestone '18.7' + + def change + # rubocop:disable Migration/PreventAddingColumns -- required for sharding + add_column :web_hook_logs_daily, :project_id, :bigint, if_not_exists: true + # rubocop:enable Migration/PreventAddingColumns + end +end diff --git a/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb new file mode 100644 index 00000000000000..eb41d36bdd9acc --- /dev/null +++ b/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddGroupIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] + milestone '18.7' + + def change + # rubocop:disable Migration/PreventAddingColumns -- required for sharding + add_column :web_hook_logs_daily, :group_id, :bigint, if_not_exists: true + # rubocop:enable Migration/PreventAddingColumns + end +end diff --git a/db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb b/db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb new file mode 100644 index 00000000000000..aea4f3452932e8 --- /dev/null +++ b/db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddIndexWebHookLogsDailyOnOrganizationId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + INDEX_NAME = 'index_web_hook_logs_daily_on_organization_id' + + def up + # rubocop:disable Migration/PreventIndexCreation -- required for sharding + add_concurrent_partitioned_index :web_hook_logs_daily, :organization_id, name: INDEX_NAME + # rubocop:enable Migration/PreventIndexCreation + end + + def down + remove_concurrent_partitioned_index_by_name :web_hook_logs_daily, INDEX_NAME + end +end diff --git a/db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb b/db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb new file mode 100644 index 00000000000000..cd1f20151f704b --- /dev/null +++ b/db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddIndexWebHookLogsDailyOnProjectId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + INDEX_NAME = 'index_web_hook_logs_daily_on_project_id' + + def up + # rubocop:disable Migration/PreventIndexCreation -- required for sharding + add_concurrent_partitioned_index :web_hook_logs_daily, :project_id, name: INDEX_NAME + # rubocop:enable Migration/PreventIndexCreation + end + + def down + remove_concurrent_partitioned_index_by_name :web_hook_logs_daily, INDEX_NAME + end +end diff --git a/db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb b/db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb new file mode 100644 index 00000000000000..9aa9e07e6b542c --- /dev/null +++ b/db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddIndexWebHookLogsDailyOnGroupId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + INDEX_NAME = 'index_web_hook_logs_daily_on_group_id' + + def up + # rubocop:disable Migration/PreventIndexCreation -- required for sharding + add_concurrent_partitioned_index :web_hook_logs_daily, :group_id, name: INDEX_NAME + # rubocop:enable Migration/PreventIndexCreation + end + + def down + remove_concurrent_partitioned_index_by_name :web_hook_logs_daily, INDEX_NAME + end +end diff --git a/db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb b/db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb new file mode 100644 index 00000000000000..c5d74c7ded8762 --- /dev/null +++ b/db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddFkWebHookLogsDailyOrganizationId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key( + :web_hook_logs_daily, + :organizations, + column: :organization_id, + on_delete: :cascade, + validate: false, + reverse_lock_order: true + ) + end + + def down + remove_partitioned_foreign_key :web_hook_logs_daily, column: :organization_id, reverse_lock_order: true + end +end diff --git a/db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb b/db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb new file mode 100644 index 00000000000000..64be68f7bfcec9 --- /dev/null +++ b/db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddFkWebHookLogsDailyProjectId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key( + :web_hook_logs_daily, + :projects, + column: :project_id, + on_delete: :cascade, + validate: false, + reverse_lock_order: true + ) + end + + def down + remove_partitioned_foreign_key :web_hook_logs_daily, column: :project_id, reverse_lock_order: true + end +end diff --git a/db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb b/db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb new file mode 100644 index 00000000000000..222f28141d81f6 --- /dev/null +++ b/db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddFkWebHookLogsDailyGroupId < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.7' + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key( + :web_hook_logs_daily, + :namespaces, + column: :group_id, + on_delete: :cascade, + validate: false, + reverse_lock_order: true + ) + end + + def down + remove_partitioned_foreign_key :web_hook_logs_daily, column: :group_id, reverse_lock_order: true + end +end diff --git a/db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb b/db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb new file mode 100644 index 00000000000000..ef76d972dcee95 --- /dev/null +++ b/db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class AddShardingKeyTriggerToWebHookLogsDaily < Gitlab::Database::Migration[2.3] + include Gitlab::Database::SchemaHelpers + + milestone '18.7' + + TABLE_NAME = :web_hook_logs_daily + PARENT_TABLE = :web_hooks + FOREIGN_KEY = :web_hook_id + TRIGGER_NAME = 'trigger_web_hook_logs_daily_assign_sharding_keys' + + def up + create_trigger_function(TRIGGER_NAME) do + <<~SQL + IF num_nonnulls(NEW.organization_id, NEW.project_id, NEW.group_id) <> 1 THEN + SELECT organization_id, project_id, group_id + INTO NEW.organization_id, NEW.project_id, NEW.group_id + FROM web_hooks + WHERE web_hooks.id = NEW.web_hook_id; + END IF; + + RETURN NEW; + SQL + end + + create_trigger(TABLE_NAME, TRIGGER_NAME, TRIGGER_NAME, fires: 'BEFORE INSERT OR UPDATE') + end + + def down + drop_trigger(TABLE_NAME, TRIGGER_NAME) + drop_function(TRIGGER_NAME) + end +end diff --git a/db/post_migrate/20251202100011_add_not_null_constraint_to_web_hook_logs_daily_sharding_keys.rb b/db/post_migrate/20251202100011_add_not_null_constraint_to_web_hook_logs_daily_sharding_keys.rb new file mode 100644 index 00000000000000..a57d37dd2ccd94 --- /dev/null +++ b/db/post_migrate/20251202100011_add_not_null_constraint_to_web_hook_logs_daily_sharding_keys.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddNotNullConstraintToWebHookLogsDailyShardingKeys < Gitlab::Database::Migration[2.3] + milestone '18.7' + disable_ddl_transaction! + + def up + add_multi_column_not_null_constraint( + :web_hook_logs_daily, + :organization_id, + :project_id, + :group_id, + validate: false + ) + end + + def down + remove_multi_column_not_null_constraint( + :web_hook_logs_daily, + :organization_id, + :project_id, + :group_id + ) + end +end diff --git a/db/post_migrate/20251202100012_queue_backfill_web_hook_logs_daily_sharding_keys.rb b/db/post_migrate/20251202100012_queue_backfill_web_hook_logs_daily_sharding_keys.rb new file mode 100644 index 00000000000000..a02b57295c0d97 --- /dev/null +++ b/db/post_migrate/20251202100012_queue_backfill_web_hook_logs_daily_sharding_keys.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillWebHookLogsDailyShardingKeys < Gitlab::Database::Migration[2.3] + milestone '18.7' + + restrict_gitlab_migration gitlab_schema: :gitlab_main_org + + MIGRATION = "BackfillWebHookLogsDailyShardingKeys" + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 500 + + def up + return if Gitlab.com_except_jh? + + queue_batched_background_migration( + MIGRATION, + :web_hook_logs_daily, + :id, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :web_hook_logs_daily, :id, []) + end +end diff --git a/db/schema_migrations/20251202100002 b/db/schema_migrations/20251202100002 new file mode 100644 index 00000000000000..75d95f084a4577 --- /dev/null +++ b/db/schema_migrations/20251202100002 @@ -0,0 +1 @@ +eb09b554bf295bb981652bb9bf3aef0a76154e9230fca84f8a64738db5e045dc \ No newline at end of file diff --git a/db/schema_migrations/20251202100003 b/db/schema_migrations/20251202100003 new file mode 100644 index 00000000000000..4fb4e8be4cd1bb --- /dev/null +++ b/db/schema_migrations/20251202100003 @@ -0,0 +1 @@ +9895aa3cae38464a5eb9c2db091afb520daf5d48e90062649d4013d6ad534fb7 \ No newline at end of file diff --git a/db/schema_migrations/20251202100004 b/db/schema_migrations/20251202100004 new file mode 100644 index 00000000000000..d7f627f43e1afa --- /dev/null +++ b/db/schema_migrations/20251202100004 @@ -0,0 +1 @@ +6bfa533b252c2930cb2c4e0ff5851f9a37998b0a9c220b3fe96c3dd3978098e5 \ No newline at end of file diff --git a/db/schema_migrations/20251202100005 b/db/schema_migrations/20251202100005 new file mode 100644 index 00000000000000..48af32b34c36b4 --- /dev/null +++ b/db/schema_migrations/20251202100005 @@ -0,0 +1 @@ +226999b5fb2195256992d8659c668c82de4bd4f3013ad82fc1e8470d5e0154c0 \ No newline at end of file diff --git a/db/schema_migrations/20251202100006 b/db/schema_migrations/20251202100006 new file mode 100644 index 00000000000000..b425420512da0b --- /dev/null +++ b/db/schema_migrations/20251202100006 @@ -0,0 +1 @@ +bb772c0dac78a98dc08a8aaad358b70879c41f3c998f0614155e036519cc12a4 \ No newline at end of file diff --git a/db/schema_migrations/20251202100007 b/db/schema_migrations/20251202100007 new file mode 100644 index 00000000000000..9ffb4a9953e676 --- /dev/null +++ b/db/schema_migrations/20251202100007 @@ -0,0 +1 @@ +8a1d71d79c3b77c3bd3db154dbbef071e13a2e90910e5738e6a134dbced4cac9 \ No newline at end of file diff --git a/db/schema_migrations/20251202100008 b/db/schema_migrations/20251202100008 new file mode 100644 index 00000000000000..4500fab902283f --- /dev/null +++ b/db/schema_migrations/20251202100008 @@ -0,0 +1 @@ +f67e4b79c7cae82af0fa0534f6d9cc915d4999cce1025d4f7ab557190c66d525 \ No newline at end of file diff --git a/db/schema_migrations/20251202100009 b/db/schema_migrations/20251202100009 new file mode 100644 index 00000000000000..92ff5863db799c --- /dev/null +++ b/db/schema_migrations/20251202100009 @@ -0,0 +1 @@ +7a00a7b7c76350638885ee700225d818415a62b5b24c1688bc4adf7621e66ffa \ No newline at end of file diff --git a/db/schema_migrations/20251202100010 b/db/schema_migrations/20251202100010 new file mode 100644 index 00000000000000..b26f99d0679a19 --- /dev/null +++ b/db/schema_migrations/20251202100010 @@ -0,0 +1 @@ +1496cf40bbcb3b9bc484b5043f0d3bcff0218d9b6c707b131ec96f21cc3b622e \ No newline at end of file diff --git a/db/schema_migrations/20251202100011 b/db/schema_migrations/20251202100011 new file mode 100644 index 00000000000000..685a4d932df295 --- /dev/null +++ b/db/schema_migrations/20251202100011 @@ -0,0 +1 @@ +31cf44e2f4f4dd77829bffaec4db85e52be7f891aa818ea7956b519b6099fb29 \ No newline at end of file diff --git a/db/schema_migrations/20251202100012 b/db/schema_migrations/20251202100012 new file mode 100644 index 00000000000000..cea00561b51aea --- /dev/null +++ b/db/schema_migrations/20251202100012 @@ -0,0 +1 @@ +49865bde2c2d4fca03c2a4623a6629a690942d8e67394ff9bcb329ff57e949dc \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6746648605376e..25834517b51619 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -5064,6 +5064,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_web_hook_logs_daily_assign_sharding_keys() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF num_nonnulls(NEW.organization_id, NEW.project_id, NEW.group_id) <> 1 THEN + SELECT organization_id, project_id, group_id + INTO NEW.organization_id, NEW.project_id, NEW.group_id + FROM web_hooks + WHERE web_hooks.id = NEW.web_hook_id; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION unset_has_issues_on_vulnerability_reads() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -6333,6 +6349,9 @@ CREATE TABLE web_hook_logs_daily ( updated_at timestamp without time zone NOT NULL, created_at timestamp without time zone NOT NULL, url_hash text, + organization_id bigint, + project_id bigint, + group_id bigint, CONSTRAINT check_df72cb58f5 CHECK ((char_length(url_hash) <= 44)) ) PARTITION BY RANGE (created_at); @@ -35057,6 +35076,9 @@ ALTER TABLE epic_issues ALTER TABLE bulk_import_batch_trackers ADD CONSTRAINT check_13004cd9a8 CHECK ((num_nonnulls(namespace_id, organization_id, project_id) = 1)) NOT VALID; +ALTER TABLE web_hook_logs_daily + ADD CONSTRAINT check_19dc80d658 CHECK ((num_nonnulls(group_id, organization_id, project_id) = 1)) NOT VALID; + ALTER TABLE workspaces ADD CONSTRAINT check_2a89035b04 CHECK ((personal_access_token_id IS NOT NULL)) NOT VALID; @@ -45229,6 +45251,12 @@ CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id ON vulnerabili CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id_and_note_id ON vulnerability_user_mentions USING btree (vulnerability_id, note_id); +CREATE INDEX index_web_hook_logs_daily_on_group_id ON ONLY web_hook_logs_daily USING btree (group_id); + +CREATE INDEX index_web_hook_logs_daily_on_organization_id ON ONLY web_hook_logs_daily USING btree (organization_id); + +CREATE INDEX index_web_hook_logs_daily_on_project_id ON ONLY web_hook_logs_daily USING btree (project_id); + CREATE INDEX index_web_hook_logs_daily_on_web_hook_id_and_created_at ON ONLY web_hook_logs_daily USING btree (web_hook_id, created_at); CREATE INDEX index_web_hook_logs_daily_part_on_created_at_and_web_hook_id ON ONLY web_hook_logs_daily USING btree (created_at, web_hook_id); @@ -49807,6 +49835,8 @@ CREATE TRIGGER trigger_update_location_on_vulnerability_occurrences_update AFTER CREATE TRIGGER trigger_update_vulnerability_reads_on_vulnerability_update AFTER UPDATE ON vulnerabilities FOR EACH ROW WHEN (((old.present_on_default_branch IS TRUE) AND ((old.severity IS DISTINCT FROM new.severity) OR (old.state IS DISTINCT FROM new.state) OR (old.resolved_on_default_branch IS DISTINCT FROM new.resolved_on_default_branch)))) EXECUTE FUNCTION update_vulnerability_reads_from_vulnerability(); +CREATE TRIGGER trigger_web_hook_logs_daily_assign_sharding_keys BEFORE INSERT OR UPDATE ON web_hook_logs_daily FOR EACH ROW EXECUTE FUNCTION trigger_web_hook_logs_daily_assign_sharding_keys(); + CREATE TRIGGER users_loose_fk_trigger AFTER DELETE ON users REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); CREATE TRIGGER virtual_registries_container_upstreams_loose_fk_trigger AFTER DELETE ON virtual_registries_container_upstreams REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); diff --git a/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys.rb b/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys.rb new file mode 100644 index 00000000000000..94bcf66677d209 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillWebHookLogsDailyShardingKeys < BatchedMigrationJob + operation_name :backfill_web_hook_logs_daily_sharding_keys + feature_category :webhooks + + def perform + each_sub_batch do |sub_batch| + records_to_update = sub_batch.where( + organization_id: nil, + project_id: nil, + group_id: nil + ) + + connection.execute(update_sql(records_to_update)) + end + end + + private + + def update_sql(sub_batch) + <<~SQL + UPDATE + web_hook_logs_daily + SET + organization_id = web_hooks.organization_id, + project_id = web_hooks.project_id, + group_id = web_hooks.group_id + FROM + web_hooks + WHERE + web_hooks.id = web_hook_logs_daily.web_hook_id + AND + web_hook_logs_daily.id IN (#{sub_batch.select(:id).to_sql}) + SQL + end + end + end +end diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 6a231a38f81095..8c8de59bc71fb6 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -247,7 +247,7 @@ # See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87584 # Fixes performance issues with the deletion of web-hooks with many log entries web_hook_logs: %w[web_hook_id], - web_hook_logs_daily: %w[web_hook_id], + web_hook_logs_daily: %w[web_hook_id organization_id group_id project_id], ml_candidates: %w[internal_id], value_stream_dashboard_counts: %w[namespace_id], vulnerability_export_parts: %w[start_id end_id], diff --git a/spec/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys_spec.rb b/spec/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys_spec.rb new file mode 100644 index 00000000000000..9542f82d8d52c7 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_web_hook_logs_daily_sharding_keys_spec.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillWebHookLogsDailyShardingKeys, feature_category: :webhooks do + let(:connection) { ApplicationRecord.connection } + let(:function_name) { 'trigger_web_hook_logs_daily_assign_sharding_keys' } + let(:trigger_name) { function_name } + let(:constraint_name) { 'check_19dc80d658' } + + let(:organizations) { table(:organizations) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:web_hooks) { table(:web_hooks) } + let(:web_hook_logs_daily) { table(:web_hook_logs_daily) } + + let(:start_id) { web_hook_logs_daily.minimum(:id) } + let(:end_id) { web_hook_logs_daily.maximum(:id) } + + let!(:organization) { organizations.create!(name: 'Default', path: 'default') } + + let!(:group) do + namespaces.create!( + name: 'test-group', + path: 'test-group', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:project_namespace) do + namespaces.create!( + name: 'test-project', + path: 'test-project', + organization_id: organization.id + ) + end + + let!(:project) do + projects.create!( + name: 'test-project', + path: 'test-project', + organization_id: organization.id, + namespace_id: group.id, + project_namespace_id: project_namespace.id + ) + end + + let!(:system_hook) do + web_hooks.create!( + type: 'SystemHook', + organization_id: organization.id + ) + end + + let!(:project_hook) do + web_hooks.create!( + type: 'ProjectHook', + project_id: project.id + ) + end + + let!(:group_hook) do + web_hooks.create!( + type: 'GroupHook', + group_id: group.id + ) + end + + let!(:system_log) do + drop_constraint_and_trigger + record = web_hook_logs_daily.create!( + web_hook_id: system_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + add_constraint_and_trigger + record + end + + let!(:project_log) do + drop_constraint_and_trigger + record = web_hook_logs_daily.create!( + web_hook_id: project_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + add_constraint_and_trigger + record + end + + let!(:group_log) do + drop_constraint_and_trigger + record = web_hook_logs_daily.create!( + web_hook_id: group_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + add_constraint_and_trigger + record + end + + subject(:migration) do + described_class.new( + start_id: start_id, + end_id: end_id, + batch_table: :web_hook_logs_daily, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + describe '#perform' do + it 'backfills sharding keys from web_hooks table' do + expect(system_log.reload.organization_id).to be_nil + expect(system_log.reload.project_id).to be_nil + expect(system_log.reload.group_id).to be_nil + + expect(project_log.reload.organization_id).to be_nil + expect(project_log.reload.project_id).to be_nil + expect(project_log.reload.group_id).to be_nil + + expect(group_log.reload.organization_id).to be_nil + expect(group_log.reload.project_id).to be_nil + expect(group_log.reload.group_id).to be_nil + + migration.perform + + expect(system_log.reload.organization_id).to eq(organization.id) + expect(system_log.reload.project_id).to be_nil + expect(system_log.reload.group_id).to be_nil + + expect(project_log.reload.project_id).to eq(project.id) + expect(project_log.reload.organization_id).to be_nil + expect(project_log.reload.group_id).to be_nil + + expect(group_log.reload.group_id).to eq(group.id) + expect(group_log.reload.organization_id).to be_nil + expect(group_log.reload.project_id).to be_nil + end + + it 'does not update records that already have sharding keys' do + # Manually set sharding keys + system_log.update!(organization_id: organization.id) + project_log.update!(project_id: project.id) + group_log.update!(group_id: group.id) + + expect { migration.perform }.not_to change { system_log.reload.updated_at } + end + end + + private + + def drop_constraint_and_trigger + connection.execute( + <<~SQL + DROP TRIGGER IF EXISTS #{trigger_name} ON web_hook_logs_daily; + + ALTER TABLE web_hook_logs_daily DROP CONSTRAINT IF EXISTS #{constraint_name}; + SQL + ) + end + + def add_constraint_and_trigger + connection.execute( + <<~SQL + ALTER TABLE web_hook_logs_daily ADD CONSTRAINT #{constraint_name} CHECK ((num_nonnulls(group_id, organization_id, project_id) = 1)) NOT VALID; + + CREATE TRIGGER #{trigger_name} BEFORE INSERT OR UPDATE ON web_hook_logs_daily FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + ) + end +end diff --git a/spec/lib/gitlab/organizations/sharding_key_spec.rb b/spec/lib/gitlab/organizations/sharding_key_spec.rb index f37a510e271a37..d7d5a0a82282b7 100644 --- a/spec/lib/gitlab/organizations/sharding_key_spec.rb +++ b/spec/lib/gitlab/organizations/sharding_key_spec.rb @@ -10,7 +10,6 @@ let(:allowed_to_be_missing_sharding_key) do [ 'ai_settings', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 - 'web_hook_logs_daily', # temporary copy of web_hook_logs 'uploads_9ba88c4165', # https://gitlab.com/gitlab-org/gitlab/-/issues/398199 'merge_request_diff_files_99208b8fac', # has a desired sharding key instead 'award_emoji_archived', # temp table: https://gitlab.com/gitlab-org/gitlab/-/issues/580326 @@ -49,6 +48,11 @@ bulk_import_batch_trackers.namespace_id bulk_import_batch_trackers.project_id ], # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/213933 + *%w[ + web_hook_logs_daily.organization_id + web_hook_logs_daily.project_id + web_hook_logs_daily.group_id + ], # https://gitlab.com/gitlab-org/gitlab/-/work_items/524820 'security_scans.project_id', # NOT NULL constraint NOT VALID 'keys.organization_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/577246 'oauth_applications.organization_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/579291 @@ -106,7 +110,11 @@ 'merge_request_commits_metadata.project_id', 'sbom_vulnerability_scans.project_id', 'p_duo_workflows_checkpoints.project_id', - 'p_duo_workflows_checkpoints.namespace_id' + 'p_duo_workflows_checkpoints.namespace_id', + # `web_hook_logs_daily` -> https://gitlab.com/gitlab-org/gitlab/-/work_items/524820 + 'web_hook_logs_daily.organization_id', + 'web_hook_logs_daily.group_id', + 'web_hook_logs_daily.project_id' ] end @@ -257,6 +265,7 @@ "fork_networks" => "https://gitlab.com/gitlab-org/gitlab/-/issues/522958", "bulk_import_configurations" => "https://gitlab.com/gitlab-org/gitlab/-/issues/536521", "pool_repositories" => "https://gitlab.com/gitlab-org/gitlab/-/issues/490484", + "web_hook_logs_daily" => "https://gitlab.com/gitlab-org/gitlab/-/work_items/524820", # All the tables below related to uploads are part of the same work to # add sharding key to the table "admin_roles" => "https://gitlab.com/gitlab-org/gitlab/-/issues/553437", diff --git a/spec/migrations/add_sharding_key_trigger_to_web_hook_logs_daily_spec.rb b/spec/migrations/add_sharding_key_trigger_to_web_hook_logs_daily_spec.rb new file mode 100644 index 00000000000000..4590a34e211612 --- /dev/null +++ b/spec/migrations/add_sharding_key_trigger_to_web_hook_logs_daily_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe AddShardingKeyTriggerToWebHookLogsDaily, :migration, feature_category: :webhooks do + let(:organizations) { table(:organizations) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:web_hooks) { table(:web_hooks) } + let(:web_hook_logs_daily) { table(:web_hook_logs_daily) } + + let!(:organization) { organizations.create!(name: 'Default', path: 'default') } + + let!(:group) do + namespaces.create!( + name: 'test-group', + path: 'test-group', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:project_namespace) do + namespaces.create!( + name: 'test-project', + path: 'test-project', + organization_id: organization.id + ) + end + + let!(:project) do + projects.create!( + name: 'test-project', + path: 'test-project', + organization_id: organization.id, + namespace_id: group.id, + project_namespace_id: project_namespace.id + ) + end + + let!(:system_hook) do + web_hooks.create!( + type: 'SystemHook', + organization_id: organization.id + ) + end + + let!(:project_hook) do + web_hooks.create!( + type: 'ProjectHook', + project_id: project.id + ) + end + + let!(:group_hook) do + web_hooks.create!( + type: 'GroupHook', + group_id: group.id + ) + end + + describe '#up' do + it 'installs triggers that assign sharding keys from web_hooks table' do + migrate! + + # Create log for system hook + system_log = web_hook_logs_daily.create!( + web_hook_id: system_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + + # Create log for project hook + project_log = web_hook_logs_daily.create!( + web_hook_id: project_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + + # Create log for group hook + group_log = web_hook_logs_daily.create!( + web_hook_id: group_hook.id, + trigger: 'push_hooks', + url: 'http://example.com', + request_headers: {}, + request_data: {}, + response_headers: {}, + response_body: '', + response_status: '200', + execution_duration: 0.1, + internal_error_message: '' + ) + + # Verify sharding keys were assigned correctly + expect(system_log.reload.organization_id).to eq(organization.id) + expect(system_log.reload.project_id).to be_nil + expect(system_log.reload.group_id).to be_nil + + expect(project_log.reload.project_id).to eq(project.id) + expect(project_log.reload.organization_id).to be_nil + expect(project_log.reload.group_id).to be_nil + + expect(group_log.reload.group_id).to eq(group.id) + expect(group_log.reload.organization_id).to be_nil + expect(group_log.reload.project_id).to be_nil + end + end +end diff --git a/spec/migrations/queue_backfill_web_hook_logs_daily_sharding_keys_spec.rb b/spec/migrations/queue_backfill_web_hook_logs_daily_sharding_keys_spec.rb new file mode 100644 index 00000000000000..c58459e1479b41 --- /dev/null +++ b/spec/migrations/queue_backfill_web_hook_logs_daily_sharding_keys_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillWebHookLogsDailyShardingKeys, migration: :gitlab_main_org, feature_category: :webhooks do + let!(:batched_migration) { described_class::MIGRATION } + + context 'when not on GitLab.com' do + before do + allow(Gitlab).to receive(:com_except_jh?).and_return(false) + end + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main_org, + table_name: :web_hook_logs_daily, + column_name: :id, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end + end + + context 'when on GitLab.com' do + before do + allow(Gitlab).to receive(:com_except_jh?).and_return(true) + end + + it 'does not schedule a batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + end + end + end +end -- GitLab From 0e572485b2c8b9196c5deef0a22ea3a9f1eed61f Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Mon, 8 Dec 2025 12:57:09 -0500 Subject: [PATCH 2/4] fixup! feat: Add sharding keys to web_hook_logs_daily table --- ...1202100004_add_index_web_hook_logs_daily_on_organization_id.rb | 0 .../20251202100005_add_index_web_hook_logs_daily_on_project_id.rb | 0 .../20251202100006_add_index_web_hook_logs_daily_on_group_id.rb | 0 .../20251202100007_add_fk_web_hook_logs_daily_organization_id.rb | 0 .../20251202100008_add_fk_web_hook_logs_daily_project_id.rb | 0 .../20251202100009_add_fk_web_hook_logs_daily_group_id.rb | 0 ...51202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename db/{migrate => post_migrate}/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb (100%) rename db/{migrate => post_migrate}/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb (100%) rename db/{migrate => post_migrate}/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb (100%) rename db/{migrate => post_migrate}/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb (100%) rename db/{migrate => post_migrate}/20251202100008_add_fk_web_hook_logs_daily_project_id.rb (100%) rename db/{migrate => post_migrate}/20251202100009_add_fk_web_hook_logs_daily_group_id.rb (100%) rename db/{migrate => post_migrate}/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb (100%) diff --git a/db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb b/db/post_migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb similarity index 100% rename from db/migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb rename to db/post_migrate/20251202100004_add_index_web_hook_logs_daily_on_organization_id.rb diff --git a/db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb b/db/post_migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb similarity index 100% rename from db/migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb rename to db/post_migrate/20251202100005_add_index_web_hook_logs_daily_on_project_id.rb diff --git a/db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb b/db/post_migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb similarity index 100% rename from db/migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb rename to db/post_migrate/20251202100006_add_index_web_hook_logs_daily_on_group_id.rb diff --git a/db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb b/db/post_migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb similarity index 100% rename from db/migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb rename to db/post_migrate/20251202100007_add_fk_web_hook_logs_daily_organization_id.rb diff --git a/db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb b/db/post_migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb similarity index 100% rename from db/migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb rename to db/post_migrate/20251202100008_add_fk_web_hook_logs_daily_project_id.rb diff --git a/db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb b/db/post_migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb similarity index 100% rename from db/migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb rename to db/post_migrate/20251202100009_add_fk_web_hook_logs_daily_group_id.rb diff --git a/db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb b/db/post_migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb similarity index 100% rename from db/migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb rename to db/post_migrate/20251202100010_add_sharding_key_trigger_to_web_hook_logs_daily.rb -- GitLab From 130bbd028426b5d3ebb6cd519517ccaab1e531c6 Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Wed, 10 Dec 2025 13:22:43 -0500 Subject: [PATCH 3/4] fixup! fixup! feat: Add sharding keys to web_hook_logs_daily table --- ...0001_add_organization_id_to_web_hook_logs_daily.rb | 11 ----------- ...202100002_add_project_id_to_web_hook_logs_daily.rb | 11 ----------- ...51202100003_add_group_id_to_web_hook_logs_daily.rb | 11 ----------- db/schema_migrations/20251202100001 | 1 - db/schema_migrations/20251202100002 | 1 - db/schema_migrations/20251202100003 | 1 - db/structure.sql | 3 --- 7 files changed, 39 deletions(-) delete mode 100644 db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb delete mode 100644 db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb delete mode 100644 db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb delete mode 100644 db/schema_migrations/20251202100001 delete mode 100644 db/schema_migrations/20251202100002 delete mode 100644 db/schema_migrations/20251202100003 diff --git a/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb deleted file mode 100644 index 184278eda29045..00000000000000 --- a/db/migrate/20251202100001_add_organization_id_to_web_hook_logs_daily.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class AddOrganizationIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] - milestone '18.7' - - def change - # rubocop:disable Migration/PreventAddingColumns -- required for sharding - add_column :web_hook_logs_daily, :organization_id, :bigint, if_not_exists: true - # rubocop:enable Migration/PreventAddingColumns - end -end diff --git a/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb deleted file mode 100644 index 205ba7e4f5563b..00000000000000 --- a/db/migrate/20251202100002_add_project_id_to_web_hook_logs_daily.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class AddProjectIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] - milestone '18.7' - - def change - # rubocop:disable Migration/PreventAddingColumns -- required for sharding - add_column :web_hook_logs_daily, :project_id, :bigint, if_not_exists: true - # rubocop:enable Migration/PreventAddingColumns - end -end diff --git a/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb b/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb deleted file mode 100644 index eb41d36bdd9acc..00000000000000 --- a/db/migrate/20251202100003_add_group_id_to_web_hook_logs_daily.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class AddGroupIdToWebHookLogsDaily < Gitlab::Database::Migration[2.3] - milestone '18.7' - - def change - # rubocop:disable Migration/PreventAddingColumns -- required for sharding - add_column :web_hook_logs_daily, :group_id, :bigint, if_not_exists: true - # rubocop:enable Migration/PreventAddingColumns - end -end diff --git a/db/schema_migrations/20251202100001 b/db/schema_migrations/20251202100001 deleted file mode 100644 index b4e4103110cda8..00000000000000 --- a/db/schema_migrations/20251202100001 +++ /dev/null @@ -1 +0,0 @@ -bf0118b1987e962bedcb480aa858efc2c24c310cfb1c35367645122f356c3b6c \ No newline at end of file diff --git a/db/schema_migrations/20251202100002 b/db/schema_migrations/20251202100002 deleted file mode 100644 index 75d95f084a4577..00000000000000 --- a/db/schema_migrations/20251202100002 +++ /dev/null @@ -1 +0,0 @@ -eb09b554bf295bb981652bb9bf3aef0a76154e9230fca84f8a64738db5e045dc \ No newline at end of file diff --git a/db/schema_migrations/20251202100003 b/db/schema_migrations/20251202100003 deleted file mode 100644 index 4fb4e8be4cd1bb..00000000000000 --- a/db/schema_migrations/20251202100003 +++ /dev/null @@ -1 +0,0 @@ -9895aa3cae38464a5eb9c2db091afb520daf5d48e90062649d4013d6ad534fb7 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 25834517b51619..8efaceddd30b2f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -6349,9 +6349,6 @@ CREATE TABLE web_hook_logs_daily ( updated_at timestamp without time zone NOT NULL, created_at timestamp without time zone NOT NULL, url_hash text, - organization_id bigint, - project_id bigint, - group_id bigint, CONSTRAINT check_df72cb58f5 CHECK ((char_length(url_hash) <= 44)) ) PARTITION BY RANGE (created_at); -- GitLab From 25fca3abba3417ec3d1aad15b808c9c773a52b19 Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Wed, 10 Dec 2025 13:37:01 -0500 Subject: [PATCH 4/4] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: GitLab Duo --- .../backfill_web_hook_logs_daily_sharding_keys.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml b/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml index 2b4246ab1c435f..a14529037fea39 100644 --- a/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml +++ b/db/docs/batched_background_migrations/backfill_web_hook_logs_daily_sharding_keys.yml @@ -1,6 +1,6 @@ --- migration_job_name: BackfillWebHookLogsDailyShardingKeys -description: Backfill web_hooks_daily sharding key +description: Backfill web_hook_logs_daily sharding key feature_category: webhooks introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215236 milestone: '18.7' -- GitLab