From 035374bee4682b539d4c3902f16b14b7da2b7809 Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Tue, 9 Dec 2025 15:53:01 -0500 Subject: [PATCH 1/4] Fully shard uploads_9ba88c4165 This table is a partitioned mirror of uploads. Each of its partitions has been sharded in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/213237. It is now possible to mark uploads_9ba88c4165 as sharded and enforce a constraint. Changelog: other --- db/docs/uploads_9ba88c4165.yml | 5 +++++ ...000_add_uploads_sharding_key_constraint.rb | 20 +++++++++++++++++++ db/schema_migrations/20251206120000 | 1 + db/structure.sql | 3 ++- spec/lib/gitlab/database/sharding_key_spec.rb | 1 - 5 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb create mode 100644 db/schema_migrations/20251206120000 diff --git a/db/docs/uploads_9ba88c4165.yml b/db/docs/uploads_9ba88c4165.yml index 7b10c7e1264f95..201ba1edd3b93f 100644 --- a/db/docs/uploads_9ba88c4165.yml +++ b/db/docs/uploads_9ba88c4165.yml @@ -16,4 +16,9 @@ introduced_by_url: milestone: '17.11' gitlab_schema: gitlab_main_org sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/398199 +sharding_key: + - organization_id + - namespace_id + - project_id + - uploaded_by_user_id table_size: small diff --git a/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb b/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb new file mode 100644 index 00000000000000..5f5dd10b0eefa1 --- /dev/null +++ b/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddUploadsShardingKeyConstraint < Gitlab::Database::Migration[2.3] + milestone '18.7' + disable_ddl_transaction! + + def up + add_multi_column_not_null_constraint( + :uploads, + :organization_id, :namespace_id, :project_id, :uploaded_by_user_id + ) + end + + def down + remove_multi_column_not_null_constraint( + :uploads, + :organization_id, :namespace_id, :project_id, :uploaded_by_user_id + ) + end +end diff --git a/db/schema_migrations/20251206120000 b/db/schema_migrations/20251206120000 new file mode 100644 index 00000000000000..dae055554b04a6 --- /dev/null +++ b/db/schema_migrations/20251206120000 @@ -0,0 +1 @@ +da68f96d8a4dc82b17bcd6484b09b737822bfa7b51319b4cd0953263570e2ca4 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 39a58ec5708501..bf7f22e8090350 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -28495,7 +28495,8 @@ CREATE TABLE uploads ( organization_id bigint, namespace_id bigint, project_id bigint, - CONSTRAINT check_5e9547379c CHECK ((store IS NOT NULL)) + CONSTRAINT check_5e9547379c CHECK ((store IS NOT NULL)), + CONSTRAINT check_c43601b80e CHECK ((num_nonnulls(namespace_id, organization_id, project_id, uploaded_by_user_id) = 1)) ); CREATE SEQUENCE uploads_id_seq diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index f37a510e271a37..e2707e8993bff0 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -11,7 +11,6 @@ [ 'ai_settings', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 'web_hook_logs_daily', # temporary copy of web_hook_logs - 'uploads_9ba88c4165', # https://gitlab.com/gitlab-org/gitlab/-/issues/398199 'merge_request_diff_files_99208b8fac', # has a desired sharding key instead 'award_emoji_archived', # temp table: https://gitlab.com/gitlab-org/gitlab/-/issues/580326 'authentication_event_archived_records' # temp table: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/202447 -- GitLab From ca1a26ebce248a23860251cb540063d11edc0277 Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Tue, 9 Dec 2025 15:58:48 -0500 Subject: [PATCH 2/4] fixup! Fully shard uploads_9ba88c4165 --- db/docs/uploads_9ba88c4165.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/docs/uploads_9ba88c4165.yml b/db/docs/uploads_9ba88c4165.yml index 201ba1edd3b93f..4ac098be77256b 100644 --- a/db/docs/uploads_9ba88c4165.yml +++ b/db/docs/uploads_9ba88c4165.yml @@ -17,8 +17,8 @@ milestone: '17.11' gitlab_schema: gitlab_main_org sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/398199 sharding_key: - - organization_id - - namespace_id - - project_id - - uploaded_by_user_id + organization_id: organizations + namespace_id: namespaces + project_id: projects + uploaded_by_user_id: users table_size: small -- GitLab From a16a72f14564d675b12686ceafe2cb21062595a4 Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Tue, 9 Dec 2025 16:00:51 -0500 Subject: [PATCH 3/4] fixup! fixup! Fully shard uploads_9ba88c4165 --- .../20251206120000_add_uploads_sharding_key_constraint.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb b/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb index 5f5dd10b0eefa1..eeb457916fcb83 100644 --- a/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb +++ b/db/post_migrate/20251206120000_add_uploads_sharding_key_constraint.rb @@ -6,14 +6,14 @@ class AddUploadsShardingKeyConstraint < Gitlab::Database::Migration[2.3] def up add_multi_column_not_null_constraint( - :uploads, + :uploads_9ba88c4165, :organization_id, :namespace_id, :project_id, :uploaded_by_user_id ) end def down remove_multi_column_not_null_constraint( - :uploads, + :uploads_9ba88c4165, :organization_id, :namespace_id, :project_id, :uploaded_by_user_id ) end -- GitLab From 58300420f552abd37af6f1c5b335a14059188b5c Mon Sep 17 00:00:00 2001 From: Tomasz Skorupa Date: Tue, 9 Dec 2025 16:11:19 -0500 Subject: [PATCH 4/4] fixup! fixup! fixup! Fully shard uploads_9ba88c4165 --- db/structure.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index bf7f22e8090350..c7a51eb84d8ac3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10596,7 +10596,8 @@ CREATE TABLE uploads_9ba88c4165 ( mount_point text, secret text, CONSTRAINT check_2849dedce7 CHECK ((char_length(path) <= 511)), - CONSTRAINT check_b888b1df14 CHECK ((char_length(checksum) <= 64)) + CONSTRAINT check_b888b1df14 CHECK ((char_length(checksum) <= 64)), + CONSTRAINT check_f4b854bf62 CHECK ((num_nonnulls(namespace_id, organization_id, project_id, uploaded_by_user_id) = 1)) ) PARTITION BY LIST (model_type); @@ -28495,8 +28496,7 @@ CREATE TABLE uploads ( organization_id bigint, namespace_id bigint, project_id bigint, - CONSTRAINT check_5e9547379c CHECK ((store IS NOT NULL)), - CONSTRAINT check_c43601b80e CHECK ((num_nonnulls(namespace_id, organization_id, project_id, uploaded_by_user_id) = 1)) + CONSTRAINT check_5e9547379c CHECK ((store IS NOT NULL)) ); CREATE SEQUENCE uploads_id_seq -- GitLab