diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 4cc010963c6c902872559e9bb486ce0d1b7e2339..c34d297ca4010758ffe80fcc44a3fce48d35fa82 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -42,6 +42,17 @@ approval_merge_request_rules: - table: approval_policy_rules column: approval_policy_rule_id on_delete: async_nullify +background_operation_jobs: + - table: organizations + column: organization_id + on_delete: async_delete +background_operation_workers: + - table: organizations + column: organization_id + on_delete: async_delete + - table: users + column: user_id + on_delete: async_delete backup_finding_evidences: - table: projects column: project_id diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index ea987ba2ec7d9f5a798bf176f09ab355740b9808..eb6fdc51c31b28f8b59d7da723d88bb133e74314 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -44,7 +44,11 @@ MergeRequest::CommitsMetadata, WebHookLog, MergeRequests::GeneratedRefCommit, - MergeRequests::MergeData + MergeRequests::MergeData, + Gitlab::Database::BackgroundOperation::Worker, + Gitlab::Database::BackgroundOperation::Job, + Gitlab::Database::BackgroundOperation::WorkerCellLocal, + Gitlab::Database::BackgroundOperation::JobCellLocal ]) if Gitlab.ee? diff --git a/db/database_connections/ci.yaml b/db/database_connections/ci.yaml index 73fbef8d258fa2117de55f2406ffac001706f5bd..f99ea9e3285e3070e688cdbb8e76d2e953400e77 100644 --- a/db/database_connections/ci.yaml +++ b/db/database_connections/ci.yaml @@ -3,6 +3,8 @@ description: Cell-local GitLab database holding all CI pipelines, builds, etc. gitlab_schemas: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_ci - gitlab_ci_cell_local lock_gitlab_schemas: diff --git a/db/database_connections/main.yaml b/db/database_connections/main.yaml index bf5e214d659fb1bffbcc5c0574646f2eb73849b0..3b924665a1127940b73cd6a6ce6faf775aec174a 100644 --- a/db/database_connections/main.yaml +++ b/db/database_connections/main.yaml @@ -3,6 +3,8 @@ description: Main GitLab database holding all projects, issues, etc. gitlab_schemas: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_org - gitlab_main_cell diff --git a/db/database_connections/sec.yaml b/db/database_connections/sec.yaml index 6e9e790201f92ba71bfdacab82f74d99efaf76ef..3ebe1cc97e68e5924a0b621af13ee76c62779057 100644 --- a/db/database_connections/sec.yaml +++ b/db/database_connections/sec.yaml @@ -3,6 +3,8 @@ description: Cell-local GitLab database holding Security feature related tables. gitlab_schemas: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_sec lock_gitlab_schemas: - gitlab_main diff --git a/db/docs/background_operation_jobs.yml b/db/docs/background_operation_jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..082ed373fcd89524e757f28250aaa17e1254dafe --- /dev/null +++ b/db/docs/background_operation_jobs.yml @@ -0,0 +1,13 @@ +--- +table_name: background_operation_jobs +classes: +- Gitlab::Database::BackgroundOperation::Job +feature_categories: +- database +description: Store jobs info for each background_operation_workers. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/203234 +milestone: '18.4' +gitlab_schema: gitlab_shared_org +sharding_key: + organization_id: organizations +table_size: small diff --git a/db/docs/background_operation_jobs_cell_local.yml b/db/docs/background_operation_jobs_cell_local.yml new file mode 100644 index 0000000000000000000000000000000000000000..e8a4f1bcae297929d30e20a7207b07394cbe8f2a --- /dev/null +++ b/db/docs/background_operation_jobs_cell_local.yml @@ -0,0 +1,11 @@ +--- +table_name: background_operation_jobs_cell_local +classes: + - Gitlab::Database::BackgroundOperation::JobCellLocal +feature_categories: + - database +description: Store jobs info for each background_operation_workers_cell_local. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/203234 +milestone: '18.4' +gitlab_schema: gitlab_shared_cell_local +table_size: small diff --git a/db/docs/background_operation_workers.yml b/db/docs/background_operation_workers.yml new file mode 100644 index 0000000000000000000000000000000000000000..1ed1901dba47467c427a37f4da4feba47cbd4763 --- /dev/null +++ b/db/docs/background_operation_workers.yml @@ -0,0 +1,15 @@ +--- +table_name: background_operation_workers +classes: +- Gitlab::Database::BackgroundOperation::Worker +feature_categories: +- database +description: Stores information about the large data operations performed async. See + https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/batched_background_operations/ + for more details. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/203234 +milestone: '18.4' +gitlab_schema: gitlab_shared_org +sharding_key: + organization_id: organizations +table_size: small diff --git a/db/docs/background_operation_workers_cell_local.yml b/db/docs/background_operation_workers_cell_local.yml new file mode 100644 index 0000000000000000000000000000000000000000..d93296776fa8c85c4c8850a726c85a8a04e08c62 --- /dev/null +++ b/db/docs/background_operation_workers_cell_local.yml @@ -0,0 +1,13 @@ +--- +table_name: background_operation_workers_cell_local +classes: + - Gitlab::Database::BackgroundOperation::WorkerCellLocal +feature_categories: + - database +description: Stores information about the cell local large data operations performed async. See + https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/batched_background_operations/ + for more details. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/203234 +milestone: '18.4' +gitlab_schema: gitlab_shared_cell_local +table_size: small diff --git a/db/gitlab_schemas/gitlab_ci.yaml b/db/gitlab_schemas/gitlab_ci.yaml index e448d85b182745c63a012f1b7bacef6bec9403a4..5d35f898684f2362047e8d28f6e5bfe647e9efbe 100644 --- a/db/gitlab_schemas/gitlab_ci.yaml +++ b/db/gitlab_schemas/gitlab_ci.yaml @@ -2,9 +2,13 @@ name: gitlab_ci description: Schema for all Organizational CI tables, ex. ci_builds, etc. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local require_sharding_key: true sharding_root_tables: - projects diff --git a/db/gitlab_schemas/gitlab_ci_cell_local.yaml b/db/gitlab_schemas/gitlab_ci_cell_local.yaml index da59ab52adab538cc8e752dad574f3c0408b8602..4e2234da0f686f4131c9c298820f3fa26ebc8bcf 100644 --- a/db/gitlab_schemas/gitlab_ci_cell_local.yaml +++ b/db/gitlab_schemas/gitlab_ci_cell_local.yaml @@ -2,9 +2,13 @@ name: gitlab_ci_cell_local description: Schema for all Cell-local CI tables, ex. ci_cost_settings, etc. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_ci allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_ci require_sharding_key: false diff --git a/db/gitlab_schemas/gitlab_main_cell_local.yaml b/db/gitlab_schemas/gitlab_main_cell_local.yaml index 8ca1ecf8d4d7f1205a1608a116c7c1e648e957d2..5ade3c5fe154ca82d386ebc9cdcaa1ed9efcffb5 100644 --- a/db/gitlab_schemas/gitlab_main_cell_local.yaml +++ b/db/gitlab_schemas/gitlab_main_cell_local.yaml @@ -2,6 +2,8 @@ name: gitlab_main_cell_local description: Schema for all Cell-local tables, ex. zoekt_nodes, geo_nodes. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_clusterwide - gitlab_main_cell @@ -9,6 +11,8 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_clusterwide - gitlab_main_cell diff --git a/db/gitlab_schemas/gitlab_main_cell_setting.yaml b/db/gitlab_schemas/gitlab_main_cell_setting.yaml index 08495a2fd3f9a82479f17e2a9e5570027a21b3f4..dc071cfbf12c5417133c06b77563942617bf2df4 100644 --- a/db/gitlab_schemas/gitlab_main_cell_setting.yaml +++ b/db/gitlab_schemas/gitlab_main_cell_setting.yaml @@ -2,12 +2,16 @@ name: gitlab_main_cell_setting description: Schema for all cell setting tables, ex. application_settings, etc. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_org allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_org diff --git a/db/gitlab_schemas/gitlab_main_jh.yaml b/db/gitlab_schemas/gitlab_main_jh.yaml index b9f374f7b43960da8b5d33314abd3008206f67a1..491f8b10ab174aff021398be1abcc94907659710 100644 --- a/db/gitlab_schemas/gitlab_main_jh.yaml +++ b/db/gitlab_schemas/gitlab_main_jh.yaml @@ -2,6 +2,8 @@ name: gitlab_main_jh description: Schema for all Jihu-specific tables. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_clusterwide @@ -9,6 +11,8 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_clusterwide diff --git a/db/gitlab_schemas/gitlab_main_org.yaml b/db/gitlab_schemas/gitlab_main_org.yaml index bc892c0642d78e564ffac3e71d1eafe28fff8d03..04daae08de7886c51929077b17ba5983a671e76e 100644 --- a/db/gitlab_schemas/gitlab_main_org.yaml +++ b/db/gitlab_schemas/gitlab_main_org.yaml @@ -2,6 +2,8 @@ name: gitlab_main_org description: Schema for all Organization Main tables, ex. namespaces, projects, etc. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_clusterwide @@ -10,6 +12,8 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_clusterwide diff --git a/db/gitlab_schemas/gitlab_main_user.yaml b/db/gitlab_schemas/gitlab_main_user.yaml index 5f7daf6247efd8a7a53dd9938718246e24614c70..1d901b8b31d039bb502c6b62a203e6d5cf27ff9b 100644 --- a/db/gitlab_schemas/gitlab_main_user.yaml +++ b/db/gitlab_schemas/gitlab_main_user.yaml @@ -5,6 +5,8 @@ description: >- but some functionality (e.g. login) is cluster-wide. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_org @@ -13,6 +15,8 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local - gitlab_main - gitlab_main_cell - gitlab_main_org diff --git a/db/gitlab_schemas/gitlab_pm.yaml b/db/gitlab_schemas/gitlab_pm.yaml index 383bae41ffd104709f76548d49441ea0c1bb87e9..b431371e61e0c574f8d4d5f96f3d581af6e747d7 100644 --- a/db/gitlab_schemas/gitlab_pm.yaml +++ b/db/gitlab_schemas/gitlab_pm.yaml @@ -2,7 +2,11 @@ name: gitlab_pm description: Schema for all Cell-local package management features. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local require_sharding_key: false diff --git a/db/gitlab_schemas/gitlab_sec.yaml b/db/gitlab_schemas/gitlab_sec.yaml index c39e9a66da4b5275498a067c0106cd8d657fa721..03ef8ffb3921860913a47c56869c2f450fa1bd5e 100644 --- a/db/gitlab_schemas/gitlab_sec.yaml +++ b/db/gitlab_schemas/gitlab_sec.yaml @@ -2,9 +2,13 @@ name: gitlab_sec description: Schema for all Organizational Security features. allow_cross_joins: - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local allow_cross_transactions: - gitlab_internal - gitlab_shared + - gitlab_shared_org + - gitlab_shared_cell_local require_sharding_key: true sharding_root_tables: - projects diff --git a/db/gitlab_schemas/gitlab_shared.yaml b/db/gitlab_schemas/gitlab_shared.yaml index c2ae7733dfe4468d11556b848ae4c5e6699e28a8..02cf2d8bbe7ad921ee00610a5cea65e3b79b0cd3 100644 --- a/db/gitlab_schemas/gitlab_shared.yaml +++ b/db/gitlab_schemas/gitlab_shared.yaml @@ -1,6 +1,6 @@ name: gitlab_shared description: - Schema for all tables implementing shared features, + (Deprecated - use gitlab_shared_cell_local or gitlab_shared_org instead) Schema for all tables implementing shared features, ex. loose foreign keys, re-indexing, etc. allow_cross_joins: - gitlab_internal diff --git a/db/gitlab_schemas/gitlab_shared_cell_local.yaml b/db/gitlab_schemas/gitlab_shared_cell_local.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c70c7c9d4b031b5c19111fa13ef8322fbd8cb000 --- /dev/null +++ b/db/gitlab_schemas/gitlab_shared_cell_local.yaml @@ -0,0 +1,8 @@ +name: gitlab_shared_cell_local +description: + Schema for all cell local shared tables that do not require sharding. +allow_cross_joins: + - gitlab_internal +allow_cross_transactions: + - gitlab_internal +require_sharding_key: false diff --git a/db/gitlab_schemas/gitlab_shared_org.yaml b/db/gitlab_schemas/gitlab_shared_org.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9adc25ad6add8f1b5b69a4424c9304b44cf97ea7 --- /dev/null +++ b/db/gitlab_schemas/gitlab_shared_org.yaml @@ -0,0 +1,10 @@ +name: gitlab_shared_org +description: + Schema for all shared tables that require sharding. +allow_cross_joins: + - gitlab_internal +allow_cross_transactions: + - gitlab_internal +require_sharding_key: true +sharding_root_tables: + - organizations diff --git a/db/migrate/20250915164710_create_background_operation_workers_cell_local.rb b/db/migrate/20250915164710_create_background_operation_workers_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..d565b2de17be303496f2edbc839e4e83fae60076 --- /dev/null +++ b/db/migrate/20250915164710_create_background_operation_workers_cell_local.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +class CreateBackgroundOperationWorkersCellLocal < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.5' + + disable_ddl_transaction! + + def up + opts = { + if_not_exists: true, + primary_key: [:partition, :id], + options: 'PARTITION BY LIST (partition)' + } + + create_table(:background_operation_workers_cell_local, **opts) do |t| + t.bigserial :id + t.bigint :total_tuple_count + t.bigint :partition, null: false, default: 1 + t.timestamptz :started_at + t.timestamptz :on_hold_until + t.timestamptz :created_at, null: false + t.timestamptz :finished_at + t.integer :batch_size, null: false + t.integer :sub_batch_size, null: false + t.integer :pause_ms, null: false, default: 100 + t.integer :max_batch_size + t.integer :priority, null: false, limit: 2, default: 0 + t.integer :status, null: false, limit: 2, default: 0 + t.integer :interval, null: false, limit: 2 + t.text :job_class_name, null: false, limit: 100 + t.text :batch_class_name, null: false, limit: 100 + t.text :table_name, null: false, limit: 63 + t.text :column_name, null: false, limit: 63 + t.text :gitlab_schema, null: false, limit: 255 + t.jsonb :job_arguments, default: '[]' + t.jsonb :min_cursor + t.jsonb :max_cursor + t.jsonb :next_min_cursor + end + + add_indexes + add_constraints + end + + def down + drop_table(:background_operation_workers_cell_local, if_exists: true) + end + + private + + def add_indexes + add_concurrent_partitioned_index( + :background_operation_workers_cell_local, + :status, + name: 'index_bow_cell_local_by_status' + ) + add_concurrent_partitioned_index( + :background_operation_workers_cell_local, + [:partition, :job_class_name, :table_name, :column_name, :job_arguments], + unique: true, + name: 'index_bow_cell_local_on_unique_configuration' + ) + end + + def add_constraints + add_check_constraint( + :background_operation_workers_cell_local, + '(batch_size >= sub_batch_size)', + check_constraint_name(:background_operation_workers_cell_local, 'batch_size', 'greater_than_sub_batch_size') + ) + add_check_constraint( + :background_operation_workers_cell_local, + '(sub_batch_size > 0)', + check_constraint_name(:background_operation_workers_cell_local, 'sub_batch_size', 'greater_than_zero') + ) + add_check_constraint( + :background_operation_workers_cell_local, + "jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'", + check_constraint_name(:background_operation_workers_cell_local, 'cursors', 'jsonb_array') + ) + add_check_constraint( + :background_operation_workers_cell_local, + 'num_nonnulls(min_cursor, max_cursor) = 2', + check_constraint_name(:background_operation_workers_cell_local, 'cursors', 'not_null') + ) + end +end diff --git a/db/migrate/20250915164741_create_background_operation_jobs_cell_local.rb b/db/migrate/20250915164741_create_background_operation_jobs_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..9f75a3a2729653a80edcf93e5ab3271dda4c4e57 --- /dev/null +++ b/db/migrate/20250915164741_create_background_operation_jobs_cell_local.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class CreateBackgroundOperationJobsCellLocal < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.5' + + disable_ddl_transaction! + + def up + opts = { + if_not_exists: true, + primary_key: [:partition, :id], + options: 'PARTITION BY LIST (partition)' + } + + create_table(:background_operation_jobs_cell_local, **opts) do |t| + t.bigserial :id + t.bigint :partition, null: false, default: 1 + t.bigint :worker_id, null: false + t.bigint :worker_partition, null: false + t.timestamptz :created_at, null: false + t.timestamptz :started_at + t.timestamptz :finished_at + t.integer :batch_size, null: false + t.integer :sub_batch_size, null: false + t.integer :pause_ms, null: false, default: 100 + t.integer :status, null: false, default: 0, limit: 2 + t.integer :attempts, null: false, default: 0, limit: 2 + t.jsonb :metrics, null: false, default: {} + t.jsonb :min_cursor + t.jsonb :max_cursor + end + + add_indexes + add_constraints + end + + def down + drop_table(:background_operation_jobs_cell_local, if_exists: true) + end + + private + + def add_indexes + add_concurrent_partitioned_index( + :background_operation_jobs_cell_local, + :status, + name: 'index_bj_cell_local_by_status' + ) + end + + def add_constraints + add_check_constraint( + :background_operation_jobs_cell_local, + "jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'", + check_constraint_name(:background_operation_jobs_cell_local, 'cursors', 'jsonb_array') + ) + + add_check_constraint( + :background_operation_jobs_cell_local, + "pause_ms >= 100", + check_constraint_name(:background_operation_jobs_cell_local, 'pause_ms', 'minimum_hundred') + ) + + add_check_constraint( + :background_operation_jobs_cell_local, + 'num_nonnulls(min_cursor, max_cursor) = 2', + check_constraint_name(:background_operation_jobs_cell_local, 'cursors', 'not_null') + ) + end +end diff --git a/db/migrate/20250915164841_create_background_operation_workers.rb b/db/migrate/20250915164841_create_background_operation_workers.rb new file mode 100644 index 0000000000000000000000000000000000000000..0366a8ace93c168df9555067d924f30060c6dd1f --- /dev/null +++ b/db/migrate/20250915164841_create_background_operation_workers.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +class CreateBackgroundOperationWorkers < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.5' + + disable_ddl_transaction! + + def up + opts = { + if_not_exists: true, + primary_key: [:partition, :id], + options: 'PARTITION BY LIST (partition)' + } + + create_table(:background_operation_workers, **opts) do |t| + t.uuid :id, default: -> { "gen_random_uuid()" }, null: false + t.bigint :organization_id, null: false + t.bigint :user_id, null: false + t.bigint :total_tuple_count + t.bigint :partition, null: false, default: 1 + t.timestamptz :started_at + t.timestamptz :on_hold_until + t.timestamptz :created_at, null: false + t.timestamptz :finished_at + t.integer :batch_size, null: false + t.integer :sub_batch_size, null: false + t.integer :pause_ms, null: false, default: 100 + t.integer :max_batch_size + t.integer :priority, null: false, limit: 2, default: 0 + t.integer :status, null: false, limit: 2, default: 0 + t.integer :interval, null: false, limit: 2 + t.text :job_class_name, null: false, limit: 100 + t.text :batch_class_name, null: false, limit: 100 + t.text :table_name, null: false, limit: 63 + t.text :column_name, null: false, limit: 63 + t.text :gitlab_schema, null: false, limit: 255 + t.jsonb :job_arguments, default: '[]' + t.jsonb :min_cursor + t.jsonb :max_cursor + t.jsonb :next_min_cursor + end + + add_indexes + add_constraints + end + + def down + drop_table(:background_operation_workers, if_exists: true) + end + + private + + def add_indexes + add_concurrent_partitioned_index( + :background_operation_workers, + :status, + name: 'index_background_operation_workers_by_status' + ) + add_concurrent_partitioned_index( + :background_operation_workers, + :organization_id, + name: 'index_background_operation_workers_by_organization' + ) + add_concurrent_partitioned_index( + :background_operation_workers, + :user_id, + name: 'index_background_operation_workers_by_user' + ) + add_concurrent_partitioned_index( + :background_operation_workers, + :created_at, + name: 'index_background_operation_workers_by_created_at' + ) + add_concurrent_partitioned_index( + :background_operation_workers, + [:partition, :organization_id, :job_class_name, :table_name, :column_name, :job_arguments], + unique: true, + name: 'index_background_operation_workers_on_unique_configuration' + ) + end + + def add_constraints + add_check_constraint( + :background_operation_workers, + '(batch_size >= sub_batch_size)', + check_constraint_name(:background_operation_workers, 'batch_size', 'greater_than_sub_batch_size') + ) + add_check_constraint( + :background_operation_workers, + '(sub_batch_size > 0)', + check_constraint_name(:background_operation_workers, 'sub_batch_size', 'greater_than_zero') + ) + add_check_constraint( + :background_operation_workers, + "jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'", + check_constraint_name(:background_operation_workers, 'cursors', 'jsonb_array') + ) + add_check_constraint( + :background_operation_workers, + 'num_nonnulls(min_cursor, max_cursor) = 2', + check_constraint_name(:background_operation_workers, 'cursors', 'not_null') + ) + end +end diff --git a/db/migrate/20250915164941_create_background_operation_jobs.rb b/db/migrate/20250915164941_create_background_operation_jobs.rb new file mode 100644 index 0000000000000000000000000000000000000000..049b0a6e5920c2e541635d0689d1eef00790878d --- /dev/null +++ b/db/migrate/20250915164941_create_background_operation_jobs.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +class CreateBackgroundOperationJobs < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + milestone '18.5' + + disable_ddl_transaction! + + def up + opts = { + if_not_exists: true, + primary_key: [:partition, :id], + options: 'PARTITION BY LIST (partition)' + } + + create_table(:background_operation_jobs, **opts) do |t| + t.uuid :id, default: -> { "gen_random_uuid()" }, null: false + t.uuid :worker_id, null: false + t.bigint :organization_id, null: false + t.bigint :partition, null: false, default: 1 + t.bigint :worker_partition, null: false + t.timestamptz :created_at, null: false + t.timestamptz :started_at + t.timestamptz :finished_at + t.integer :batch_size, null: false + t.integer :sub_batch_size, null: false + t.integer :pause_ms, null: false, default: 100 + t.integer :status, null: false, default: 0, limit: 2 + t.integer :attempts, null: false, default: 0, limit: 2 + t.jsonb :metrics, null: false, default: {} + t.jsonb :min_cursor + t.jsonb :max_cursor + end + + add_indexes + add_constraints + end + + def down + drop_table(:background_operation_jobs, if_exists: true) + end + + private + + def add_indexes + add_concurrent_partitioned_index( + :background_operation_jobs, + :status, + name: 'index_background_jobs_by_status' + ) + add_concurrent_partitioned_index( + :background_operation_jobs, + :organization_id, + name: 'index_background_operation_jobs_by_organization' + ) + add_concurrent_partitioned_index( + :background_operation_jobs, + :created_at, + name: 'index_background_operation_jobs_by_created_at' + ) + end + + def add_constraints + add_check_constraint( + :background_operation_jobs, + "jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'", + check_constraint_name(:background_operation_jobs, 'cursors', 'jsonb_array') + ) + + add_check_constraint( + :background_operation_jobs, + "pause_ms >= 100", + check_constraint_name(:background_operation_jobs, 'pause_ms', 'minimum_hundred') + ) + + add_check_constraint( + :background_operation_jobs, + 'num_nonnulls(min_cursor, max_cursor) = 2', + check_constraint_name(:background_operation_jobs, 'cursors', 'not_null') + ) + end +end diff --git a/db/schema_migrations/20250915164710 b/db/schema_migrations/20250915164710 new file mode 100644 index 0000000000000000000000000000000000000000..12b662e85ffbf4ee32691e947fee1b135c49ae16 --- /dev/null +++ b/db/schema_migrations/20250915164710 @@ -0,0 +1 @@ +0eceffc4bc04a3f827dd5b2bc8e6c15096890d307c4df8f11ebfa3f8fea5442f \ No newline at end of file diff --git a/db/schema_migrations/20250915164741 b/db/schema_migrations/20250915164741 new file mode 100644 index 0000000000000000000000000000000000000000..5e8e60f64983f0d06582ed8d0a043d0e8a20022f --- /dev/null +++ b/db/schema_migrations/20250915164741 @@ -0,0 +1 @@ +a1a06b3ecae01c65514e92773dc438633f4c6f38d6bc80dd10b7e11ca7f36293 \ No newline at end of file diff --git a/db/schema_migrations/20250915164841 b/db/schema_migrations/20250915164841 new file mode 100644 index 0000000000000000000000000000000000000000..8d4d64526eee39ff2dd41072a66727617c407e6d --- /dev/null +++ b/db/schema_migrations/20250915164841 @@ -0,0 +1 @@ +f0bf60e933ed6adb4aca74ba1149784e1fc386ad4711c1d55570c14462659995 \ No newline at end of file diff --git a/db/schema_migrations/20250915164941 b/db/schema_migrations/20250915164941 new file mode 100644 index 0000000000000000000000000000000000000000..333ee199eaff9ad78925b9793e255f150bf04d8a --- /dev/null +++ b/db/schema_migrations/20250915164941 @@ -0,0 +1 @@ +95a1e57803edcfb3c9bdf862cf5a51980bfb6684ff09fb42371e8d786e5301df \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 73340652aedb470b4445c2974b78903e3a5a1499..5636beea022ee7d10e7fa60ced6069c3910d88fb 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4699,6 +4699,125 @@ CREATE TABLE audit_events ( ) PARTITION BY RANGE (created_at); +CREATE TABLE background_operation_jobs ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + worker_id uuid NOT NULL, + organization_id bigint NOT NULL, + partition bigint DEFAULT 1 NOT NULL, + worker_partition bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + started_at timestamp with time zone, + finished_at timestamp with time zone, + batch_size integer NOT NULL, + sub_batch_size integer NOT NULL, + pause_ms integer DEFAULT 100 NOT NULL, + status smallint DEFAULT 0 NOT NULL, + attempts smallint DEFAULT 0 NOT NULL, + metrics jsonb DEFAULT '{}'::jsonb NOT NULL, + min_cursor jsonb, + max_cursor jsonb, + CONSTRAINT check_22e75767e4 CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))), + CONSTRAINT check_b922a72749 CHECK ((pause_ms >= 100)), + CONSTRAINT check_fc1d4517f5 CHECK ((num_nonnulls(min_cursor, max_cursor) = 2)) +) +PARTITION BY LIST (partition); + +CREATE TABLE background_operation_jobs_cell_local ( + id bigint NOT NULL, + partition bigint DEFAULT 1 NOT NULL, + worker_id bigint NOT NULL, + worker_partition bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + started_at timestamp with time zone, + finished_at timestamp with time zone, + batch_size integer NOT NULL, + sub_batch_size integer NOT NULL, + pause_ms integer DEFAULT 100 NOT NULL, + status smallint DEFAULT 0 NOT NULL, + attempts smallint DEFAULT 0 NOT NULL, + metrics jsonb DEFAULT '{}'::jsonb NOT NULL, + min_cursor jsonb, + max_cursor jsonb, + CONSTRAINT check_00bb39bb33 CHECK ((pause_ms >= 100)), + CONSTRAINT check_5b84acc749 CHECK ((num_nonnulls(min_cursor, max_cursor) = 2)), + CONSTRAINT check_ebc3302442 CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))) +) +PARTITION BY LIST (partition); + +CREATE TABLE background_operation_workers ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + organization_id bigint NOT NULL, + user_id bigint NOT NULL, + total_tuple_count bigint, + partition bigint DEFAULT 1 NOT NULL, + started_at timestamp with time zone, + on_hold_until timestamp with time zone, + created_at timestamp with time zone NOT NULL, + finished_at timestamp with time zone, + batch_size integer NOT NULL, + sub_batch_size integer NOT NULL, + pause_ms integer DEFAULT 100 NOT NULL, + max_batch_size integer, + priority smallint DEFAULT 0 NOT NULL, + status smallint DEFAULT 0 NOT NULL, + "interval" smallint NOT NULL, + job_class_name text NOT NULL, + batch_class_name text NOT NULL, + table_name text NOT NULL, + column_name text NOT NULL, + gitlab_schema text NOT NULL, + job_arguments jsonb DEFAULT '"[]"'::jsonb, + min_cursor jsonb, + max_cursor jsonb, + next_min_cursor jsonb, + CONSTRAINT check_10f672741a CHECK ((char_length(column_name) <= 63)), + CONSTRAINT check_510f6260d5 CHECK ((char_length(gitlab_schema) <= 255)), + CONSTRAINT check_63fe8b8121 CHECK ((sub_batch_size > 0)), + CONSTRAINT check_7f88b7751b CHECK ((char_length(job_class_name) <= 100)), + CONSTRAINT check_91cc32fc67 CHECK ((char_length(batch_class_name) <= 100)), + CONSTRAINT check_c316362d95 CHECK ((char_length(table_name) <= 63)), + CONSTRAINT check_c74b62c410 CHECK ((batch_size >= sub_batch_size)), + CONSTRAINT check_e91dfde154 CHECK ((num_nonnulls(min_cursor, max_cursor) = 2)), + CONSTRAINT check_f1affe613c CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))) +) +PARTITION BY LIST (partition); + +CREATE TABLE background_operation_workers_cell_local ( + id bigint NOT NULL, + total_tuple_count bigint, + partition bigint DEFAULT 1 NOT NULL, + started_at timestamp with time zone, + on_hold_until timestamp with time zone, + created_at timestamp with time zone NOT NULL, + finished_at timestamp with time zone, + batch_size integer NOT NULL, + sub_batch_size integer NOT NULL, + pause_ms integer DEFAULT 100 NOT NULL, + max_batch_size integer, + priority smallint DEFAULT 0 NOT NULL, + status smallint DEFAULT 0 NOT NULL, + "interval" smallint NOT NULL, + job_class_name text NOT NULL, + batch_class_name text NOT NULL, + table_name text NOT NULL, + column_name text NOT NULL, + gitlab_schema text NOT NULL, + job_arguments jsonb DEFAULT '"[]"'::jsonb, + min_cursor jsonb, + max_cursor jsonb, + next_min_cursor jsonb, + CONSTRAINT check_1da63db6a8 CHECK ((char_length(table_name) <= 63)), + CONSTRAINT check_4cc5ecb4f2 CHECK ((char_length(column_name) <= 63)), + CONSTRAINT check_5f184cd88f CHECK ((char_length(gitlab_schema) <= 255)), + CONSTRAINT check_9d0c37a905 CHECK ((char_length(batch_class_name) <= 100)), + CONSTRAINT check_be878382ae CHECK ((batch_size >= sub_batch_size)), + CONSTRAINT check_d94474cbf2 CHECK ((char_length(job_class_name) <= 100)), + CONSTRAINT check_e40b641a88 CHECK ((num_nonnulls(min_cursor, max_cursor) = 2)), + CONSTRAINT check_f9383a3f2e CHECK ((sub_batch_size > 0)), + CONSTRAINT check_f9caba0499 CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))) +) +PARTITION BY LIST (partition); + CREATE TABLE backup_finding_evidences ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, @@ -12326,6 +12445,24 @@ CREATE TABLE aws_roles ( CONSTRAINT check_57adedab55 CHECK ((char_length(region) <= 255)) ); +CREATE SEQUENCE background_operation_jobs_cell_local_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE background_operation_jobs_cell_local_id_seq OWNED BY background_operation_jobs_cell_local.id; + +CREATE SEQUENCE background_operation_workers_cell_local_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE background_operation_workers_cell_local_id_seq OWNED BY background_operation_workers_cell_local.id; + CREATE TABLE badges ( id bigint NOT NULL, link_url character varying NOT NULL, @@ -30178,6 +30315,10 @@ ALTER TABLE ONLY automation_rules ALTER COLUMN id SET DEFAULT nextval('automatio ALTER TABLE ONLY award_emoji ALTER COLUMN id SET DEFAULT nextval('award_emoji_id_seq'::regclass); +ALTER TABLE ONLY background_operation_jobs_cell_local ALTER COLUMN id SET DEFAULT nextval('background_operation_jobs_cell_local_id_seq'::regclass); + +ALTER TABLE ONLY background_operation_workers_cell_local ALTER COLUMN id SET DEFAULT nextval('background_operation_workers_cell_local_id_seq'::regclass); + ALTER TABLE ONLY badges ALTER COLUMN id SET DEFAULT nextval('badges_id_seq'::regclass); ALTER TABLE ONLY batched_background_migration_job_transition_logs ALTER COLUMN id SET DEFAULT nextval('batched_background_migration_job_transition_logs_id_seq'::regclass); @@ -32626,6 +32767,18 @@ ALTER TABLE ONLY award_emoji ALTER TABLE ONLY aws_roles ADD CONSTRAINT aws_roles_pkey PRIMARY KEY (user_id); +ALTER TABLE ONLY background_operation_jobs_cell_local + ADD CONSTRAINT background_operation_jobs_cell_local_pkey PRIMARY KEY (partition, id); + +ALTER TABLE ONLY background_operation_jobs + ADD CONSTRAINT background_operation_jobs_pkey PRIMARY KEY (partition, id); + +ALTER TABLE ONLY background_operation_workers_cell_local + ADD CONSTRAINT background_operation_workers_cell_local_pkey PRIMARY KEY (partition, id); + +ALTER TABLE ONLY background_operation_workers + ADD CONSTRAINT background_operation_workers_pkey PRIMARY KEY (partition, id); + ALTER TABLE ONLY backup_finding_evidences ADD CONSTRAINT backup_finding_evidences_pkey PRIMARY KEY (original_record_identifier, date); @@ -38307,6 +38460,22 @@ CREATE UNIQUE INDEX index_aws_roles_on_role_external_id ON aws_roles USING btree CREATE UNIQUE INDEX index_aws_roles_on_user_id ON aws_roles USING btree (user_id); +CREATE INDEX index_background_jobs_by_status ON ONLY background_operation_jobs USING btree (status); + +CREATE INDEX index_background_operation_jobs_by_created_at ON ONLY background_operation_jobs USING btree (created_at); + +CREATE INDEX index_background_operation_jobs_by_organization ON ONLY background_operation_jobs USING btree (organization_id); + +CREATE INDEX index_background_operation_workers_by_created_at ON ONLY background_operation_workers USING btree (created_at); + +CREATE INDEX index_background_operation_workers_by_organization ON ONLY background_operation_workers USING btree (organization_id); + +CREATE INDEX index_background_operation_workers_by_status ON ONLY background_operation_workers USING btree (status); + +CREATE INDEX index_background_operation_workers_by_user ON ONLY background_operation_workers USING btree (user_id); + +CREATE UNIQUE INDEX index_background_operation_workers_on_unique_configuration ON ONLY background_operation_workers USING btree (partition, organization_id, job_class_name, table_name, column_name, job_arguments); + CREATE INDEX index_backup_finding_evidences_on_fk ON ONLY backup_finding_evidences USING btree (finding_id); CREATE INDEX index_backup_finding_evidences_on_project_id ON ONLY backup_finding_evidences USING btree (project_id); @@ -38381,6 +38550,8 @@ CREATE INDEX index_batched_jobs_on_batched_migration_id_and_status ON batched_ba CREATE UNIQUE INDEX index_batched_migrations_on_gl_schema_and_unique_configuration ON batched_background_migrations USING btree (gitlab_schema, job_class_name, table_name, column_name, job_arguments); +CREATE INDEX index_bj_cell_local_by_status ON ONLY background_operation_jobs_cell_local USING btree (status); + CREATE INDEX index_board_assignees_on_assignee_id ON board_assignees USING btree (assignee_id); CREATE UNIQUE INDEX index_board_assignees_on_board_id_and_assignee_id ON board_assignees USING btree (board_id, assignee_id); @@ -38467,6 +38638,10 @@ CREATE INDEX index_boards_on_milestone_id ON boards USING btree (milestone_id); CREATE INDEX index_boards_on_project_id ON boards USING btree (project_id); +CREATE INDEX index_bow_cell_local_by_status ON ONLY background_operation_workers_cell_local USING btree (status); + +CREATE UNIQUE INDEX index_bow_cell_local_on_unique_configuration ON ONLY background_operation_workers_cell_local USING btree (partition, job_class_name, table_name, column_name, job_arguments); + CREATE UNIQUE INDEX index_branch_rule_squash_options_on_protected_branch_id ON projects_branch_rules_squash_options USING btree (protected_branch_id); CREATE UNIQUE INDEX index_broadcast_dismissals_on_user_id_and_broadcast_message_id ON user_broadcast_message_dismissals USING btree (user_id, broadcast_message_id); diff --git a/doc/development/cells/_index.md b/doc/development/cells/_index.md index 1c4df59625f3cf2ec2f50dd6bb5fa20d51b213bd..d360e42545874c62307f7aed2c6090d3f47dea09 100644 --- a/doc/development/cells/_index.md +++ b/doc/development/cells/_index.md @@ -22,6 +22,8 @@ Below are available schemas related to Cells and Organizations: | `gitlab_ci` | Use for all tables in the `ci:` database that are for an Organization. For example, `ci_pipelines` and `ci_builds` | | `gitlab_ci_cell_local` | For tables in the `ci:` database that are related to features that is distinct for each cell. For example, `instance_type_ci_runners`, or `ci_cost_settings`. These cell-local tables should not have any foreign key references from/to organization tables. | | `gitlab_main_user` | Schema for all User-related tables, ex. `users`, `emails`, etc. Most user functionality is organizational level so should use `gitlab_main_org` instead (e.g. commenting on an issue). For user functionality that is not organizational level, use this schema. Tables on this schema must strictly belong to a user. | +| `gitlab_shared_org` | Schema for tables with data across multiple databases and has `organization_id` for sharding. These tables inherit from `Gitlab::Database::SharedModel`. | +| `gitlab_shared_cell_local` | Schema for cell local shared tables that do not require sharding and exist across multiple databases. For example, `loose_foreign_keys_deleted_records`. These tables also inherit from `Gitlab::Database::SharedModel`. | Most tables will require a [sharding key](../organization/_index.md#defining-a-sharding-key-for-all-organizational-tables) to be defined. diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index 91eb235931113eff03996e2c8fe47ba6ed0e3fac..8e9c11e87b2a9f47584721bddcfc95c0190ca6cb 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -33,10 +33,12 @@ Each table of GitLab needs to have a `gitlab_schema` assigned: | `gitlab_ci` | All CI tables that are being stored in the `ci:` database (for example, `ci_pipelines`, `ci_builds`) | | | `gitlab_ci_cell_local` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | | `gitlab_geo` | All Geo tables that are being stored in the `geo:` database (for example, like `project_registry`, `secondary_usage_data`) | | -| `gitlab_shared` | All application tables that contain data across all decomposed databases (for example, `loose_foreign_keys_deleted_records`) for models that inherit from `Gitlab::Database::SharedModel`. | | | `gitlab_internal` | All internal tables of Rails and PostgreSQL (for example, `ar_internal_metadata`, `schema_migrations`, `pg_*`) | | | `gitlab_pm` | All tables that store `package_metadata`| It is an alias for `gitlab_main`, to be replaced with `gitlab_sec` | | `gitlab_sec` | All Security and Vulnerability feature tables to be stored in the `sec:` database | [Decomposition in progress](https://gitlab.com/groups/gitlab-org/-/epics/13043) | +| `gitlab_shared` | Deprecated, refer `gitlab_shared_cell_local` or `gitlab_shared_org` | | +| `gitlab_shared_cell_local` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | +| `gitlab_shared_org` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | More schemas to be introduced with additional decomposed databases diff --git a/lib/gitlab/database/background_operation/common_job.rb b/lib/gitlab/database/background_operation/common_job.rb new file mode 100644 index 0000000000000000000000000000000000000000..f16ba7fb9655d06414acbe7051b670024f2e3609 --- /dev/null +++ b/lib/gitlab/database/background_operation/common_job.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + module CommonJob + extend ActiveSupport::Concern + + include PartitionedTable + + MINIMUM_PAUSE_MS = 100 + PARTITION_DURATION = 14.days + + REQUIRED_COLUMNS = %i[ + batch_size + sub_batch_size + worker_id + worker_partition + ].freeze + + included do |job_class| + REQUIRED_COLUMNS.each do |column| + validates column, presence: true + end + + validates :pause_ms, numericality: { greater_than_or_equal_to: MINIMUM_PAUSE_MS } + + delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name, + to: :worker, prefix: :worker + + scope :for_partition, ->(partition) { where(partition: partition) } + scope :executable, -> { with_statuses(:pending, :running) } + + # Partition should not be changed once the record is created + attr_readonly :partition + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: ->(active_partition) do + oldest_record_in_partition = job_class + .select(:id, :created_at) + .for_partition(active_partition.value) + .order(:created_at) + .limit(1) + .take + + oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: ->(partition) do + !job_class + .for_partition(partition.value) + .executable + .exists? + end + + state_machine :status, initial: :pending do + state :pending, value: 0 + state :running, value: 1 + state :failed, value: 2 + state :succeeded, value: 3 + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_operation/common_worker.rb b/lib/gitlab/database/background_operation/common_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..b32478ecad4471249c4243f35a6bc079cdd7274e --- /dev/null +++ b/lib/gitlab/database/background_operation/common_worker.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + module CommonWorker + extend ActiveSupport::Concern + + include PartitionedTable + + MINIMUM_PAUSE_MS = 100 + PARTITION_DURATION = 14.days + + REQUIRED_COLUMNS = %i[ + batch_size + sub_batch_size + priority + interval + job_class_name + batch_class_name + table_name + column_name + gitlab_schema + ].freeze + + included do |worker_class| + # Partition should not be changed once the record is created + attr_readonly :partition + + REQUIRED_COLUMNS.each do |column| + validates column, presence: true + end + + validates :pause_ms, numericality: { greater_than_or_equal_to: MINIMUM_PAUSE_MS } + + validates :job_arguments, uniqueness: { + scope: [:job_class_name, :table_name, :column_name] + } + + scope :for_partition, ->(partition) { where(partition: partition) } + scope :executable, -> { with_statuses(:queued, :active, :paused) } + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: ->(active_partition) do + oldest_record_in_partition = worker_class + .select(:id, :created_at) + .for_partition(active_partition.value) + .order(:created_at) + .limit(1) + .take + + oldest_record_in_partition.present? && + oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: ->(partition) do + !worker_class + .for_partition(partition.value) + .executable + .exists? + end + + state_machine :status, initial: :paused do + state :queued, value: 0 + state :active, value: 1 + state :paused, value: 2 + state :finished, value: 3 + state :failed, value: 4 + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_operation/job.rb b/lib/gitlab/database/background_operation/job.rb new file mode 100644 index 0000000000000000000000000000000000000000..eb0ca574e7fb1b57b753b286c1167ba444fdf57a --- /dev/null +++ b/lib/gitlab/database/background_operation/job.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + class Job < SharedModel + include CommonJob + + self.table_name = :background_operation_jobs + + belongs_to :worker, + ->(job) { where(partition: job.worker_partition) }, + class_name: 'Gitlab::Database::BackgroundOperation::Worker', + foreign_key: :worker_id, + partition_foreign_key: :worker_partition, + inverse_of: :jobs + + belongs_to :organization + end + end + end +end diff --git a/lib/gitlab/database/background_operation/job_cell_local.rb b/lib/gitlab/database/background_operation/job_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..6b987839c995afa9fedc72350e2ad95933a38654 --- /dev/null +++ b/lib/gitlab/database/background_operation/job_cell_local.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + class JobCellLocal < SharedModel + include CommonJob + + self.table_name = :background_operation_jobs_cell_local + + belongs_to :worker, + ->(job) { where(partition: job.worker_partition) }, + class_name: 'Gitlab::Database::BackgroundOperation::WorkerCellLocal', + foreign_key: :worker_id, + partition_foreign_key: :worker_partition, + inverse_of: :jobs + end + end + end +end diff --git a/lib/gitlab/database/background_operation/queueable.rb b/lib/gitlab/database/background_operation/queueable.rb new file mode 100644 index 0000000000000000000000000000000000000000..ae57c40b755588c1094afa918fd9eb12d42f5b1e --- /dev/null +++ b/lib/gitlab/database/background_operation/queueable.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + module Queueable + extend ActiveSupport::Concern + + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + BATCH_MIN_VALUE = 1 + BATCH_MIN_DELAY = 2.minutes + BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' + MINIMUM_PAUSE_MS = 100 + MAX_BATCH_SIZE = 50_000 + MIN_BATCH_SIZE = 10 + MAX_SUB_BATCH_SIZE = 10_000 + MIN_SUB_BATCH_SIZE = 1 + + included do + scope :for_configuration do |job_class_name, table_name, column_name, job_arguments| + where( + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments.to_json) + end + end + + class_methods do + def enqueue(job_class_name, table_name, column_name, gitlab_schema, user:, job_arguments: [], **options) + operation_connection = table_connection(table_name) + + Gitlab::Database::SharedModel.using_connection(operation_connection) do + existing = for_configuration(job_class_name, table_name, column_name, job_arguments).exists? + return if existing # rubocop:disable Cop/AvoidReturnFromBlocks -- we need to return + + create_operation( + operation_connection, + job_class_name, + table_name, + column_name, + job_arguments, + gitlab_schema, + user, + **options) + end + end + + def table_connection(table_name) + table_schema = Gitlab::Database::GitlabSchema.table_schema!(table_name) + base_model = Gitlab::Database.schemas_to_base_models[table_schema.to_s].first + + base_model.connection + end + + private + + def create_operation( + connection, + job_class_name, + table_name, + column_name, + job_arguments, + gitlab_schema, # rubocop:disable Lint/UnusedMethodArgument -- TBD + user, + **options + ) + min_cursor = options[:min_cursor] || get_column_value(connection, table_name, column_name, 'MIN') + max_cursor = options[:max_cursor] || get_column_value(connection, table_name, column_name, 'MAX') + + # Default values if table is empty + min_cursor ||= BATCH_MIN_VALUE + max_cursor ||= min_cursor + + operation_attrs = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments.to_json, + interval: [options[:interval]&.to_i || BATCH_MIN_DELAY.to_i, BATCH_MIN_DELAY.to_i].max, + min_cursor: min_cursor, + max_cursor: max_cursor, + batch_size: options[:batch_size] || BATCH_SIZE, + sub_batch_size: options[:sub_batch_size] || SUB_BATCH_SIZE, + pause_ms: options[:pause_ms] || MINIMUM_PAUSE_MS, + batch_class_name: options[:batch_class_name] || BATCH_CLASS_NAME, + status_event: :queued + } + + operation_attrs.merge!(user_id: user.id, organization_id: user.organization_id) if user.present? + + create!(operation_attrs) + end + + def get_column_value(connection, table_name, column_name, function) + connection.select_value(<<~SQL) + SELECT #{function}(#{Gitlab::Database.quote_column_name(column_name)}) + FROM #{Gitlab::Database.quote_table_name(table_name)} + SQL + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_operation/worker.rb b/lib/gitlab/database/background_operation/worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..f95d7cf0c3b58eda7cebde557d9ad296b59baa45 --- /dev/null +++ b/lib/gitlab/database/background_operation/worker.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + class Worker < SharedModel + include CommonWorker + include Queueable + + self.table_name = :background_operation_workers + + has_many :jobs, + ->(worker) { where(worker_partition: worker.partition) }, + class_name: 'Gitlab::Database::BackgroundOperation::Job', + foreign_key: :worker_id, + inverse_of: :worker, + partition_foreign_key: :worker_partition + + belongs_to :organization + belongs_to :user + end + end + end +end diff --git a/lib/gitlab/database/background_operation/worker_cell_local.rb b/lib/gitlab/database/background_operation/worker_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..751620bb0de414162ed0cb1f5407f26b7094c8b4 --- /dev/null +++ b/lib/gitlab/database/background_operation/worker_cell_local.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundOperation + class WorkerCellLocal < SharedModel + include CommonWorker + include Queueable + + self.table_name = :background_operation_workers_cell_local + + has_many :jobs, + ->(worker) { where(worker_partition: worker.partition) }, + class_name: 'Gitlab::Database::BackgroundOperation::JobCellLocal', + foreign_key: :worker_id, + inverse_of: :worker, + partition_foreign_key: :worker_partition + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 796db20fc08e2cf929c408c88a831941b84dc1bc..a61518feb7bfba1c0f4b8b509b19710988054646 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -37,6 +37,8 @@ def self.table_schemas!(tables) '_test_gitlab_embedding_' => :gitlab_embedding, '_test_gitlab_geo_' => :gitlab_geo, '_test_gitlab_pm_' => :gitlab_pm, + '_test_gitlab_shared_org_' => :gitlab_shared_org, + '_test_gitlab_shared_cell_local_' => :gitlab_shared_cell_local, '_test_' => :gitlab_shared, 'pg_' => :gitlab_internal }.freeze diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 238df2d93e4dbbc30b876e565d86e39c40cf86ad..a7c5def1ee2210e9e2019938d0549f06df2e7ad9 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -8,6 +8,8 @@ class SharedModel < ActiveRecord::Base self.abstract_class = true + SHARED_SCHEMAS = %i[gitlab_shared gitlab_shared_org gitlab_shared_cell_local].freeze + # if shared model is used, this allows to limit connections # on which this model is being shared class_attribute :limit_connection_names, default: nil @@ -26,9 +28,9 @@ def using_connection(connection) # in such cases it is fine to ignore such connections gitlab_schemas = Gitlab::Database.gitlab_schemas_for_connection(connection) - unless gitlab_schemas.nil? || gitlab_schemas.include?(:gitlab_shared) + unless gitlab_schemas.nil? || (gitlab_schemas & SHARED_SCHEMAS).present? raise "Cannot set `SharedModel` to connection from `#{Gitlab::Database.db_config_name(connection)}` " \ - "since this connection does not include `:gitlab_shared` schema." + "since this connection does not include any of the shared gitlab_schema." end self.overriding_connection = connection diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 5be5aeb0327695c86cbe1ee67108fc3b118aa03b..80532866427afd9dc68012e31bdc3b97064fd944 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -295,7 +295,9 @@ # temp entry, removing FK on source_type_id and target_type_id until table is dropped in follow up MR work_item_related_link_restrictions: %w[source_type_id target_type_id], sbom_vulnerability_scans: %w[project_id build_id], # referenced records are in different DB and no LFK as the table contains references to object storage - security_trainings: %w[training_provider_id] # training_provider_id is a fixed items model reference. + security_trainings: %w[training_provider_id], # training_provider_id is a fixed items model reference. + background_operation_jobs_cell_local: %w[worker_id], # background operation workers partitions have to dropped independently. + background_operation_jobs: %w[worker_id] # background operation workers partitions have to dropped independently. }.with_indifferent_access.freeze end @@ -553,7 +555,8 @@ class #{model.name} models.each do |model| # Skip migration models - next if model.name.include?('Gitlab::BackgroundMigration') + next if model.name.match?(/Gitlab::Background(?:Migration|Operation)/) + next if ignored_jsonb_columns(model.name).include?(column_name) has_validator = model.validators.any? do |v| diff --git a/spec/factories/gitlab/database/background_operation/jobs.rb b/spec/factories/gitlab/database/background_operation/jobs.rb new file mode 100644 index 0000000000000000000000000000000000000000..1b84be1758436d3573e6a99513b72d6aa60e9c43 --- /dev/null +++ b/spec/factories/gitlab/database/background_operation/jobs.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :background_operation_job, class: 'Gitlab::Database::BackgroundOperation::Job' do + association :worker, factory: :background_operation_worker + + organization_id { create(:common_organization).id } + batch_size { 100 } + sub_batch_size { 10 } + min_cursor { [1] } + max_cursor { [1000] } + worker_partition { 1 } + + trait :pending do + status { 0 } + end + + trait :running do + status { 1 } + started_at { Time.current } + end + + trait :failed do + status { 2 } + attempts { 1 } + end + + trait :succeeded do + status { 3 } + started_at { 1.hour.ago } + finished_at { Time.current } + end + end +end diff --git a/spec/factories/gitlab/database/background_operation/jobs_cell_local.rb b/spec/factories/gitlab/database/background_operation/jobs_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..6dd12203ab2b6ff680426631d333aac39217ae66 --- /dev/null +++ b/spec/factories/gitlab/database/background_operation/jobs_cell_local.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :background_operation_job_cell_local, class: 'Gitlab::Database::BackgroundOperation::JobCellLocal' do + association :worker, factory: :background_operation_worker_cell_local + + batch_size { 100 } + sub_batch_size { 10 } + min_cursor { [1] } + max_cursor { [1000] } + worker_id { 1 } + worker_partition { 1 } + + trait :pending do + status { 0 } + end + + trait :running do + status { 1 } + started_at { Time.current } + end + + trait :failed do + status { 2 } + attempts { 1 } + end + + trait :succeeded do + status { 3 } + started_at { 1.hour.ago } + finished_at { Time.current } + end + end +end diff --git a/spec/factories/gitlab/database/background_operation/workers.rb b/spec/factories/gitlab/database/background_operation/workers.rb new file mode 100644 index 0000000000000000000000000000000000000000..aeb388e7987968e9d88193567923345b114d975f --- /dev/null +++ b/spec/factories/gitlab/database/background_operation/workers.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :background_operation_worker, class: 'Gitlab::Database::BackgroundOperation::Worker' do + organization_id { create(:common_organization).id } + job_class_name { 'CopyColumnUsingBackgroundMigrationJob' } + batch_class_name { 'PrimaryKeyBatchingStrategy' } + table_name { :users } + column_name { :id } + gitlab_schema { :gitlab_main_org } + batch_size { 1000 } + sub_batch_size { 100 } + pause_ms { 100 } + priority { 0 } + interval { 2.minutes } + sequence(:job_arguments) { |n| [["column_#{n}"], ["column_#{n}_convert_to_bigint"]] } + min_cursor { [1] } + max_cursor { [1000] } + user + + trait :queued do + status { 0 } + end + + trait :active do + status { 1 } + end + + trait :paused do + status { 2 } + end + + trait :finished do + status { 3 } + end + + trait :failed do + status { 4 } + end + end +end diff --git a/spec/factories/gitlab/database/background_operation/workers_cell_local.rb b/spec/factories/gitlab/database/background_operation/workers_cell_local.rb new file mode 100644 index 0000000000000000000000000000000000000000..baddc38d8d369efda0d950b8f1f9c414adb0bba5 --- /dev/null +++ b/spec/factories/gitlab/database/background_operation/workers_cell_local.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :background_operation_worker_cell_local, class: 'Gitlab::Database::BackgroundOperation::WorkerCellLocal' do + job_class_name { 'CopyColumnUsingBackgroundMigrationJob' } + batch_class_name { 'PrimaryKeyBatchingStrategy' } + table_name { :users } + column_name { :id } + gitlab_schema { :gitlab_main_org } + batch_size { 1000 } + sub_batch_size { 100 } + pause_ms { 100 } + priority { 0 } + interval { 2.minutes } + sequence(:job_arguments) { |n| [["column_#{n}"], ["column_#{n}_convert_to_bigint"]] } + min_cursor { [1] } + max_cursor { [1000] } + + trait :queued do + status { 0 } + end + + trait :active do + status { 1 } + end + + trait :paused do + status { 2 } + end + + trait :finished do + status { 3 } + end + + trait :failed do + status { 4 } + end + end +end diff --git a/spec/lib/gitlab/database/background_operation/job_cell_local_spec.rb b/spec/lib/gitlab/database/background_operation/job_cell_local_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..20facc031b41c7f0a762f65fddb25263421ac3cb --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/job_cell_local_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative 'job_shared_examples' + +RSpec.describe Gitlab::Database::BackgroundOperation::JobCellLocal, type: :model, feature_category: :database do + it_behaves_like 'background operation job functionality', + :background_operation_job_cell_local, + :background_operation_worker_cell_local +end diff --git a/spec/lib/gitlab/database/background_operation/job_shared_examples.rb b/spec/lib/gitlab/database/background_operation/job_shared_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..11c1ce4e8591d78dee0bf0e5c784fd74850397ba --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/job_shared_examples.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'background operation job functionality' do |job_factory, worker_factory| + using RSpec::Parameterized::TableSyntax + + it { is_expected.to be_a Gitlab::Database::SharedModel } + + describe 'associations' do + let_it_be(:worker) { create!(worker_factory) } + let_it_be(:job) { create!(job_factory, worker: worker, worker_partition: worker.partition) } + + it { is_expected.to belong_to(:worker).inverse_of(:jobs) } + + it 'maintains inverse relationship with the worker' do + expect(worker.jobs).to match_array([job]) + expect(job.worker).to eq(worker) + end + end + + describe 'validations' do + subject { build(job_factory) } + + described_class::REQUIRED_COLUMNS.each do |column| + it { is_expected.to validate_presence_of(column) } + end + + it { is_expected.to validate_numericality_of(:pause_ms).is_greater_than_or_equal_to(100) } + end + + describe 'scopes' do + let_it_be(:job_1) { create(job_factory, :pending) } + let_it_be(:job_2) { create(job_factory, :running) } + let_it_be(:job_3) { create(job_factory, :failed) } + let_it_be(:job_4) { create(job_factory, :succeeded) } + + describe '.executable' do + it 'returns jobs with only with pending or running status' do + expect(described_class.executable).to contain_exactly(job_1, job_2) + end + end + end + + describe 'sliding_list partitioning' do + let(:connection) { described_class.connection } + let(:partition_manager) { Gitlab::Database::Partitioning::PartitionManager.new(described_class) } + + describe 'next_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition } + + subject(:value) { described_class.partitioning_strategy.next_partition_if.call(active_partition) } + + context 'when the partition is empty' do + it { is_expected.to be(false) } + end + + context 'when the partition has recent records' do + before do + create(job_factory, created_at: 1.day.ago) + end + + it { is_expected.to be(false) } + end + + context 'when the first record of the partition is older than PARTITION_DURATION' do + before do + create(job_factory, created_at: (described_class::PARTITION_DURATION + 1.day).ago) + create(job_factory, created_at: 1.day.ago) + end + + it { is_expected.to be(true) } + end + end + + describe 'detach_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition } + + subject(:value) { described_class.partitioning_strategy.detach_partition_if.call(active_partition) } + + context 'when the partition contains executable jobs' do + before do + create(job_factory, :pending) + create(job_factory, :running) + create(job_factory, :succeeded) + end + + it { is_expected.to be(false) } + end + + context 'when the partition contains only non-executable jobs' do + before do + create(job_factory, :succeeded) + create(job_factory, :failed) + end + + it { is_expected.to be(true) } + end + + context 'when the partition is empty' do + it { is_expected.to be(true) } + end + end + + describe 'the behavior of the strategy' do + it 'moves records to new partitions as time passes', :freeze_time do + # We start with partition 1 + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1]) + + # it's not a day old yet so no new partitions are created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1]) + + # add one record so the next partition will be created + create(job_factory) # rubocop:disable Rails/SaveBang -- factory + + # after traveling forward past PARTITION_DURATION + travel(Gitlab::Database::BackgroundOperation::Worker::PARTITION_DURATION + 1.second) + + # a new partition is created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1, 2]) + + # and we can insert to the new partition + expect { create(job_factory) }.not_to raise_error # rubocop:disable Rails/SaveBang -- factory + + # after marking old records as non-executable + described_class.for_partition(1).update_all(status: 3) + + partition_manager.sync_partitions + + # the old one is removed + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([2]) + + # and we only have the newly created partition left. + expect(described_class.count).to eq(1) + end + end + end +end diff --git a/spec/lib/gitlab/database/background_operation/job_spec.rb b/spec/lib/gitlab/database/background_operation/job_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5ff6e9dd685492d944abd3971142aa51039a1813 --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/job_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative 'job_shared_examples' + +RSpec.describe Gitlab::Database::BackgroundOperation::Job, type: :model, feature_category: :database do + it_behaves_like 'background operation job functionality', :background_operation_job, :background_operation_worker +end diff --git a/spec/lib/gitlab/database/background_operation/worker_cell_local_spec.rb b/spec/lib/gitlab/database/background_operation/worker_cell_local_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1e4499919e4ac6ff25ab1dc3120d3088ded5890b --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/worker_cell_local_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative 'worker_shared_examples' + +RSpec.describe Gitlab::Database::BackgroundOperation::WorkerCellLocal, type: :model, feature_category: :database do + it_behaves_like 'background operation worker functionality', :background_operation_worker_cell_local +end diff --git a/spec/lib/gitlab/database/background_operation/worker_shared_examples.rb b/spec/lib/gitlab/database/background_operation/worker_shared_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..58445f624513dedb853e935fd6e8c4e8b9641d33 --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/worker_shared_examples.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'background operation worker functionality' do |worker_factory| + using RSpec::Parameterized::TableSyntax + + it { is_expected.to be_a Gitlab::Database::SharedModel } + + describe 'associations' do + it { is_expected.to have_many(:jobs).inverse_of(:worker) } + end + + describe 'validations' do + subject { build(worker_factory) } + + described_class::REQUIRED_COLUMNS.each do |column| + it { is_expected.to validate_presence_of(column) } + end + + it { is_expected.to validate_numericality_of(:pause_ms).is_greater_than_or_equal_to(100) } + it { is_expected.to validate_uniqueness_of(:job_arguments).scoped_to(:job_class_name, :table_name, :column_name) } + end + + describe 'scopes' do + let_it_be(:queued_worker) { create(worker_factory, :queued) } + let_it_be(:active_worker) { create(worker_factory, :active) } + let_it_be(:paused_worker) { create(worker_factory, :paused) } + let_it_be(:finished_worker) { create(worker_factory, :finished) } + + let(:executable_workers) { [queued_worker, active_worker, paused_worker] } + + describe '.executable' do + it 'returns workers with queued, active or paused status' do + expect(described_class.executable).to match_array(executable_workers) + end + end + end + + describe 'sliding_list partitioning' do + let(:connection) { described_class.connection } + let(:partition_manager) { Gitlab::Database::Partitioning::PartitionManager.new(described_class) } + + describe 'next_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition } + + subject(:value) { described_class.partitioning_strategy.next_partition_if.call(active_partition) } + + context 'when the partition is empty' do + it { is_expected.to be(false) } + end + + context 'when the partition has recent records' do + before do + create(worker_factory, created_at: 1.day.ago) + end + + it { is_expected.to be(false) } + end + + context 'when the first record of the partition is older than PARTITION_DURATION' do + before do + create(worker_factory, created_at: (described_class::PARTITION_DURATION + 1.day).ago) + create(worker_factory, created_at: 1.day.ago) + end + + it { is_expected.to be(true) } + end + end + + describe 'detach_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition } + + subject(:value) { described_class.partitioning_strategy.detach_partition_if.call(active_partition) } + + context 'when the partition contains executable workers' do + before do + create(worker_factory, :active) + create(worker_factory, :paused) + create(worker_factory, :finished) + end + + it { is_expected.to be(false) } + end + + context 'when the partition contains only non-executable workers' do + before do + create(worker_factory, :finished) + create(worker_factory, :failed) + end + + it { is_expected.to be(true) } + end + + context 'when the partition is empty' do + it { is_expected.to be(true) } + end + end + + describe 'the behavior of the strategy' do + it 'moves records to new partitions as time passes', :freeze_time do + # We start with partition 1 + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1]) + + # it's not 14 days old yet so no new partitions are created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1]) + + # add one record so the next partition will be created + create(worker_factory) # rubocop:disable Rails/SaveBang -- factory + + # after traveling forward past PARTITION_DURATION + travel(described_class::PARTITION_DURATION + 1.minute) + + # a new partition is created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([1, 2]) + + # and we can insert to the new partition + expect { create(worker_factory) }.not_to raise_error # rubocop:disable Rails/SaveBang -- factory + + # after marking old records as non-executable + described_class.for_partition(1).update_all(status: 3) + + partition_manager.sync_partitions + + # the old one is removed + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to match_array([2]) + + # and we only have the newly created partition left. + expect(described_class.count).to eq(1) + end + end + end +end diff --git a/spec/lib/gitlab/database/background_operation/worker_spec.rb b/spec/lib/gitlab/database/background_operation/worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f44d0be6a28530ec41153e3ac5d84b25afdc960d --- /dev/null +++ b/spec/lib/gitlab/database/background_operation/worker_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative 'worker_shared_examples' + +RSpec.describe Gitlab::Database::BackgroundOperation::Worker, type: :model, feature_category: :database do + it_behaves_like 'background operation worker functionality', :background_operation_worker +end diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index ae5859066b793933e5d8d3714a853fffe9fe0cc7..f1634e943b42ebff7e6113915cdb5fd320f3035b 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -17,7 +17,8 @@ .values database_connections.permutation(2) do |db, other_db| - gitlab_schemas = db.gitlab_schemas - [:gitlab_shared, :gitlab_internal] + gitlab_schemas = db.gitlab_schemas - + [:gitlab_shared, :gitlab_internal, :gitlab_shared_org, :gitlab_shared_cell_local] expect(other_db.lock_gitlab_schemas).to include(*gitlab_schemas), "Expected `#{other_db.name}` lock_gitlab_schemas to include `#{db.name}` gitlab_schemas:" @@ -42,6 +43,8 @@ '_test_gitlab_main_org_table' | :gitlab_main_org '_test_gitlab_pm_table' | :gitlab_pm '_test_gitlab_sec_table' | :gitlab_sec + '_test_gitlab_shared_org_table' | :gitlab_shared_org + '_test_gitlab_shared_cell_local_table' | :gitlab_shared_cell_local '_test_my_table' | :gitlab_shared 'pg_attribute' | :gitlab_internal end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 2828063eb8e0e5d27b4cc4633e675856fb9c52d8..01e26572b2b0ebc4331e50adf0de3daf31c0e8bd 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -325,6 +325,7 @@ "jira_connect_installations" => "https://gitlab.com/gitlab-org/gitlab/-/issues/524682", "system_note_metadata" => "https://gitlab.com/gitlab-org/gitlab/-/issues/571215" } + has_lfk = ->(lfks) { lfks.any? { |k| k.options[:column] == 'organization_id' && k.to_table == 'organizations' } } columns_to_check = organization_id_columns.reject { |column| work_in_progress[column[0]] } diff --git a/spec/lib/gitlab/database/shared_model_spec.rb b/spec/lib/gitlab/database/shared_model_spec.rb index 2ae6ccf6c6ab7951357a5f10ffb9d6568dee89cf..af509a806407a07753d9d8dc59baf1a758859fa0 100644 --- a/spec/lib/gitlab/database/shared_model_spec.rb +++ b/spec/lib/gitlab/database/shared_model_spec.rb @@ -3,10 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::SharedModel, feature_category: :database do - describe 'using an external connection' do - let!(:original_connection) { described_class.connection } - let(:new_connection) { double('connection') } - + shared_examples 'shared model using the correct connection' do it 'overrides the connection for the duration of the block', :aggregate_failures do expect_original_connection_around do described_class.using_connection(new_connection) do @@ -14,6 +11,13 @@ end end end + end + + describe 'using an external connection' do + let!(:original_connection) { described_class.connection } + let(:new_connection) { double('connection') } + + it_behaves_like 'shared model using the correct connection' it 'does not affect connections in other threads', :aggregate_failures do expect_original_connection_around do @@ -27,7 +31,7 @@ end end - it 'raises an error if the connection does not include `:gitlab_shared` schema' do + it 'raises an error if the connection does not include shared gitlab_schema' do allow(Gitlab::Database) .to receive(:gitlab_schemas_for_connection) .with(new_connection) @@ -40,6 +44,26 @@ end end + context 'with connection including gitlab_shared_org' do + before do + allow(Gitlab::Database).to receive(:gitlab_schemas_for_connection) + .with(new_connection) + .and_return([:gitlab_shared_org]) + end + + it_behaves_like 'shared model using the correct connection' + end + + context 'with connection including gitlab_shared_cell_local' do + before do + allow(Gitlab::Database).to receive(:gitlab_schemas_for_connection) + .with(new_connection) + .and_return([:gitlab_shared_cell_local]) + end + + it_behaves_like 'shared model using the correct connection' + end + context 'when multiple connection overrides are nested', :aggregate_failures do let(:second_connection) { double('connection') }