diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 33092e881f045a877b048bbf914e5da23882b070..b4f10758c1de290067a2e30a84594f784832ba0f 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -13,6 +13,7 @@ class BuildMetadata < Ci::ApplicationRecord self.table_name = 'ci_builds_metadata' self.primary_key = 'id' + self.sequence_name = 'ci_builds_metadata_id_seq' partitionable scope: :build belongs_to :build, class_name: 'CommitStatus' diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 1530c681eb060d7901f122eed5435c98c78698f1..9e1bf15dee19fe668053dce25e4fc4d2e87d3691 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -210,6 +210,10 @@ merge_trains: - table: ci_pipelines column: pipeline_id on_delete: async_nullify +p_ci_builds_metadata: + - table: projects + column: project_id + on_delete: async_delete packages_build_infos: - table: ci_pipelines column: pipeline_id diff --git a/db/docs/p_ci_builds_metadata.yml b/db/docs/p_ci_builds_metadata.yml new file mode 100644 index 0000000000000000000000000000000000000000..676cb3bfb1c876047b2e1440c2d52993a6290e5b --- /dev/null +++ b/db/docs/p_ci_builds_metadata.yml @@ -0,0 +1,9 @@ +--- +table_name: p_ci_builds_metadata +classes: +- Ci::BuildMetadata +feature_categories: +- continuous_integration +description: Routing table that holds information for job execution +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100115 +milestone: '15.5' diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb index 8e9952e9ba1186c78bf4968128f0759462ae4b46..5c71e1ecdace814bbaff4bedcfcc039ba9c259ca 100644 --- a/db/fixtures/development/17_cycle_analytics.rb +++ b/db/fixtures/development/17_cycle_analytics.rb @@ -104,7 +104,7 @@ def seed_code_stage! def seed_test_stage! merge_requests.each do |merge_request| - pipeline = FactoryBot.create(:ci_pipeline, :success, project: project) + pipeline = FactoryBot.create(:ci_pipeline, :success, project: project, partition_id: Ci::Pipeline.current_partition_value) build = FactoryBot.create(:ci_build, pipeline: pipeline, project: project, user: developers.sample) # Required because seeds run in a transaction and these are now diff --git a/db/post_migrate/20221004074910_routing_table_prepare_constraint_for_builds_metadata.rb b/db/post_migrate/20221004074910_routing_table_prepare_constraint_for_builds_metadata.rb new file mode 100644 index 0000000000000000000000000000000000000000..013984154ae1df7cec8d17e0400cdbe24359d8b8 --- /dev/null +++ b/db/post_migrate/20221004074910_routing_table_prepare_constraint_for_builds_metadata.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class RoutingTablePrepareConstraintForBuildsMetadata < Gitlab::Database::Migration[2.0] + include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + PARENT_TABLE_NAME = :p_ci_builds_metadata + FIRST_PARTITION = 100 + PARTITION_COLUMN = :partition_id + + def up + prepare_constraint_for_list_partitioning( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION + ) + end + + def down + revert_preparing_constraint_for_list_partitioning( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION + ) + end +end diff --git a/db/post_migrate/20221004074914_create_routing_table_for_builds_metadata.rb b/db/post_migrate/20221004074914_create_routing_table_for_builds_metadata.rb new file mode 100644 index 0000000000000000000000000000000000000000..71a8625dce24c8a1a06e2a3e37cd9c0e510a3eba --- /dev/null +++ b/db/post_migrate/20221004074914_create_routing_table_for_builds_metadata.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class CreateRoutingTableForBuildsMetadata < Gitlab::Database::Migration[2.0] + include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + PARENT_TABLE_NAME = :p_ci_builds_metadata + FIRST_PARTITION = 100 + PARTITION_COLUMN = :partition_id + + def up + convert_table_to_first_list_partition( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION + ) + end + + def down + revert_converting_table_to_first_list_partition( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION + ) + end +end diff --git a/db/schema_migrations/20221004074910 b/db/schema_migrations/20221004074910 new file mode 100644 index 0000000000000000000000000000000000000000..e316408ef0f22f860dae0371d3605d856f967e58 --- /dev/null +++ b/db/schema_migrations/20221004074910 @@ -0,0 +1 @@ +df8b2f83f93fecd3450cb7fc2619e7ddbfde014a885d0a414076908bbcbbdf9f \ No newline at end of file diff --git a/db/schema_migrations/20221004074914 b/db/schema_migrations/20221004074914 new file mode 100644 index 0000000000000000000000000000000000000000..a197fe1b9ea4083ba91cf4ba0400e640fa040425 --- /dev/null +++ b/db/schema_migrations/20221004074914 @@ -0,0 +1 @@ +d6459263b828e6807f473adb7ba534d78055ab1b5137478a8e96cd500297ff54 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 2dab5e7abc90bd4a786648c5035f814069d3598c..ae5498273b3718546dfaa8575bbe06aa93c3b0c0 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12646,7 +12646,7 @@ CREATE SEQUENCE ci_builds_id_seq ALTER SEQUENCE ci_builds_id_seq OWNED BY ci_builds.id; -CREATE TABLE ci_builds_metadata ( +CREATE TABLE p_ci_builds_metadata ( project_id integer NOT NULL, timeout integer, timeout_source integer DEFAULT 1 NOT NULL, @@ -12662,7 +12662,8 @@ CREATE TABLE ci_builds_metadata ( runtime_runner_features jsonb DEFAULT '{}'::jsonb NOT NULL, id_tokens jsonb DEFAULT '{}'::jsonb NOT NULL, partition_id bigint DEFAULT 100 NOT NULL -); +) +PARTITION BY LIST (partition_id); CREATE SEQUENCE ci_builds_metadata_id_seq START WITH 1 @@ -12671,7 +12672,26 @@ CREATE SEQUENCE ci_builds_metadata_id_seq NO MAXVALUE CACHE 1; -ALTER SEQUENCE ci_builds_metadata_id_seq OWNED BY ci_builds_metadata.id; +ALTER SEQUENCE ci_builds_metadata_id_seq OWNED BY p_ci_builds_metadata.id; + +CREATE TABLE ci_builds_metadata ( + project_id integer NOT NULL, + timeout integer, + timeout_source integer DEFAULT 1 NOT NULL, + interruptible boolean, + config_options jsonb, + config_variables jsonb, + has_exposed_artifacts boolean, + environment_auto_stop_in character varying(255), + expanded_environment_name character varying(255), + secrets jsonb DEFAULT '{}'::jsonb NOT NULL, + build_id bigint NOT NULL, + id bigint DEFAULT nextval('ci_builds_metadata_id_seq'::regclass) NOT NULL, + runtime_runner_features jsonb DEFAULT '{}'::jsonb NOT NULL, + id_tokens jsonb DEFAULT '{}'::jsonb NOT NULL, + partition_id bigint DEFAULT 100 NOT NULL +); +ALTER TABLE ONLY p_ci_builds_metadata ATTACH PARTITION ci_builds_metadata FOR VALUES IN ('100'); CREATE TABLE ci_builds_runner_session ( id bigint NOT NULL, @@ -23441,8 +23461,6 @@ ALTER TABLE ONLY ci_build_trace_chunks ALTER COLUMN id SET DEFAULT nextval('ci_b ALTER TABLE ONLY ci_builds ALTER COLUMN id SET DEFAULT nextval('ci_builds_id_seq'::regclass); -ALTER TABLE ONLY ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_builds_metadata_id_seq'::regclass); - ALTER TABLE ONLY ci_builds_runner_session ALTER COLUMN id SET DEFAULT nextval('ci_builds_runner_session_id_seq'::regclass); ALTER TABLE ONLY ci_daily_build_group_report_results ALTER COLUMN id SET DEFAULT nextval('ci_daily_build_group_report_results_id_seq'::regclass); @@ -23947,6 +23965,8 @@ ALTER TABLE ONLY operations_strategies_user_lists ALTER COLUMN id SET DEFAULT ne ALTER TABLE ONLY operations_user_lists ALTER COLUMN id SET DEFAULT nextval('operations_user_lists_id_seq'::regclass); +ALTER TABLE ONLY p_ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_builds_metadata_id_seq'::regclass); + ALTER TABLE ONLY packages_build_infos ALTER COLUMN id SET DEFAULT nextval('packages_build_infos_id_seq'::regclass); ALTER TABLE ONLY packages_composer_cache_files ALTER COLUMN id SET DEFAULT nextval('packages_composer_cache_files_id_seq'::regclass); @@ -25169,6 +25189,9 @@ ALTER TABLE ONLY ci_build_trace_chunks ALTER TABLE ONLY ci_build_trace_metadata ADD CONSTRAINT ci_build_trace_metadata_pkey PRIMARY KEY (build_id); +ALTER TABLE ONLY p_ci_builds_metadata + ADD CONSTRAINT p_ci_builds_metadata_pkey PRIMARY KEY (id, partition_id); + ALTER TABLE ONLY ci_builds_metadata ADD CONSTRAINT ci_builds_metadata_pkey PRIMARY KEY (id, partition_id); @@ -28181,14 +28204,24 @@ CREATE UNIQUE INDEX index_ci_build_trace_chunks_on_build_id_and_chunk_index ON c CREATE INDEX index_ci_build_trace_metadata_on_trace_artifact_id ON ci_build_trace_metadata USING btree (trace_artifact_id); +CREATE INDEX p_ci_builds_metadata_build_id_idx ON ONLY p_ci_builds_metadata USING btree (build_id) WHERE (has_exposed_artifacts IS TRUE); + CREATE INDEX index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts ON ci_builds_metadata USING btree (build_id) WHERE (has_exposed_artifacts IS TRUE); +CREATE INDEX p_ci_builds_metadata_build_id_id_idx ON ONLY p_ci_builds_metadata USING btree (build_id) INCLUDE (id) WHERE (interruptible = true); + CREATE INDEX index_ci_builds_metadata_on_build_id_and_id_and_interruptible ON ci_builds_metadata USING btree (build_id) INCLUDE (id) WHERE (interruptible = true); +CREATE UNIQUE INDEX p_ci_builds_metadata_build_id_partition_id_idx ON ONLY p_ci_builds_metadata USING btree (build_id, partition_id); + CREATE UNIQUE INDEX index_ci_builds_metadata_on_build_id_partition_id_unique ON ci_builds_metadata USING btree (build_id, partition_id); +CREATE UNIQUE INDEX p_ci_builds_metadata_id_partition_id_idx ON ONLY p_ci_builds_metadata USING btree (id, partition_id); + CREATE UNIQUE INDEX index_ci_builds_metadata_on_id_partition_id_unique ON ci_builds_metadata USING btree (id, partition_id); +CREATE INDEX p_ci_builds_metadata_project_id_idx ON ONLY p_ci_builds_metadata USING btree (project_id); + CREATE INDEX index_ci_builds_metadata_on_project_id ON ci_builds_metadata USING btree (project_id); CREATE INDEX index_ci_builds_on_auto_canceled_by_id ON ci_builds USING btree (auto_canceled_by_id); @@ -32409,6 +32442,18 @@ ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_p ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_partitions_static.product_analytics_events_experimental_63_pkey; +ALTER INDEX p_ci_builds_metadata_pkey ATTACH PARTITION ci_builds_metadata_pkey; + +ALTER INDEX p_ci_builds_metadata_build_id_idx ATTACH PARTITION index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts; + +ALTER INDEX p_ci_builds_metadata_build_id_id_idx ATTACH PARTITION index_ci_builds_metadata_on_build_id_and_id_and_interruptible; + +ALTER INDEX p_ci_builds_metadata_build_id_partition_id_idx ATTACH PARTITION index_ci_builds_metadata_on_build_id_partition_id_unique; + +ALTER INDEX p_ci_builds_metadata_id_partition_id_idx ATTACH PARTITION index_ci_builds_metadata_on_id_partition_id_unique; + +ALTER INDEX p_ci_builds_metadata_project_id_idx ATTACH PARTITION index_ci_builds_metadata_on_project_id; + CREATE TRIGGER chat_names_loose_fk_trigger AFTER DELETE ON chat_names REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); CREATE TRIGGER ci_builds_loose_fk_trigger AFTER DELETE ON ci_builds REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); @@ -33296,7 +33341,7 @@ ALTER TABLE ONLY ci_resources ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT fk_e1bad85861 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; -ALTER TABLE ONLY ci_builds_metadata +ALTER TABLE p_ci_builds_metadata ADD CONSTRAINT fk_e20479742e FOREIGN KEY (build_id) REFERENCES ci_builds(id) ON DELETE CASCADE; ALTER TABLE ONLY gitlab_subscriptions diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index c4a9cf8b80ff3b5f000ff4fbd3d9d7290022eab7..db563f109c46f18705a3908c3798728578eda58e 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -363,6 +363,7 @@ operations_scopes: :gitlab_main operations_strategies: :gitlab_main operations_strategies_user_lists: :gitlab_main operations_user_lists: :gitlab_main +p_ci_builds_metadata: :gitlab_ci packages_build_infos: :gitlab_main packages_cleanup_policies: :gitlab_main packages_composer_cache_files: :gitlab_main diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 4aeafed5712649056d065be0e9194df23b694e59..604d654e4e74b53d4dbbf0ad0b7e852c802971b1 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -33,6 +33,7 @@ chat_teams: %w[team_id], ci_builds: %w[erased_by_id trigger_request_id partition_id], ci_builds_metadata: %w[partition_id], + p_ci_builds_metadata: %w[partition_id], ci_job_artifacts: %w[partition_id], ci_namespace_monthly_usages: %w[namespace_id], ci_pipeline_variables: %w[partition_id], diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb index 304d77e85210360e7419a7c461b75471c120d38d..7569e832c6092bb2a551ef2ef932a4c5a08bbb5e 100644 --- a/spec/factories/ci/job_artifacts.rb +++ b/spec/factories/ci/job_artifacts.rb @@ -20,6 +20,8 @@ after :build do |artifact| artifact.project ||= artifact.job.project + + artifact.job&.valid? end trait :raw do diff --git a/spec/factories/ci/pipelines.rb b/spec/factories/ci/pipelines.rb index 650b8647237fb8e2b94f90eec9f2aae91fe941f1..dd93459f9108cf420bcf6705f7053a551261f590 100644 --- a/spec/factories/ci/pipelines.rb +++ b/spec/factories/ci/pipelines.rb @@ -8,7 +8,7 @@ sha { 'b83d6e391c22777fca1ed3012fce84f633d7fed0' } status { 'pending' } add_attribute(:protected) { false } - partition_id { 1234 } + partition_id { 100 } project @@ -54,7 +54,7 @@ end factory :ci_pipeline do - partition_id { 1234 } + partition_id { 100 } transient { ci_ref_presence { true } } before(:create) do |pipeline, evaluator| diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb index 41266cb24dac1ac19b47616e389139696b52f94a..10597e659102e0478f7341a19c09cdcc7a02bdb5 100644 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -85,8 +85,9 @@ private def create_job_artifact(id:, file_type:, expire_at:) - job = table(:ci_builds, database: :ci).create!(id: id) - job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + job = table(:ci_builds, database: :ci).create!(id: id, partition_id: 100) + job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, + file_type: file_type, partition_id: 100) end end end diff --git a/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb index 98939e15952f4b36c7a984f821d6b356656af6ff..fad10aba882d766ebff295bcc06483c7b9a2787e 100644 --- a/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb +++ b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb @@ -26,8 +26,8 @@ let(:locked) { 1 } let(:unknown) { 2 } - let(:unlocked_pipeline) { pipelines.create!(locked: unlocked) } - let(:locked_pipeline) { pipelines.create!(locked: locked) } + let(:unlocked_pipeline) { pipelines.create!(locked: unlocked, partition_id: 100) } + let(:locked_pipeline) { pipelines.create!(locked: locked, partition_id: 100) } # rubocop:disable Layout/LineLength let!(:locked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: locked_pipeline.id, size: 1024, file_type: 0, file_format: 'gzip', file: 'a.gz', locked: unknown) } diff --git a/spec/services/ci/create_pipeline_service/partitioning_spec.rb b/spec/services/ci/create_pipeline_service/partitioning_spec.rb index 43fbb74ede498116d6eaa12b7062536347423ff5..f34d103d965c89451991d1ee86111b6633a9d4d7 100644 --- a/spec/services/ci/create_pipeline_service/partitioning_spec.rb +++ b/spec/services/ci/create_pipeline_service/partitioning_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Ci::CreatePipelineService, :yaml_processor_feature_flag_corectness, :aggregate_failures do +RSpec.describe Ci::CreatePipelineService, :yaml_processor_feature_flag_corectness, :aggregate_failures, +:ci_partitionable do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { project.first_owner } @@ -31,7 +32,7 @@ end let(:pipeline) { service.execute(:push).payload } - let(:current_partition_id) { 123 } + let(:current_partition_id) { ci_testing_partition_id } before do stub_ci_pipeline_yaml_file(config) diff --git a/spec/services/ci/job_artifacts/create_service_spec.rb b/spec/services/ci/job_artifacts/create_service_spec.rb index 030ba84951ebc54632bf791fecd9d34e3f39fcb4..5df590a1b7870d28693acf1e4ab6d511488e47d2 100644 --- a/spec/services/ci/job_artifacts/create_service_spec.rb +++ b/spec/services/ci/job_artifacts/create_service_spec.rb @@ -181,8 +181,8 @@ def file_to_upload(path, params = {}) end end - context 'with job partitioning' do - let(:pipeline) { create(:ci_pipeline, project: project, partition_id: 123) } + context 'with job partitioning', :ci_partitionable do + let(:pipeline) { create(:ci_pipeline, project: project, partition_id: ci_testing_partition_id) } let(:job) { create(:ci_build, pipeline: pipeline) } it 'sets partition_id on artifacts' do @@ -190,7 +190,7 @@ def file_to_upload(path, params = {}) artifacts_partitions = job.job_artifacts.map(&:partition_id).uniq - expect(artifacts_partitions).to eq([123]) + expect(artifacts_partitions).to eq([ci_testing_partition_id]) end end diff --git a/spec/support/models/partitionable_check.rb b/spec/support/models/ci/partitioning_testing/cascade_check.rb similarity index 55% rename from spec/support/models/partitionable_check.rb rename to spec/support/models/ci/partitioning_testing/cascade_check.rb index 2c09c1b34086b001fc31199ccd570f94f9fcad95..f553a47ef4f3c284753e334d86ad40ed15aaea1c 100644 --- a/spec/support/models/partitionable_check.rb +++ b/spec/support/models/ci/partitioning_testing/cascade_check.rb @@ -10,37 +10,18 @@ module CascadeCheck def check_partition_cascade_value raise 'Partition value not found' unless partition_scope_value - raise 'Default value detected' if partition_id == 100 return if partition_id == partition_scope_value raise "partition_id was expected to equal #{partition_scope_value} but it was #{partition_id}." end end - - module DefaultPartitionValue - extend ActiveSupport::Concern - - class_methods do - def current_partition_value - current = super - - if current == 100 - 54321 - else - current - end - end - end - end end Ci::Partitionable::Testing::PARTITIONABLE_MODELS.each do |klass| + next if klass == 'Ci::Pipeline' + model = klass.safe_constantize - if klass == 'Ci::Pipeline' - model.prepend(PartitioningTesting::DefaultPartitionValue) - else - model.include(PartitioningTesting::CascadeCheck) - end + model.include(PartitioningTesting::CascadeCheck) end diff --git a/spec/support/models/ci/partitioning_testing/partition_identifiers.rb b/spec/support/models/ci/partitioning_testing/partition_identifiers.rb new file mode 100644 index 0000000000000000000000000000000000000000..aa091095fb6ab61b4d252db2f8f3f1626cefe3f3 --- /dev/null +++ b/spec/support/models/ci/partitioning_testing/partition_identifiers.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + module PartitioningTesting + module PartitionIdentifiers + module_function + + def ci_testing_partition_id + 99999 + end + end + end +end diff --git a/spec/support/models/ci/partitioning_testing/rspec_hooks.rb b/spec/support/models/ci/partitioning_testing/rspec_hooks.rb new file mode 100644 index 0000000000000000000000000000000000000000..39b15ba87216dbc63e7a4d81cf31821681ff035d --- /dev/null +++ b/spec/support/models/ci/partitioning_testing/rspec_hooks.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.include Ci::PartitioningTesting::PartitionIdentifiers + + config.around(:each, :ci_partitionable) do |example| + Ci::PartitioningTesting::SchemaHelpers.with_routing_tables do + example.run + end + end + + config.before(:all) do + Ci::PartitioningTesting::SchemaHelpers.setup + end + + config.after(:all) do + Ci::PartitioningTesting::SchemaHelpers.teardown + end +end diff --git a/spec/support/models/ci/partitioning_testing/schema_helpers.rb b/spec/support/models/ci/partitioning_testing/schema_helpers.rb new file mode 100644 index 0000000000000000000000000000000000000000..712178710da0e18ee7048362d1e290dc80120d95 --- /dev/null +++ b/spec/support/models/ci/partitioning_testing/schema_helpers.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Ci + module PartitioningTesting + module SchemaHelpers + DEFAULT_PARTITION = 100 + + module_function + + def with_routing_tables + Ci::BuildMetadata.table_name = :p_ci_builds_metadata + yield + ensure + Ci::BuildMetadata.table_name = :ci_builds_metadata + end + + # We're dropping the default values here to ensure that the application code + # populates the `partition_id` value and it's not falling back on the + # database default one. We should be able to clean this up after + # partitioning the tables and substituting the routing table in the model: + # https://gitlab.com/gitlab-org/gitlab/-/issues/377822 + # + def setup(connection: Ci::ApplicationRecord.connection) + each_partitionable_table do |table_name| + change_column_default(table_name, from: DEFAULT_PARTITION, to: nil, connection: connection) + change_column_default("p_#{table_name}", from: DEFAULT_PARTITION, to: nil, connection: connection) + create_test_partition(table_name, connection: connection) + end + end + + def teardown(connection: Ci::ApplicationRecord.connection) + each_partitionable_table do |table_name| + drop_test_partition(table_name, connection: connection) + change_column_default(table_name, from: nil, to: DEFAULT_PARTITION, connection: connection) + change_column_default("p_#{table_name}", from: nil, to: DEFAULT_PARTITION, connection: connection) + end + end + + def each_partitionable_table + ::Ci::Partitionable::Testing::PARTITIONABLE_MODELS.each do |klass| + model = klass.safe_constantize + table_name = model.table_name.delete_prefix('p_') + + yield(table_name) + + model.reset_column_information if model.connected? + end + end + + def change_column_default(table_name, from:, to:, connection:) + return unless table_available?(table_name, connection: connection) + + connection.change_column_default(table_name, :partition_id, from: from, to: to) + end + + def create_test_partition(table_name, connection:) + return unless table_available?("p_#{table_name}", connection: connection) + + drop_test_partition(table_name, connection: connection) + + connection.execute(<<~SQL) + CREATE TABLE #{full_partition_name(table_name)} + PARTITION OF p_#{table_name} + FOR VALUES IN (#{PartitioningTesting::PartitionIdentifiers.ci_testing_partition_id}); + SQL + end + + def drop_test_partition(table_name, connection:) + return unless table_available?(table_name, connection: connection) + + connection.execute(<<~SQL) + DROP TABLE IF EXISTS #{full_partition_name(table_name)}; + SQL + end + + def table_available?(table_name, connection:) + connection.table_exists?(table_name) && + connection.column_exists?(table_name, :partition_id) + end + + def full_partition_name(table_name) + "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_#{table_name}_partition" + end + end + end +end