From 9a97a1a047d0b44b7c8c806e0ae2160dc795d5a0 Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Wed, 25 Jan 2023 11:00:35 +0100 Subject: [PATCH 1/2] Rebalance partition_id for ci_builds & ci_pipelines This commits updates all invalid records in production and sets the partition_id to 100. Changelog: added --- ...3723_rebalance_partition_id_ci_pipeline.rb | 30 ++++++++++ ...5093840_rebalance_partition_id_ci_build.rb | 30 ++++++++++ db/schema_migrations/20230125093723 | 1 + db/schema_migrations/20230125093840 | 1 + .../rebalance_partition_id.rb | 21 +++++++ .../rebalance_partition_id_spec.rb | 49 ++++++++++++++++ ...rebalance_partition_id_ci_pipeline_spec.rb | 58 +++++++++++++++++++ ...40_rebalance_partition_id_ci_build_spec.rb | 58 +++++++++++++++++++ 8 files changed, 248 insertions(+) create mode 100644 db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb create mode 100644 db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb create mode 100644 db/schema_migrations/20230125093723 create mode 100644 db/schema_migrations/20230125093840 create mode 100644 lib/gitlab/background_migration/rebalance_partition_id.rb create mode 100644 spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb create mode 100644 spec/migrations/20230125093723_rebalance_partition_id_ci_pipeline_spec.rb create mode 100644 spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb diff --git a/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb b/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb new file mode 100644 index 00000000000000..3f31e765b4626d --- /dev/null +++ b/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class RebalancePartitionIdCiPipeline < Gitlab::Database::Migration[2.1] + MIGRATION = 'RebalancePartitionId' + DELAY_INTERVAL = 2.minutes + TABLE = :ci_pipelines + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + return unless Gitlab.com? + + queue_batched_background_migration( + MIGRATION, + TABLE, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + return unless Gitlab.com? + + delete_batched_background_migration(MIGRATION, TABLE, :id, []) + end +end diff --git a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb new file mode 100644 index 00000000000000..7c096021178d56 --- /dev/null +++ b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class RebalancePartitionIdCiBuild < Gitlab::Database::Migration[2.1] + MIGRATION = 'RebalancePartitionId' + DELAY_INTERVAL = 2.minutes.freeze + TABLE = :ci_builds + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + return unless Gitlab.com? + + queue_batched_background_migration( + MIGRATION, + TABLE, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + return unless Gitlab.com? + + delete_batched_background_migration(MIGRATION, TABLE, :id, []) + end +end diff --git a/db/schema_migrations/20230125093723 b/db/schema_migrations/20230125093723 new file mode 100644 index 00000000000000..9c5782dcdacdcd --- /dev/null +++ b/db/schema_migrations/20230125093723 @@ -0,0 +1 @@ +364f785b564d92d2956c5ffea71091561231888ffa6f4cd9125fc8ebf9150f77 \ No newline at end of file diff --git a/db/schema_migrations/20230125093840 b/db/schema_migrations/20230125093840 new file mode 100644 index 00000000000000..1d2fab2561942d --- /dev/null +++ b/db/schema_migrations/20230125093840 @@ -0,0 +1 @@ +c66f77a9de07e2f88b6d371b14f7f72068a5b8e25cb382cb08e578021affbeb7 \ No newline at end of file diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb new file mode 100644 index 00000000000000..7000ae5a8566aa --- /dev/null +++ b/lib/gitlab/background_migration/rebalance_partition_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This rebalances partition_id to fix invalid records in production + class RebalancePartitionId < BatchedMigrationJob + INVALID_PARTITION_ID = 101 + VALID_PARTITION_ID = 100 + + scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) } + operation_name :update_all + feature_category :continuous_integration + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(partition_id: VALID_PARTITION_ID) + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb b/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb new file mode 100644 index 00000000000000..2c9483835a52bd --- /dev/null +++ b/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RebalancePartitionId, + :migration, + schema: 20230125093723, + feature_category: :continuous_integration do + let(:ci_builds_table) { table(:ci_builds, database: :ci) } + let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) } + + let!(:valid_ci_pipeline) { ci_pipelines_table.create!(id: 1, partition_id: 100) } + let!(:valid_ci_build) { ci_builds_table.create!(id: 1, partition_id: 100) } + let!(:invalid_ci_build) { ci_builds_table.create!(id: 2, partition_id: 101) } + let!(:invalid_ci_pipeline) { ci_pipelines_table.create!(id: 2, partition_id: 101) } + + describe '#perform' do + using RSpec::Parameterized::TableSyntax + + where(:table_name, :invalid_record, :valid_record) do + :ci_builds | invalid_ci_build | valid_ci_build + :ci_pipelines | invalid_ci_pipeline | valid_ci_pipeline + end + + subject(:perform) do + described_class.new( + start_id: 1, + end_id: 2, + batch_table: table_name, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ).perform + end + + shared_examples 'fix invalid records' do + it 'rebalances partition_id to 100 when partition_id is 101' do + expect { perform } + .to change { invalid_record.reload.partition_id }.from(101).to(100) + .and not_change { valid_record.reload.partition_id } + end + end + + with_them do + it_behaves_like 'fix invalid records' + end + end +end diff --git a/spec/migrations/20230125093723_rebalance_partition_id_ci_pipeline_spec.rb b/spec/migrations/20230125093723_rebalance_partition_id_ci_pipeline_spec.rb new file mode 100644 index 00000000000000..3ccd92e15a4822 --- /dev/null +++ b/spec/migrations/20230125093723_rebalance_partition_id_ci_pipeline_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe RebalancePartitionIdCiPipeline, migration: :gitlab_ci, feature_category: :continuous_integration do + let(:migration) { described_class::MIGRATION } + + context 'when on sass' do + before do + allow(Gitlab).to receive(:com?).and_return(true) + end + + describe '#up' do + it 'schedules background jobs for each batch of ci_builds' do + migrate! + + expect(migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :ci_pipelines, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + end + end + + describe '#down' do + it 'deletes all batched migration records' do + migrate! + schema_migrate_down! + + expect(migration).not_to have_scheduled_batched_migration + end + end + end + + context 'when on self-managed instance' do + let(:migration) { described_class.new } + + describe '#up' do + it 'does not schedule background job' do + expect(migration).not_to receive(:queue_batched_background_migration) + + migration.up + end + end + + describe '#down' do + it 'does not delete background job' do + expect(migration).not_to receive(:delete_batched_background_migration) + + migration.down + end + end + end +end diff --git a/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb b/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb new file mode 100644 index 00000000000000..b983564a2d997b --- /dev/null +++ b/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe RebalancePartitionIdCiBuild, migration: :gitlab_ci, feature_category: :continuous_integration do + let(:migration) { described_class::MIGRATION } + + context 'when on sass' do + before do + allow(Gitlab).to receive(:com?).and_return(true) + end + + describe '#up' do + it 'schedules background jobs for each batch of ci_builds' do + migrate! + + expect(migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :ci_builds, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + end + end + + describe '#down' do + it 'deletes all batched migration records' do + migrate! + schema_migrate_down! + + expect(migration).not_to have_scheduled_batched_migration + end + end + end + + context 'when on self-managed instance' do + let(:migration) { described_class.new } + + describe '#up' do + it 'does not schedule background job' do + expect(migration).not_to receive(:queue_batched_background_migration) + + migration.up + end + end + + describe '#down' do + it 'does not delete background job' do + expect(migration).not_to receive(:delete_batched_background_migration) + + migration.down + end + end + end +end -- GitLab From a6b9eb45c313a5cf0578eb8812a7ee1a7aefcbed Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Tue, 7 Feb 2023 09:53:42 +0100 Subject: [PATCH 2/2] Run only pipeline migration --- ...3723_rebalance_partition_id_ci_pipeline.rb | 4 +- ...5093840_rebalance_partition_id_ci_build.rb | 30 ---------- db/schema_migrations/20230125093840 | 1 - .../rebalance_partition_id_spec.rb | 3 - ...40_rebalance_partition_id_ci_build_spec.rb | 58 ------------------- 5 files changed, 2 insertions(+), 94 deletions(-) delete mode 100644 db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb delete mode 100644 db/schema_migrations/20230125093840 delete mode 100644 spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb diff --git a/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb b/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb index 3f31e765b4626d..aaea55ce3311bf 100644 --- a/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb +++ b/db/post_migrate/20230125093723_rebalance_partition_id_ci_pipeline.rb @@ -4,8 +4,8 @@ class RebalancePartitionIdCiPipeline < Gitlab::Database::Migration[2.1] MIGRATION = 'RebalancePartitionId' DELAY_INTERVAL = 2.minutes TABLE = :ci_pipelines - BATCH_SIZE = 1_000 - SUB_BATCH_SIZE = 100 + BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 200 restrict_gitlab_migration gitlab_schema: :gitlab_ci diff --git a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb deleted file mode 100644 index 7c096021178d56..00000000000000 --- a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -class RebalancePartitionIdCiBuild < Gitlab::Database::Migration[2.1] - MIGRATION = 'RebalancePartitionId' - DELAY_INTERVAL = 2.minutes.freeze - TABLE = :ci_builds - BATCH_SIZE = 1_000 - SUB_BATCH_SIZE = 100 - - restrict_gitlab_migration gitlab_schema: :gitlab_ci - - def up - return unless Gitlab.com? - - queue_batched_background_migration( - MIGRATION, - TABLE, - :id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - end - - def down - return unless Gitlab.com? - - delete_batched_background_migration(MIGRATION, TABLE, :id, []) - end -end diff --git a/db/schema_migrations/20230125093840 b/db/schema_migrations/20230125093840 deleted file mode 100644 index 1d2fab2561942d..00000000000000 --- a/db/schema_migrations/20230125093840 +++ /dev/null @@ -1 +0,0 @@ -c66f77a9de07e2f88b6d371b14f7f72068a5b8e25cb382cb08e578021affbeb7 \ No newline at end of file diff --git a/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb b/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb index 2c9483835a52bd..195e57e4e59e93 100644 --- a/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb +++ b/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb @@ -10,15 +10,12 @@ let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) } let!(:valid_ci_pipeline) { ci_pipelines_table.create!(id: 1, partition_id: 100) } - let!(:valid_ci_build) { ci_builds_table.create!(id: 1, partition_id: 100) } - let!(:invalid_ci_build) { ci_builds_table.create!(id: 2, partition_id: 101) } let!(:invalid_ci_pipeline) { ci_pipelines_table.create!(id: 2, partition_id: 101) } describe '#perform' do using RSpec::Parameterized::TableSyntax where(:table_name, :invalid_record, :valid_record) do - :ci_builds | invalid_ci_build | valid_ci_build :ci_pipelines | invalid_ci_pipeline | valid_ci_pipeline end diff --git a/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb b/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb deleted file mode 100644 index b983564a2d997b..00000000000000 --- a/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe RebalancePartitionIdCiBuild, migration: :gitlab_ci, feature_category: :continuous_integration do - let(:migration) { described_class::MIGRATION } - - context 'when on sass' do - before do - allow(Gitlab).to receive(:com?).and_return(true) - end - - describe '#up' do - it 'schedules background jobs for each batch of ci_builds' do - migrate! - - expect(migration).to have_scheduled_batched_migration( - gitlab_schema: :gitlab_ci, - table_name: :ci_builds, - column_name: :id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - sub_batch_size: described_class::SUB_BATCH_SIZE - ) - end - end - - describe '#down' do - it 'deletes all batched migration records' do - migrate! - schema_migrate_down! - - expect(migration).not_to have_scheduled_batched_migration - end - end - end - - context 'when on self-managed instance' do - let(:migration) { described_class.new } - - describe '#up' do - it 'does not schedule background job' do - expect(migration).not_to receive(:queue_batched_background_migration) - - migration.up - end - end - - describe '#down' do - it 'does not delete background job' do - expect(migration).not_to receive(:delete_batched_background_migration) - - migration.down - end - end - end -end -- GitLab