From d05fa2aa3431af184323c3e3fe5f029e1ee45245 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 19 Jan 2023 17:55:00 +0200 Subject: [PATCH 1/2] Fix partition ids for ci_sources_pipelines records Changelog: fixed --- ...x_partition_ids_on_ci_sources_pipelines.rb | 34 ++++++++++++++ db/schema_migrations/20230214154101 | 1 + ...tition_ids_on_ci_sources_pipelines_spec.rb | 45 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb create mode 100644 db/schema_migrations/20230214154101 create mode 100644 spec/migrations/20230214154101_fix_partition_ids_on_ci_sources_pipelines_spec.rb diff --git a/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb b/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb new file mode 100644 index 00000000000000..86d60bea0503e2 --- /dev/null +++ b/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class FixPartitionIdsOnCiSourcesPipelines < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + BATCH_SIZE = 250 + + def up + return unless Gitlab.com? + + model = define_batchable_model(:ci_sources_pipelines) + + batch_update_records(model, :partition_id, from: 101, to: 100) + batch_update_records(model, :source_partition_id, from: 101, to: 100) + end + + def down + # no-op + end + + private + + def batch_update_records(model, column, from:, to:) + model + .where(model.arel_table[column].eq(from)) + .each_batch(of: BATCH_SIZE) { |batch| update_records(batch, { column => to }) } + end + + def update_records(relation, updates) + relation.update_all(updates) + sleep 0.1 + end +end diff --git a/db/schema_migrations/20230214154101 b/db/schema_migrations/20230214154101 new file mode 100644 index 00000000000000..97ea6dfb25919d --- /dev/null +++ b/db/schema_migrations/20230214154101 @@ -0,0 +1 @@ +89f5c87983f2739ad20f1f3f3542aea5cd9373879d399835207028792e1e097c \ No newline at end of file diff --git a/spec/migrations/20230214154101_fix_partition_ids_on_ci_sources_pipelines_spec.rb b/spec/migrations/20230214154101_fix_partition_ids_on_ci_sources_pipelines_spec.rb new file mode 100644 index 00000000000000..44031175497259 --- /dev/null +++ b/spec/migrations/20230214154101_fix_partition_ids_on_ci_sources_pipelines_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe FixPartitionIdsOnCiSourcesPipelines, migration: :gitlab_ci, feature_category: :continuous_integration do + let(:sources_pipelines) { table(:ci_sources_pipelines, database: :ci) } + + before do + sources_pipelines.insert_all!([ + { partition_id: 100, source_partition_id: 100 }, + { partition_id: 100, source_partition_id: 101 }, + { partition_id: 101, source_partition_id: 100 }, + { partition_id: 101, source_partition_id: 101 } + ]) + end + + describe '#up' do + context 'when on sass' do + before do + allow(Gitlab).to receive(:com?).and_return(true) + end + + it 'fixes partition_id and source_partition_id' do + expect { migrate! }.not_to raise_error + + expect(sources_pipelines.where(partition_id: 100).count).to eq(4) + expect(sources_pipelines.where(partition_id: 101).count).to eq(0) + expect(sources_pipelines.where(source_partition_id: 100).count).to eq(4) + expect(sources_pipelines.where(source_partition_id: 101).count).to eq(0) + end + end + + context 'when on self managed' do + it 'does not change partition_id or source_partition_id' do + expect { migrate! }.not_to raise_error + + expect(sources_pipelines.where(partition_id: 100).count).to eq(2) + expect(sources_pipelines.where(partition_id: 100).count).to eq(2) + expect(sources_pipelines.where(source_partition_id: 101).count).to eq(2) + expect(sources_pipelines.where(source_partition_id: 101).count).to eq(2) + end + end + end +end -- GitLab From 32d09f01cbf3a6aad68f6a22aeef7b1e6e407a35 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 16 Feb 2023 11:59:50 +0200 Subject: [PATCH 2/2] Apply review feedback --- ...154101_fix_partition_ids_on_ci_sources_pipelines.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb b/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb index 86d60bea0503e2..c05b759c2d0438 100644 --- a/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb +++ b/db/post_migrate/20230214154101_fix_partition_ids_on_ci_sources_pipelines.rb @@ -4,14 +4,14 @@ class FixPartitionIdsOnCiSourcesPipelines < Gitlab::Database::Migration[2.1] disable_ddl_transaction! restrict_gitlab_migration gitlab_schema: :gitlab_ci - BATCH_SIZE = 250 + BATCH_SIZE = 50 def up return unless Gitlab.com? model = define_batchable_model(:ci_sources_pipelines) - batch_update_records(model, :partition_id, from: 101, to: 100) + batch_update_records(model, :partition_id, from: 101, to: 100, source_partition_id: 100) batch_update_records(model, :source_partition_id, from: 101, to: 100) end @@ -21,10 +21,12 @@ def down private - def batch_update_records(model, column, from:, to:) + def batch_update_records(model, column, from:, to:, **updates) + updates.reverse_merge!(column => to) + model .where(model.arel_table[column].eq(from)) - .each_batch(of: BATCH_SIZE) { |batch| update_records(batch, { column => to }) } + .each_batch(of: BATCH_SIZE) { |batch| update_records(batch, updates) } end def update_records(relation, updates) -- GitLab