From 6b115272150d57c3a1544f7eb011ae70075245f9 Mon Sep 17 00:00:00 2001
From: Maxime Orefice <morefice@gitlab.com>
Date: Wed, 25 Jan 2023 11:00:35 +0100
Subject: [PATCH 1/2] Rebalance partition_id for ci_builds & ci_pipelines

This commits updates all invalid records in production and
sets the partition_id to 100.

Changelog: added
---
 ...5093840_rebalance_partition_id_ci_build.rb | 30 ++++++++++
 db/schema_migrations/20230125093840           |  1 +
 ...40_rebalance_partition_id_ci_build_spec.rb | 58 +++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
 create mode 100644 db/schema_migrations/20230125093840
 create mode 100644 spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb

diff --git a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
new file mode 100644
index 00000000000000..7c096021178d56
--- /dev/null
+++ b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class RebalancePartitionIdCiBuild < Gitlab::Database::Migration[2.1]
+  MIGRATION = 'RebalancePartitionId'
+  DELAY_INTERVAL = 2.minutes.freeze
+  TABLE = :ci_builds
+  BATCH_SIZE = 1_000
+  SUB_BATCH_SIZE = 100
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_ci
+
+  def up
+    return unless Gitlab.com?
+
+    queue_batched_background_migration(
+      MIGRATION,
+      TABLE,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    return unless Gitlab.com?
+
+    delete_batched_background_migration(MIGRATION, TABLE, :id, [])
+  end
+end
diff --git a/db/schema_migrations/20230125093840 b/db/schema_migrations/20230125093840
new file mode 100644
index 00000000000000..1d2fab2561942d
--- /dev/null
+++ b/db/schema_migrations/20230125093840
@@ -0,0 +1 @@
+c66f77a9de07e2f88b6d371b14f7f72068a5b8e25cb382cb08e578021affbeb7
\ No newline at end of file
diff --git a/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb b/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb
new file mode 100644
index 00000000000000..b983564a2d997b
--- /dev/null
+++ b/spec/migrations/20230125093840_rebalance_partition_id_ci_build_spec.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe RebalancePartitionIdCiBuild, migration: :gitlab_ci, feature_category: :continuous_integration do
+  let(:migration) { described_class::MIGRATION }
+
+  context 'when on sass' do
+    before do
+      allow(Gitlab).to receive(:com?).and_return(true)
+    end
+
+    describe '#up' do
+      it 'schedules background jobs for each batch of ci_builds' do
+        migrate!
+
+        expect(migration).to have_scheduled_batched_migration(
+          gitlab_schema: :gitlab_ci,
+          table_name: :ci_builds,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      end
+    end
+
+    describe '#down' do
+      it 'deletes all batched migration records' do
+        migrate!
+        schema_migrate_down!
+
+        expect(migration).not_to have_scheduled_batched_migration
+      end
+    end
+  end
+
+  context 'when on self-managed instance' do
+    let(:migration) { described_class.new }
+
+    describe '#up' do
+      it 'does not schedule background job' do
+        expect(migration).not_to receive(:queue_batched_background_migration)
+
+        migration.up
+      end
+    end
+
+    describe '#down' do
+      it 'does not delete background job' do
+        expect(migration).not_to receive(:delete_batched_background_migration)
+
+        migration.down
+      end
+    end
+  end
+end
-- 
GitLab


From 8226dde87b1487fbe5c9c0622e483e0d0bdc2624 Mon Sep 17 00:00:00 2001
From: Maxime Orefice <morefice@gitlab.com>
Date: Fri, 10 Feb 2023 11:50:08 +0100
Subject: [PATCH 2/2] Increase batch size

---
 .../20230125093840_rebalance_partition_id_ci_build.rb         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
index 7c096021178d56..6165c266a8219b 100644
--- a/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
+++ b/db/post_migrate/20230125093840_rebalance_partition_id_ci_build.rb
@@ -4,8 +4,8 @@ class RebalancePartitionIdCiBuild < Gitlab::Database::Migration[2.1]
   MIGRATION = 'RebalancePartitionId'
   DELAY_INTERVAL = 2.minutes.freeze
   TABLE = :ci_builds
-  BATCH_SIZE = 1_000
-  SUB_BATCH_SIZE = 100
+  BATCH_SIZE = 5_000
+  SUB_BATCH_SIZE = 500
 
   restrict_gitlab_migration gitlab_schema: :gitlab_ci
 
-- 
GitLab