From 8fd8e1657f18ff755dce9635f09cd67c6a3cc63e Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Tue, 4 Mar 2025 15:45:53 +0530 Subject: [PATCH 1/7] Add migration to create hosted runner records Only create records created by admin bot Run the migration as background migration in batches Changelog: added EE: true MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/183329 --- .../mark_admin_bot_runners_as_hosted.yml | 8 +++ ..._queue_mark_admin_bot_runners_as_hosted.rb | 28 +++++++++ db/schema_migrations/20250305060745 | 1 + .../mark_admin_bot_runners_as_hosted.rb | 36 ++++++++++++ .../mark_admin_bot_runners_as_hosted_spec.rb | 58 +++++++++++++++++++ ...e_mark_admin_bot_runners_as_hosted_spec.rb | 27 +++++++++ 6 files changed, 158 insertions(+) create mode 100644 db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml create mode 100644 db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb create mode 100644 db/schema_migrations/20250305060745 create mode 100644 lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb create mode 100644 spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb create mode 100644 spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb diff --git a/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml b/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml new file mode 100644 index 00000000000000..82889b9c239c7c --- /dev/null +++ b/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: MarkAdminBotRunnersAsHosted +description: Mark runners created by admin bot as hosted on GitLab Dedicated +feature_category: hosted_runners +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/183329 +milestone: '17.10' +queued_migration_version: 20250305060745 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb b/db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb new file mode 100644 index 00000000000000..6144fa68d277be --- /dev/null +++ b/db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueMarkAdminBotRunnersAsHosted < Gitlab::Database::Migration[2.2] + milestone '17.10' + + # Select the applicable gitlab schema for your batched background migration + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = "MarkAdminBotRunnersAsHosted" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_runners, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_runners, :id, []) + end +end diff --git a/db/schema_migrations/20250305060745 b/db/schema_migrations/20250305060745 new file mode 100644 index 00000000000000..de52a4145735c1 --- /dev/null +++ b/db/schema_migrations/20250305060745 @@ -0,0 +1 @@ +4407850dd9ca274293b12a43070dd3c428e71d2508a6dd488528bdd8f56f8c7d \ No newline at end of file diff --git a/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb new file mode 100644 index 00000000000000..c7996aaa895d1a --- /dev/null +++ b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class MarkAdminBotRunnersAsHosted < BatchedMigrationJob + operation_name :mark_admin_bot_runners_as_hosted + feature_category :hosted_runners + + def perform + return unless Gitlab::CurrentSettings.gitlab_dedicated_instance + + each_sub_batch do |sub_batch| + sub_batch.each do |runner| + next unless runner.creator_id == admin_bot.id + + CiHostedRunner.find_or_create_by!(runner_id: runner.id) + end + end + end + + class ApplicationSetting < ApplicationRecord + self.table_name = :application_settings + end + + class CiHostedRunner < ::Ci::ApplicationRecord + self.table_name = :ci_hosted_runners + end + + private + + def admin_bot + @_admin_bot ||= Users::Internal.admin_bot + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb b/spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb new file mode 100644 index 00000000000000..05cb42c9ddfa83 --- /dev/null +++ b/spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MarkAdminBotRunnersAsHosted, + feature_category: :hosted_runners do + let!(:admin_bot) { Users::Internal.admin_bot } + + let!(:admin_bot_runner) { table(:ci_runners, database: :ci).create!(runner_type: 1, creator_id: admin_bot.id) } + let!(:admin_bot_runner2) { table(:ci_runners, database: :ci).create!(runner_type: 1, creator_id: admin_bot.id) } + let!(:non_admin_bot_runner) { table(:ci_runners, database: :ci).create!(runner_type: 1) } + + before do + allow(Gitlab::CurrentSettings).to receive(:gitlab_dedicated_instance).and_return(true) + end + + subject(:migration) do + described_class.new( + start_id: Ci::Runner.minimum(:id), + end_id: Ci::Runner.maximum(:id), + batch_table: :ci_runners, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: ::Ci::ApplicationRecord.connection + ) + end + + describe '#perform' do + context 'when on a dedicated instance' do + it 'marks runners created by admin bot as hosted' do + expect { migration.perform }.to change { Ci::HostedRunner.count }.by(2) + + expect(Ci::HostedRunner.exists?(runner_id: admin_bot_runner.id)).to be_truthy + expect(Ci::HostedRunner.exists?(runner_id: admin_bot_runner2.id)).to be_truthy + expect(Ci::HostedRunner.exists?(runner_id: non_admin_bot_runner.id)).to be_falsey + end + + it 'does not create duplicate hosted runner records' do + Ci::HostedRunner.create!(runner_id: admin_bot_runner.id) + + expect { migration.perform }.to change { Ci::HostedRunner.count }.by(1) + + expect(Ci::HostedRunner.exists?(runner_id: admin_bot_runner2.id)).to be_truthy + end + end + + context 'when not on a dedicated instance' do + before do + allow(Gitlab::CurrentSettings).to receive(:gitlab_dedicated_instance).and_return(false) + end + + it 'does not create any hosted runner records' do + expect { migration.perform }.not_to change { Ci::HostedRunner.count } + end + end + end +end diff --git a/spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb b/spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb new file mode 100644 index 00000000000000..4738d49a36d306 --- /dev/null +++ b/spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueMarkAdminBotRunnersAsHosted, migration: :gitlab_ci, feature_category: :hosted_runners do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :ci_runners, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end -- GitLab From 1a9093938dc8de1b4c6a5a98ef6737296a43ca21 Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Fri, 7 Mar 2025 15:48:55 +0530 Subject: [PATCH 2/7] Move the background migration to ee sub folder and create empty ce class --- .../mark_admin_bot_runners_as_hosted.rb | 44 +++++++++++++++++++ .../mark_admin_bot_runners_as_hosted_spec.rb | 0 .../mark_admin_bot_runners_as_hosted.rb | 33 +++----------- 3 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 ee/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb rename {spec/lib => ee/spec/lib/ee}/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb (100%) diff --git a/ee/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb b/ee/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb new file mode 100644 index 00000000000000..3332ec1e8d7aa9 --- /dev/null +++ b/ee/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module EE + module Gitlab + module BackgroundMigration + module MarkAdminBotRunnersAsHosted + extend ActiveSupport::Concern + extend ::Gitlab::Utils::Override + + prepended do + operation_name :mark_admin_bot_runners_as_hosted + feature_category :hosted_runners + end + + override :perform + def perform + return unless ::Gitlab::CurrentSettings.gitlab_dedicated_instance + + each_sub_batch do |sub_batch| + sub_batch.each do |runner| + next unless runner.creator_id == admin_bot.id + + CiHostedRunner.find_or_create_by!(runner_id: runner.id) + end + end + end + + class ApplicationSetting < ApplicationRecord + self.table_name = :application_settings + end + + class CiHostedRunner < ::Ci::ApplicationRecord + self.table_name = :ci_hosted_runners + end + + private + + def admin_bot + @_admin_bot ||= ::Users::Internal.admin_bot + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb similarity index 100% rename from spec/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb rename to ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb diff --git a/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb index c7996aaa895d1a..b81bcc85e21d53 100644 --- a/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb +++ b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb @@ -2,35 +2,16 @@ module Gitlab module BackgroundMigration + # This class doesn't create SecuritySetting + # as this feature exists only in EE class MarkAdminBotRunnersAsHosted < BatchedMigrationJob - operation_name :mark_admin_bot_runners_as_hosted feature_category :hosted_runners - def perform - return unless Gitlab::CurrentSettings.gitlab_dedicated_instance - - each_sub_batch do |sub_batch| - sub_batch.each do |runner| - next unless runner.creator_id == admin_bot.id - - CiHostedRunner.find_or_create_by!(runner_id: runner.id) - end - end - end - - class ApplicationSetting < ApplicationRecord - self.table_name = :application_settings - end - - class CiHostedRunner < ::Ci::ApplicationRecord - self.table_name = :ci_hosted_runners - end - - private - - def admin_bot - @_admin_bot ||= Users::Internal.admin_bot - end + def perform(_from_id, _to_id); end end end end + +# rubocop:disable Layout/LineLength -- If I do multiline, another cop complains about prepend should be last line +Gitlab::BackgroundMigration::MarkAdminBotRunnersAsHosted.prepend_mod_with('Gitlab::BackgroundMigration::MarkAdminBotRunnersAsHosted') +# rubocop:enable Layout/LineLength -- GitLab From 2ee15a1325e5964c13ec2e5c0104a156a6019341 Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Fri, 7 Mar 2025 18:15:24 +0530 Subject: [PATCH 3/7] Fix batched migration spec --- .../background_migration/mark_admin_bot_runners_as_hosted.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb index b81bcc85e21d53..edf5cc61aee5a8 100644 --- a/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb +++ b/lib/gitlab/background_migration/mark_admin_bot_runners_as_hosted.rb @@ -7,7 +7,7 @@ module BackgroundMigration class MarkAdminBotRunnersAsHosted < BatchedMigrationJob feature_category :hosted_runners - def perform(_from_id, _to_id); end + def perform; end end end end -- GitLab From 4bd0f13f1df6c473a0a5ef3ea3b0b7479184c28f Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Mon, 5 May 2025 15:25:57 +0530 Subject: [PATCH 4/7] Update migration timestamps --- .../mark_admin_bot_runners_as_hosted.yml | 4 ++-- ... 20250505095336_queue_mark_admin_bot_runners_as_hosted.rb} | 2 +- db/schema_migrations/{20250305060745 => 20250505095336} | 0 ...0505095336_queue_mark_admin_bot_runners_as_hosted_spec.rb} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename db/post_migrate/{20250305060745_queue_mark_admin_bot_runners_as_hosted.rb => 20250505095336_queue_mark_admin_bot_runners_as_hosted.rb} (97%) rename db/schema_migrations/{20250305060745 => 20250505095336} (100%) rename spec/migrations/{20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb => 20250505095336_queue_mark_admin_bot_runners_as_hosted_spec.rb} (100%) diff --git a/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml b/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml index 82889b9c239c7c..358c4124bbe1cb 100644 --- a/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml +++ b/db/docs/batched_background_migrations/mark_admin_bot_runners_as_hosted.yml @@ -3,6 +3,6 @@ migration_job_name: MarkAdminBotRunnersAsHosted description: Mark runners created by admin bot as hosted on GitLab Dedicated feature_category: hosted_runners introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/183329 -milestone: '17.10' -queued_migration_version: 20250305060745 +milestone: '18.0' +queued_migration_version: 20250505095336 finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb b/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb similarity index 97% rename from db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb rename to db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb index 6144fa68d277be..d84b7e8f72b7de 100644 --- a/db/post_migrate/20250305060745_queue_mark_admin_bot_runners_as_hosted.rb +++ b/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class QueueMarkAdminBotRunnersAsHosted < Gitlab::Database::Migration[2.2] - milestone '17.10' + milestone '18.0' # Select the applicable gitlab schema for your batched background migration restrict_gitlab_migration gitlab_schema: :gitlab_ci diff --git a/db/schema_migrations/20250305060745 b/db/schema_migrations/20250505095336 similarity index 100% rename from db/schema_migrations/20250305060745 rename to db/schema_migrations/20250505095336 diff --git a/spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb b/spec/migrations/20250505095336_queue_mark_admin_bot_runners_as_hosted_spec.rb similarity index 100% rename from spec/migrations/20250305060745_queue_mark_admin_bot_runners_as_hosted_spec.rb rename to spec/migrations/20250505095336_queue_mark_admin_bot_runners_as_hosted_spec.rb -- GitLab From 8a92604bd942cdd703c6b90ab55aa2d9f0ca846b Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Mon, 5 May 2025 16:10:53 +0530 Subject: [PATCH 5/7] Fix cop offense for migration super class --- .../20250505095336_queue_mark_admin_bot_runners_as_hosted.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb b/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb index d84b7e8f72b7de..a1a2eacf4aa9a7 100644 --- a/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb +++ b/db/post_migrate/20250505095336_queue_mark_admin_bot_runners_as_hosted.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class QueueMarkAdminBotRunnersAsHosted < Gitlab::Database::Migration[2.2] +class QueueMarkAdminBotRunnersAsHosted < Gitlab::Database::Migration[2.3] milestone '18.0' # Select the applicable gitlab schema for your batched background migration -- GitLab From d501687c58bbb5329140c076bfca3ae1ef5e707c Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Mon, 5 May 2025 18:01:11 +0530 Subject: [PATCH 6/7] Re-run the migrations --- db/schema_migrations/20250505095336 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema_migrations/20250505095336 b/db/schema_migrations/20250505095336 index de52a4145735c1..28ff1b5608bacf 100644 --- a/db/schema_migrations/20250505095336 +++ b/db/schema_migrations/20250505095336 @@ -1 +1 @@ -4407850dd9ca274293b12a43070dd3c428e71d2508a6dd488528bdd8f56f8c7d \ No newline at end of file +75166bbf5d04065b48b810200cada005c73109b8da8ccebc6ed9f3195951928a \ No newline at end of file -- GitLab From d0dd07f9305bf3b1f0d7cf335e8b5798a4460509 Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Tue, 6 May 2025 11:11:50 +0530 Subject: [PATCH 7/7] Fix spec failures by adding primary key arg to table creation --- .../mark_admin_bot_runners_as_hosted_spec.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb index 05cb42c9ddfa83..4dd7a2113e5f9a 100644 --- a/ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb +++ b/ee/spec/lib/ee/gitlab/background_migration/mark_admin_bot_runners_as_hosted_spec.rb @@ -6,9 +6,17 @@ feature_category: :hosted_runners do let!(:admin_bot) { Users::Internal.admin_bot } - let!(:admin_bot_runner) { table(:ci_runners, database: :ci).create!(runner_type: 1, creator_id: admin_bot.id) } - let!(:admin_bot_runner2) { table(:ci_runners, database: :ci).create!(runner_type: 1, creator_id: admin_bot.id) } - let!(:non_admin_bot_runner) { table(:ci_runners, database: :ci).create!(runner_type: 1) } + let!(:admin_bot_runner) do + table(:ci_runners, database: :ci, primary_key: :id) + .create!(runner_type: 1, creator_id: admin_bot.id) + end + + let!(:admin_bot_runner2) do + table(:ci_runners, database: :ci, primary_key: :id) + .create!(runner_type: 1, creator_id: admin_bot.id) + end + + let!(:non_admin_bot_runner) { table(:ci_runners, database: :ci, primary_key: :id).create!(runner_type: 1) } before do allow(Gitlab::CurrentSettings).to receive(:gitlab_dedicated_instance).and_return(true) -- GitLab