diff --git a/db/docs/batched_background_migrations/backfill_user_agent_details_organization_id.yml b/db/docs/batched_background_migrations/backfill_user_agent_details_organization_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..fa05a4fe07c70420ae27ca75c9f4e2f8037962a4 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_user_agent_details_organization_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillUserAgentDetailsOrganizationId +description: Backfill the sharding key on the user_agent_details table +feature_category: instance_resiliency +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215327 +milestone: '18.8' +queued_migration_version: 20251215170942 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251215170942_queue_backfill_user_agent_details_organization_id.rb b/db/post_migrate/20251215170942_queue_backfill_user_agent_details_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..2dbae2a3ada5107fd0dcca3c92b31544b491d746 --- /dev/null +++ b/db/post_migrate/20251215170942_queue_backfill_user_agent_details_organization_id.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillUserAgentDetailsOrganizationId < Gitlab::Database::Migration[2.3] + milestone '18.8' + + restrict_gitlab_migration gitlab_schema: :gitlab_main_org + + MIGRATION = "BackfillUserAgentDetailsOrganizationId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :user_agent_details, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :user_agent_details, :id, []) + end +end diff --git a/db/schema_migrations/20251215170942 b/db/schema_migrations/20251215170942 new file mode 100644 index 0000000000000000000000000000000000000000..d5040f74eee713037413eb8bd1485fe9ac754492 --- /dev/null +++ b/db/schema_migrations/20251215170942 @@ -0,0 +1 @@ +d46b4e76569d49fc319328e2a6fa6e4518e2cd816fecef517813701408825ffd \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_user_agent_details_organization_id.rb b/lib/gitlab/background_migration/backfill_user_agent_details_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..c4a5719dede5d3d3a7a664db6a8111f5b5c1cfcc --- /dev/null +++ b/lib/gitlab/background_migration/backfill_user_agent_details_organization_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillUserAgentDetailsOrganizationId < BatchedMigrationJob + operation_name :backfill_user_agent_details_organization_id + feature_category :instance_resiliency + + ORGANIZATION_ID = 1 + + def perform + each_sub_batch do |sub_batch| + sub_batch + .where(organization_id: nil) + .update_all(organization_id: ORGANIZATION_ID) + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_user_agent_details_organization_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_agent_details_organization_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1589c4128a71ebc28d3915887fb46db78ca1a595 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_user_agent_details_organization_id_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillUserAgentDetailsOrganizationId, feature_category: :instance_resiliency do + let(:connection) { ApplicationRecord.connection } + + let(:issues) { table(:issues) } + let(:namespaces) { table(:namespaces) } + let(:organizations) { table(:organizations) } + let(:projects) { table(:projects) } + let(:user_agent_details) { table(:user_agent_details) { |t| t.belongs_to :subject, polymorphic: true } } + + let!(:default_organization) { organizations.create!(id: 1, name: 'default', path: 'default') } + let(:organization) { organizations.create!(name: 'organization', path: 'organization') } + let(:namespace1) { namespaces.create!(name: 'namespace 1', path: 'namespace1', organization_id: organization.id) } + let(:project1) do + projects.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id, organization_id: organization.id) + end + + let(:issue1) do + issues.create!(namespace_id: namespace1.id, project_id: project1.id, created_at: 5.days.ago, closed_at: 3.days.ago, + work_item_type_id: work_item_type_id) + end + + let(:issue2) do + issues.create!(namespace_id: namespace1.id, project_id: project1.id, created_at: 7.days.ago, closed_at: 2.days.ago, + work_item_type_id: work_item_type_id) + end + + let(:work_item_type_id) { table(:work_item_types).where(base_type: 1).first.id } + + describe '#perform' do + subject(:migration) do + described_class.new( + start_id: user_agent_details.minimum(:id), + end_id: user_agent_details.maximum(:id), + batch_table: :user_agent_details, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: connection + ) + end + + before do + create_user_agent_detail!(organization: organization, target: issue1) + create_user_agent_detail!(organization: organization, target: issue2) + create_user_agent_detail!(organization: nil, target: issue1) + create_user_agent_detail!(organization: nil, target: issue2) + end + + it 'updates records without an organization' do + expect { migration.perform }.to change { user_agent_details.where(organization_id: nil).count }.from(2).to(0) + end + end + + def create_user_agent_detail!(target:, organization: nil) + user_agent_details.create!( + organization_id: organization&.id, + ip_address: '127.0.0.1', + user_agent: 'Mozilla/5.0 (X11; Linux i686; rv:136.0) Gecko/20100101 Firefox/136.0', + subject_id: target&.id, + subject_type: target.class.name + ) + end +end diff --git a/spec/migrations/20251215170942_queue_backfill_user_agent_details_organization_id_spec.rb b/spec/migrations/20251215170942_queue_backfill_user_agent_details_organization_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f693440b021e3cd9c5093866eeabae14e80e23eb --- /dev/null +++ b/spec/migrations/20251215170942_queue_backfill_user_agent_details_organization_id_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillUserAgentDetailsOrganizationId, migration: :gitlab_main_org, feature_category: :instance_resiliency do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main_org, + table_name: :user_agent_details, + column_name: :id, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end