diff --git a/db/migrate/20220316022508_backfill_namespace_details.rb b/db/migrate/20220316022508_backfill_namespace_details.rb new file mode 100644 index 0000000000000000000000000000000000000000..eda9b135d44b74d51fb954e57a8798c1f5db5e7b --- /dev/null +++ b/db/migrate/20220316022508_backfill_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillNamespaceDetails < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillNamespaceDetails' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 200 + + def up + queue_batched_background_migration( + MIGRATION, + :namespaces, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) + end +end diff --git a/db/migrate/20220406011508_backfill_project_namespace_details.rb b/db/migrate/20220406011508_backfill_project_namespace_details.rb new file mode 100644 index 0000000000000000000000000000000000000000..a2501d40b5d2b7a753f5037020bffe801aae6c45 --- /dev/null +++ b/db/migrate/20220406011508_backfill_project_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillProjectNamespaceDetails < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillProjectNamespaceDetails' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 200 + + def up + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/schema_migrations/20220316022508 b/db/schema_migrations/20220316022508 new file mode 100644 index 0000000000000000000000000000000000000000..06734edec8d1f0323c3e48cb2298275029bbca28 --- /dev/null +++ b/db/schema_migrations/20220316022508 @@ -0,0 +1 @@ +0f4487bcba4e60c442e477e05037d05d4cb36fe5cc88445d4d2021e14a193fc5 \ No newline at end of file diff --git a/db/schema_migrations/20220406011508 b/db/schema_migrations/20220406011508 new file mode 100644 index 0000000000000000000000000000000000000000..e35d7fee09edf7a7085bc9346e7af5baee8f0f28 --- /dev/null +++ b/db/schema_migrations/20220406011508 @@ -0,0 +1 @@ +a3c1a618e4b7e16b425fb5624db3f78fe053a72ff5273956d88320fbb3f3ecb3 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb new file mode 100644 index 0000000000000000000000000000000000000000..431557a67cd719359ffbe880fc0d7a198800722f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill namespace_details for a range of namespaces + class BackfillNamespaceDetails + def perform(start_id, end_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id + FROM namespaces + WHERE namespaces.id BETWEEN #{start_id} AND #{end_id} + AND namespaces.type <> 'Project' + ON CONFLICT (namespace_id) DO NOTHING; + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_namespace_details.rb b/lib/gitlab/background_migration/backfill_project_namespace_details.rb new file mode 100644 index 0000000000000000000000000000000000000000..fd08895f58c24de676056730438da016dc9d1d01 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_details.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_namespace namespace_details for a range of project namespaces + class BackfillProjectNamespaceDetails + def perform(start_id, end_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT projects.description, projects.description_html, projects.cached_markdown_version, now(), now(), projects.project_namespace_id + FROM projects + WHERE projects.project_namespace_id BETWEEN #{start_id} AND #{end_id} + ON CONFLICT (namespace_id) DO NOTHING; + SQL + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..cacb3509d9aa73a2e6c29ac5c3a215af09bcefc3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails do + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + + subject { described_class.new } + + describe '#perform' do + it 'creates details for all namespaces in range' do + namespaces.create!(id: 5, name: 'test1', path: 'test1') + namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project') + namespaces.create!(id: 7, name: 'test3', path: 'test3') + namespaces.create!(id: 8, name: 'test4', path: 'test4') + + subject.perform(5, 7) + + expect(namespace_details.pluck(:namespace_id)).to contain_exactly(5, 7) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1256a3738a16d8908e33f767fd05b19c5cb579fb --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +# rubocop:disable RSpec/FactoriesInMigrationSpecs +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails do + let(:namespace_details) { table(:namespace_details) } + let(:namespaces) { table(:namespaces) } + + subject { described_class.new } + + describe '#perform' do + let(:project_namespace1) { create(:project_namespace) } + let(:project_namespace2) { create(:project_namespace) } + let(:project_namespace3) { create(:project_namespace) } + + it 'creates details for all project namespaces in range' do + subject.perform(project_namespace1.id, project_namespace2.id) + + expect(namespace_details.pluck(:namespace_id) & namespaces.all.where(type: "Project").pluck(:id)) + .to contain_exactly(project_namespace1.id, project_namespace2.id) + end + end +end