From 55f66b67090c57605657650d1e2a255126f60e7c Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Wed, 12 Feb 2025 17:05:27 +1300 Subject: [PATCH] Back-fill new partitioned uploads table Back-fill `uploads_9ba88c4165` by joining with the related model table, in order to also populate the charding key(s) and exclude parentless uploads. https://gitlab.com/gitlab-org/gitlab/-/issues/398199 Changelog: other --- app/models/bulk_imports/export_upload.rb | 4 +- app/models/design_management/action.rb | 2 +- .../import_export/relation_export_upload.rb | 2 +- .../backfill_partitioned_uploads.yml | 8 + ...4216_queue_backfill_partitioned_uploads.rb | 27 + db/schema_migrations/20250609164216 | 1 + ee/app/models/issuable_metric_image.rb | 2 +- ee/spec/models/issuable_metric_image_spec.rb | 3 +- .../backfill_partitioned_uploads.rb | 147 ++++ .../backfill_partitioned_uploads_spec.rb | 704 ++++++++++++++++++ ...queue_backfill_partitioned_uploads_spec.rb | 26 + .../models/bulk_imports/export_upload_spec.rb | 3 + spec/models/design_management/action_spec.rb | 3 +- .../relation_export_upload_spec.rb | 3 +- 14 files changed, 924 insertions(+), 11 deletions(-) create mode 100644 db/docs/batched_background_migrations/backfill_partitioned_uploads.yml create mode 100644 db/post_migrate/20250609164216_queue_backfill_partitioned_uploads.rb create mode 100644 db/schema_migrations/20250609164216 create mode 100644 lib/gitlab/background_migration/backfill_partitioned_uploads.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_partitioned_uploads_spec.rb create mode 100644 spec/migrations/queue_backfill_partitioned_uploads_spec.rb diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb index ee837b3c36bd7a..c4147bf9ed38b9 100644 --- a/app/models/bulk_imports/export_upload.rb +++ b/app/models/bulk_imports/export_upload.rb @@ -25,8 +25,8 @@ def retrieve_upload(_identifier, paths) def uploads_sharding_key { - project_id: export&.project_id, - namespace_id: export&.group_id + project_id: project_id, + namespace_id: group_id } end end diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb index 3d055873633977..d53acbc69ec49e 100644 --- a/app/models/design_management/action.rb +++ b/app/models/design_management/action.rb @@ -45,7 +45,7 @@ class Action < ApplicationRecord end def uploads_sharding_key - { namespace_id: design&.namespace_id } + { namespace_id: namespace_id } end end end diff --git a/app/models/projects/import_export/relation_export_upload.rb b/app/models/projects/import_export/relation_export_upload.rb index 72f2bc1b676812..f6675853f7ec94 100644 --- a/app/models/projects/import_export/relation_export_upload.rb +++ b/app/models/projects/import_export/relation_export_upload.rb @@ -29,7 +29,7 @@ class RelationExportUpload < ApplicationRecord end def uploads_sharding_key - { project_id: relation_export&.project_id } + { project_id: project_id } end end end diff --git a/db/docs/batched_background_migrations/backfill_partitioned_uploads.yml b/db/docs/batched_background_migrations/backfill_partitioned_uploads.yml new file mode 100644 index 00000000000000..91827ca5ecb988 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_partitioned_uploads.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillPartitionedUploads +description: Back-fill partitoned uploads_9ba88c4165 table and populate sharding keys +feature_category: database +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181349 +milestone: '18.2' +queued_migration_version: 20250609164216 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20250609164216_queue_backfill_partitioned_uploads.rb b/db/post_migrate/20250609164216_queue_backfill_partitioned_uploads.rb new file mode 100644 index 00000000000000..ab0affe6c6526c --- /dev/null +++ b/db/post_migrate/20250609164216_queue_backfill_partitioned_uploads.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillPartitionedUploads < Gitlab::Database::Migration[2.3] + milestone '18.2' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillPartitionedUploads" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 7000 + SUB_BATCH_SIZE = 300 + + def up + queue_batched_background_migration( + MIGRATION, + :uploads, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :uploads, :id, []) + end +end diff --git a/db/schema_migrations/20250609164216 b/db/schema_migrations/20250609164216 new file mode 100644 index 00000000000000..2354b55752d8da --- /dev/null +++ b/db/schema_migrations/20250609164216 @@ -0,0 +1 @@ +83db7e1f7c0cfb8e7eb3a83c1f5cd9de04967d31e94f9573b0f23335e28a8768 \ No newline at end of file diff --git a/ee/app/models/issuable_metric_image.rb b/ee/app/models/issuable_metric_image.rb index 4299477460ab44..78cde6eed96bf3 100644 --- a/ee/app/models/issuable_metric_image.rb +++ b/ee/app/models/issuable_metric_image.rb @@ -10,7 +10,7 @@ def self.available_for?(project) end def uploads_sharding_key - { namespace_id: issue&.namespace_id } + { namespace_id: namespace_id } end private diff --git a/ee/spec/models/issuable_metric_image_spec.rb b/ee/spec/models/issuable_metric_image_spec.rb index 01168efcf64309..17c183478365c4 100644 --- a/ee/spec/models/issuable_metric_image_spec.rb +++ b/ee/spec/models/issuable_metric_image_spec.rb @@ -88,8 +88,7 @@ describe '#uploads_sharding_key' do it 'returns namespace_id' do namespace = build_stubbed(:namespace) - issue = build_stubbed(:issue, namespace: namespace) - issuable_metric_image = build_stubbed(:issuable_metric_image, issue: issue) + issuable_metric_image = build_stubbed(:issuable_metric_image, namespace_id: namespace.id) expect(issuable_metric_image.uploads_sharding_key).to eq(namespace_id: namespace.id) end diff --git a/lib/gitlab/background_migration/backfill_partitioned_uploads.rb b/lib/gitlab/background_migration/backfill_partitioned_uploads.rb new file mode 100644 index 00000000000000..2ecc8b0bd377b9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partitioned_uploads.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPartitionedUploads < BatchedMigrationJob + feature_category :database + operation_name :backfill + + class Upload < ApplicationRecord + self.table_name = 'uploads_9ba88c4165' + end + + def perform + each_sub_batch do |sub_batch| + tables_and_models.each do |model_table, model_name, sources, targets, join_key, db_name| + process_upload_type(sub_batch, model_table, model_name, sources, targets, join_key, db_name) + end + end + end + + private + + def sharding_key_columns + %w[organization_id namespace_id project_id] + end + + def columns + Upload.column_names - sharding_key_columns + end + + def tables_and_models + # model_table, model_name, sources, targets, join_key, db_name + [ + ['abuse_reports', 'AbuseReport', %w[]], + ['achievements', 'Achievements::Achievement', %w[namespace_id]], + ['ai_vectorizable_files', 'Ai::VectorizableFile', %w[project_id]], + ['alert_management_alert_metric_images', 'AlertManagement::MetricImage', %w[project_id]], + ['appearances', 'Appearance', %w[]], + ['bulk_import_export_uploads', 'BulkImports::ExportUpload', %w[group_id project_id], + %w[namespace_id project_id]], + ['design_management_designs_versions', 'DesignManagement::Action', %w[namespace_id]], + ['import_export_uploads', 'ImportExportUpload', %w[group_id project_id], %w[namespace_id project_id]], + ['issuable_metric_images', 'IssuableMetricImage', %w[namespace_id]], + ['namespaces', 'Namespace', %w[id], %w[namespace_id]], + ['organization_details', 'Organizations::OrganizationDetail', %w[organization_id], nil, 'organization_id'], + ['project_relation_export_uploads', 'Projects::ImportExport::RelationExportUpload', %w[project_id]], + ['topics', 'Projects::Topic', %w[organization_id]], + ['projects', 'Project', %w[id], %w[project_id]], + ['snippets', 'Snippet', %w[organization_id]], + ['user_permission_export_uploads', 'UserPermissionExportUpload', %w[]], + ['users', 'User', %w[]], + # Sec tables + ['dependency_list_exports', 'Dependencies::DependencyListExport', %w[organization_id group_id project_id], + %w[organization_id namespace_id project_id], nil, :sec], + ['dependency_list_export_parts', 'Dependencies::DependencyListExport::Part', %w[organization_id], nil, nil, + :sec], + ['vulnerability_exports', 'Vulnerabilities::Export', %w[organization_id], nil, nil, :sec], + ['vulnerability_export_parts', 'Vulnerabilities::Export::Part', %w[organization_id], nil, nil, :sec], + ['vulnerability_remediations', 'Vulnerabilities::Remediation', %w[project_id], nil, nil, :sec], + ['vulnerability_archive_exports', 'Vulnerabilities::ArchiveExport', %w[project_id], nil, nil, :sec] + ] + end + + # Back-fill partitioned table `uploads_9ba88c4165`. For each sub-batch execute an + # upsert query per model_type, joining with the respective model_table. + # This join will exclude uploads belonging to records that no longer exist. + # + # Arguments are: + # sub_batch - batch to operate on. + # model_table - table storing the parent model + # model_name - model class name + # source - columns to source the sharding key values from + # targets - sharding key columns to back-fill + # join_key - column to join with the model table, defaults to id + # db_name - database the model table belongs to + def process_upload_type(sub_batch, model_table, model_name, sources, targets, join_key, db_name) + relation = sub_batch.select(:id, :model_type).limit(sub_batch_size) + targets ||= sources + join_key ||= 'id' + # Columns that will be reset (nullified) as they are not used for sharding keys + reset_columns = sharding_key_columns - targets + # All columns to back-fill + target_columns = (columns + targets + reset_columns).join(', ') + # All columns to source from + source_columns = source_columns_sql(sources, reset_columns) + # For existing records update only sharding key columns (if any) + on_conflict = if targets.any? + "UPDATE SET #{sharding_key_columns.map { |c| "#{c} = EXCLUDED.#{c}" }.join(', ')}" + else + "NOTHING" + end + + # For models stored in the Sec database we need to first fetch the values needed, + # and add them to the upsert as CTE + model_values_cte = "" + if db_name == :sec + sec_cte = sec_model_values_cte(sub_batch, model_name, join_key, sources, model_table) + return unless sec_cte + + source_columns = source_columns_sql(sources, reset_columns, nullif: true) + model_values_cte = sec_cte + end + + upsert = <<~SQL + WITH relation AS MATERIALIZED (#{relation.to_sql}), + filtered_relation AS MATERIALIZED ( + SELECT id FROM relation WHERE model_type = '#{model_name}' LIMIT #{sub_batch_size} + ) + #{model_values_cte} + INSERT INTO uploads_9ba88c4165 (#{target_columns}) + SELECT #{source_columns} FROM uploads + JOIN #{model_table} AS model ON uploads.model_id = model.#{join_key} + WHERE uploads.id IN (SELECT id FROM filtered_relation) + ON CONFLICT ON CONSTRAINT uploads_9ba88c4165_pkey DO #{on_conflict} + SQL + + connection.execute(upsert) + end + + def source_columns_sql(sources, reset_columns, nullif: false) + ( + columns.map { |c| "uploads.#{c}" } + + # Convert -1 back to NULL using NULLIF + sources.map { |c| nullif ? "NULLIF(model.#{c}, -1)" : "model.#{c}" } + + reset_columns.map { 'NULL' } + ).join(', ') + end + + def sec_model_values_cte(sub_batch, model_name, join_key, sources, model_table) + model_ids = sub_batch.where(model_type: model_name).limit(sub_batch_size).pluck(:model_id) + return unless model_ids.any? + + columns = [join_key] + sources + + rows = ::SecApplicationRecord.connection.select_rows( + "SELECT #{columns.join(', ')} FROM #{model_table} WHERE #{join_key} IN (#{model_ids.join(', ')})" + ) + return unless rows.any? + + # replace NULL values with -1 to avoid PG::DatatypeMismatch errors + values = rows.map { |row| "(#{row.map { |v| v.presence || -1 }.join(', ')})" }.join(', ') + + ", #{model_table}(#{columns.join(', ')}) AS (VALUES #{values})" + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_partitioned_uploads_spec.rb b/spec/lib/gitlab/background_migration/backfill_partitioned_uploads_spec.rb new file mode 100644 index 00000000000000..e211512a5fa48d --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_partitioned_uploads_spec.rb @@ -0,0 +1,704 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/MultipleMemoizedHelpers -- Polymorphic table used by many models requires complex setup +RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedUploads, :aggregate_failures, + :migration_with_transaction, + feature_category: :database do + let(:uploads_table) { table(:uploads) } + let(:partitioned_uploads_table) { table(:uploads_9ba88c4165) } + + let(:organizations) { table(:organizations) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:abuse_reports) { table(:abuse_reports) } + let(:achievements) { table(:achievements) } + let(:ai_vectorizable_files) { table(:ai_vectorizable_files) } + let(:alert_management_alerts) { table(:alert_management_alerts) } + let(:alert_management_alert_metric_images) { table(:alert_management_alert_metric_images) } + let(:appearances) { table(:appearances) } + let(:dependency_list_exports) { table(:dependency_list_exports, database: :sec) } + let(:import_export_uploads) { table(:import_export_uploads) } + let(:issuable_metric_images) { table(:issuable_metric_images) } + let(:organization_details) { table(:organization_details) } + let(:topics) { table(:topics) } + let(:snippets) { table(:snippets) } + let(:user_permission_export_uploads) { table(:user_permission_export_uploads) } + let(:dependency_list_export_parts) { table(:dependency_list_export_parts, database: :sec) } + let(:vulnerability_exports) { table(:vulnerability_exports, database: :sec) } + let(:vulnerability_export_parts) { table(:vulnerability_export_parts, database: :sec) } + let(:vulnerability_remediations) { table(:vulnerability_remediations, database: :sec) } + let(:vulnerability_archive_exports) { table(:vulnerability_archive_exports, database: :sec) } + let(:project_export_jobs) { table(:project_export_jobs) } + let(:project_relation_exports) { table(:project_relation_exports) } + let(:project_relation_export_uploads) { table(:project_relation_export_uploads) } + let(:design_management_designs) { table(:design_management_designs) } + let(:design_management_versions) { table(:design_management_versions) } + let(:design_management_designs_versions) { table(:design_management_designs_versions) } + let(:bulk_import_exports) { table(:bulk_import_exports) } + let(:bulk_import_export_uploads) { table(:bulk_import_export_uploads) } + + let(:connection) { ApplicationRecord.connection } + + describe '#perform' do + before do + # Partitioned table vulnerability_archive_exports is registered only for GitLab EE, + # when running tests for FOSS_ONLY we need at least one partition to be able to + # create the parent record. + SecApplicationRecord.connection.execute <<~SQL + CREATE TABLE _test_vulnerability_archive_exports_default + PARTITION OF vulnerability_archive_exports DEFAULT + SQL + end + + it 'backfill missing uploads' do + # Uploads to be_truthy copied + abuse_report_upload_to_be_synced = create_abuse_report_upload + appearance_upload_to_be_synced = create_appearance_upload + achievement_1 = create_achievement + achievement_upload_to_be_synced = create_achievement_upload(model: achievement_1) + ai_vectorizable_file_1 = create_ai_vectorizable_file + ai_vectorizable_file_upload_to_be_synced = create_ai_vectorizable_file_upload(model: ai_vectorizable_file_1) + alert_metric_image_1 = create_alert_metric_image + alert_metric_image_upload_to_be_synced = create_alert_metric_image_upload(model: alert_metric_image_1) + dependency_list_export_1 = create_dependency_list_export + dependency_list_export_upload_to_be_synced = create_dependency_list_export_upload(model: dependency_list_export_1) + dependency_list_export_part_1 = create_dependency_list_export_part + dependency_list_export_part_upload_to_be_synced = create_dependency_list_export_part_upload( + model: dependency_list_export_part_1) + import_export_upload_1 = create_import_export_upload + import_export_upload_upload_to_be_synced = create_import_export_upload_upload(model: import_export_upload_1) + issuable_metric_image_1 = create_issuable_metric_image + issuable_metric_image_upload_to_be_synced = create_issuable_metric_image_upload(model: issuable_metric_image_1) + organization_detail_1 = create_organization_detail + organization_detail_upload_to_be_synced = create_organization_detail_upload(model: organization_detail_1) + topic_1 = create_topic + topic_upload_to_be_synced = create_topic_upload(model: topic_1) + snippet_1 = create_snippet + snippet_upload_to_be_synced = create_snippet_upload(model: snippet_1) + user_permission_export_upload_1 = create_user_permission_export_upload + user_permission_export_upload_upload_to_be_synced = create_user_permission_export_upload_upload( + model: user_permission_export_upload_1) + user_1 = create_user + user_upload_to_be_synced = create_user_upload(model: user_1) + vulnerability_export_1 = create_vulnerability_export + vulnerability_export_upload_to_be_synced = create_vulnerability_export_upload(model: vulnerability_export_1) + vulnerability_export_part_1 = create_vulnerability_export_part + vulnerability_export_part_upload_to_be_synced = create_vulnerability_export_part_upload( + model: vulnerability_export_part_1) + vulnerability_remediation_1 = create_vulnerability_remediation + vulnerability_remediation_upload_to_be_synced = create_vulnerability_remediation_upload( + model: vulnerability_remediation_1) + vulnerability_archive_export_1 = create_vulnerability_archive_export + vulnerability_archive_export_upload_to_be_synced = create_vulnerability_archive_export_upload( + model: vulnerability_archive_export_1) + project_1 = create_project + project_upload_to_be_synced = create_project_upload(model: project_1) + namespace_1 = create_namespace + namespace_upload_to_be_synced = create_namespace_upload(model: namespace_1) + project_relation_export_upload_1 = create_project_relation_export_upload + project_relation_export_upload_upload_to_be_synced = create_project_relation_export_upload_upload( + model: project_relation_export_upload_1) + designs_version_1 = create_designs_version + designs_version_upload_to_be_synced = create_designs_version_upload(model: designs_version_1) + bulk_import_export_upload_1 = create_bulk_import_export_upload + bulk_import_export_upload_upload_to_be_synced = create_bulk_import_export_upload_upload( + model: bulk_import_export_upload_1) + + # Parentless uploads not to be back-filled + abuse_report_upload_to_be_removed = create_abuse_report_upload(delete_model: true) + appearance_upload_to_be_removed = create_appearance_upload(delete_model: true) + achievement_upload_to_be_removed = create_achievement_upload(delete_model: true) + ai_vectorizable_file_upload_to_be_removed = create_ai_vectorizable_file_upload(delete_model: true) + alert_metric_image_upload_to_be_removed = create_alert_metric_image_upload(delete_model: true) + dependency_list_export_upload_to_be_removed = create_dependency_list_export_upload(delete_model: true) + dependency_list_export_part_upload_to_be_removed = create_dependency_list_export_part_upload(delete_model: true) + import_export_upload_upload_to_be_removed = create_import_export_upload_upload(delete_model: true) + issuable_metric_image_upload_to_be_removed = create_issuable_metric_image_upload(delete_model: true) + organization_detail_upload_to_be_removed = create_organization_detail_upload(delete_model: true) + topic_upload_to_be_removed = create_topic_upload(delete_model: true) + snippet_upload_to_be_removed = create_snippet_upload(delete_model: true) + user_permission_export_upload_upload_to_be_removed = create_user_permission_export_upload_upload( + delete_model: true) + user_upload_to_be_removed = create_user_upload(delete_model: true) + vulnerability_export_upload_to_be_removed = create_vulnerability_export_upload(delete_model: true) + vulnerability_export_part_upload_to_be_removed = create_vulnerability_export_part_upload(delete_model: true) + vulnerability_remediation_upload_to_be_removed = create_vulnerability_remediation_upload(delete_model: true) + vulnerability_archive_export_upload_to_be_removed = create_vulnerability_archive_export_upload(delete_model: true) + project_upload_to_be_removed = create_project_upload(delete_model: true) + namespace_upload_to_be_removed = create_namespace_upload(delete_model: true) + project_relation_export_upload_upload_to_be_removed = create_project_relation_export_upload_upload( + delete_model: true) + designs_version_upload_to_be_removed = create_designs_version_upload(delete_model: true) + bulk_import_export_upload_upload_to_be_removed = create_bulk_import_export_upload_upload(delete_model: true) + + connection.truncate(partitioned_uploads_table.table_name) + + # Uploads already copied + abuse_report_upload_synced = create_abuse_report_upload + appearance_upload_synced = create_appearance_upload + achievement_2 = create_achievement + achievement_upload_synced = create_achievement_upload(model: achievement_2) + ai_vectorizable_file_2 = create_ai_vectorizable_file + ai_vectorizable_file_upload_synced = create_ai_vectorizable_file_upload(model: ai_vectorizable_file_2) + alert_metric_image_2 = create_alert_metric_image + alert_metric_image_upload_synced = create_alert_metric_image_upload(model: alert_metric_image_2) + dependency_list_export_2 = create_dependency_list_export + dependency_list_export_upload_synced = create_dependency_list_export_upload(model: dependency_list_export_2) + dependency_list_export_part_2 = create_dependency_list_export_part + dependency_list_export_part_upload_synced = create_dependency_list_export_part_upload( + model: dependency_list_export_part_2) + import_export_upload_2 = create_import_export_upload + import_export_upload_upload_synced = create_import_export_upload_upload(model: import_export_upload_2) + issuable_metric_image_2 = create_issuable_metric_image + issuable_metric_image_upload_synced = create_issuable_metric_image_upload(model: issuable_metric_image_2) + organization_detail_2 = create_organization_detail + organization_detail_upload_synced = create_organization_detail_upload(model: organization_detail_2) + topic_2 = create_topic + topic_upload_synced = create_topic_upload(model: topic_2) + snippet_2 = create_snippet + snippet_upload_synced = create_snippet_upload(model: snippet_2) + user_permission_export_upload_2 = create_user_permission_export_upload + user_permission_export_upload_upload_synced = create_user_permission_export_upload_upload( + model: user_permission_export_upload_2) + user_2 = create_user + user_upload_synced = create_user_upload(model: user_2) + vulnerability_export_2 = create_vulnerability_export + vulnerability_export_upload_synced = create_vulnerability_export_upload(model: vulnerability_export_2) + vulnerability_export_part_2 = create_vulnerability_export_part + vulnerability_export_part_upload_synced = create_vulnerability_export_part_upload( + model: vulnerability_export_part_2) + vulnerability_remediation_2 = create_vulnerability_remediation + vulnerability_remediation_upload_synced = create_vulnerability_remediation_upload( + model: vulnerability_remediation_2) + vulnerability_archive_export_2 = create_vulnerability_archive_export + vulnerability_archive_export_upload_synced = create_vulnerability_archive_export_upload( + model: vulnerability_archive_export_2) + project_2 = create_project + project_upload_synced = create_project_upload(model: project_2) + namespace_2 = create_namespace + namespace_upload_synced = create_namespace_upload(model: namespace_2) + project_relation_export_upload_2 = create_project_relation_export_upload + project_relation_export_upload_upload_synced = create_project_relation_export_upload_upload( + model: project_relation_export_upload_2) + designs_version_2 = create_designs_version + designs_version_upload_synced = create_designs_version_upload(model: designs_version_2) + bulk_import_export_upload_2 = create_bulk_import_export_upload + bulk_import_export_upload_upload_synced = create_bulk_import_export_upload_upload( + model: bulk_import_export_upload_2) + + expect(uploads_table.find_by_id(abuse_report_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(abuse_report_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(abuse_report_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(appearance_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(appearance_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(appearance_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(achievement_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(achievement_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(achievement_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(ai_vectorizable_file_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(ai_vectorizable_file_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(ai_vectorizable_file_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(alert_metric_image_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(alert_metric_image_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(alert_metric_image_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(dependency_list_export_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(dependency_list_export_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(dependency_list_export_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(dependency_list_export_part_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(dependency_list_export_part_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(dependency_list_export_part_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(import_export_upload_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(import_export_upload_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(import_export_upload_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(issuable_metric_image_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(issuable_metric_image_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(issuable_metric_image_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(organization_detail_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(organization_detail_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(organization_detail_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(snippet_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(snippet_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(snippet_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(user_permission_export_upload_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(user_permission_export_upload_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(user_permission_export_upload_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(user_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(user_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(user_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(vulnerability_export_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(vulnerability_export_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(vulnerability_export_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(vulnerability_export_part_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(vulnerability_export_part_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(vulnerability_export_part_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(vulnerability_remediation_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(vulnerability_remediation_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(vulnerability_remediation_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(vulnerability_archive_export_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(vulnerability_archive_export_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(vulnerability_archive_export_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(project_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(project_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(project_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(namespace_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(namespace_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(namespace_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(project_relation_export_upload_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(project_relation_export_upload_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(project_relation_export_upload_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(designs_version_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(designs_version_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(designs_version_upload_synced.id)).to be_truthy + + expect(uploads_table.find_by_id(bulk_import_export_upload_upload_to_be_removed.id)).to be_truthy + expect(find_partitioned_upload(bulk_import_export_upload_upload_to_be_removed.id)).not_to be_truthy + expect(find_partitioned_upload(bulk_import_export_upload_upload_synced.id)).to be_truthy + + expect do + described_class.new( + start_id: uploads_table.minimum(:id), + end_id: uploads_table.maximum(:id), + batch_table: :uploads, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: connection + ).perform + end.not_to raise_error + + expect(find_partitioned_upload(abuse_report_upload_to_be_synced.id)).to be_truthy + expect(find_partitioned_upload(abuse_report_upload_synced.id)).to be_truthy + expect(find_partitioned_upload(abuse_report_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(appearance_upload_to_be_synced.id)).to be_truthy + expect(find_partitioned_upload(appearance_upload_synced.id)).to be_truthy + expect(find_partitioned_upload(appearance_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(achievement_upload_to_be_synced.id).namespace_id) + .to eq(achievement_1.namespace_id) + expect(find_partitioned_upload(achievement_upload_synced.id).namespace_id) + .to eq(achievement_2.namespace_id) + expect(find_partitioned_upload(achievement_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(ai_vectorizable_file_upload_to_be_synced.id).project_id) + .to eq(ai_vectorizable_file_1.project_id) + expect(find_partitioned_upload(ai_vectorizable_file_upload_synced.id).project_id) + .to eq(ai_vectorizable_file_2.project_id) + expect(find_partitioned_upload(ai_vectorizable_file_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(alert_metric_image_upload_to_be_synced.id).project_id) + .to eq(alert_metric_image_1.project_id) + expect(find_partitioned_upload(alert_metric_image_upload_synced.id).project_id) + .to eq(alert_metric_image_2.project_id) + expect(find_partitioned_upload(alert_metric_image_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(dependency_list_export_upload_to_be_synced.id).namespace_id) + .to eq(dependency_list_export_1.group_id) + expect(find_partitioned_upload(dependency_list_export_upload_synced.id).namespace_id) + .to eq(dependency_list_export_2.group_id) + expect(find_partitioned_upload(dependency_list_export_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(dependency_list_export_part_upload_to_be_synced.id).organization_id) + .to eq(dependency_list_export_part_1.organization_id) + expect(find_partitioned_upload(dependency_list_export_part_upload_synced.id).organization_id) + .to eq(dependency_list_export_part_2.organization_id) + expect(find_partitioned_upload(dependency_list_export_part_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(import_export_upload_upload_to_be_synced.id).namespace_id) + .to eq(import_export_upload_1.group_id) + expect(find_partitioned_upload(import_export_upload_upload_synced.id).namespace_id) + .to eq(import_export_upload_2.group_id) + expect(find_partitioned_upload(import_export_upload_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(issuable_metric_image_upload_to_be_synced.id).namespace_id) + .to eq(issuable_metric_image_1.namespace_id) + expect(find_partitioned_upload(issuable_metric_image_upload_synced.id).namespace_id) + .to eq(issuable_metric_image_2.namespace_id) + expect(find_partitioned_upload(issuable_metric_image_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(organization_detail_upload_to_be_synced.id).organization_id) + .to eq(organization_detail_1.organization_id) + expect(find_partitioned_upload(organization_detail_upload_synced.id).organization_id) + .to eq(organization_detail_2.organization_id) + expect(find_partitioned_upload(organization_detail_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(topic_upload_to_be_synced.id).organization_id).to eq(topic_1.organization_id) + expect(find_partitioned_upload(topic_upload_synced.id).organization_id).to eq(topic_2.organization_id) + expect(find_partitioned_upload(topic_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(snippet_upload_to_be_synced.id).organization_id).to eq(snippet_1.organization_id) + expect(find_partitioned_upload(snippet_upload_synced.id).organization_id).to eq(snippet_2.organization_id) + expect(find_partitioned_upload(snippet_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(user_permission_export_upload_upload_to_be_synced.id)).to be_truthy + expect(find_partitioned_upload(user_permission_export_upload_upload_synced.id)).to be_truthy + expect(find_partitioned_upload(user_permission_export_upload_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(user_upload_to_be_synced.id)).to be_truthy + expect(find_partitioned_upload(user_upload_synced.id)).to be_truthy + expect(find_partitioned_upload(user_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(vulnerability_export_upload_to_be_synced.id).organization_id) + .to eq(vulnerability_export_1.organization_id) + expect(find_partitioned_upload(vulnerability_export_upload_synced.id).organization_id) + .to eq(vulnerability_export_2.organization_id) + expect(find_partitioned_upload(vulnerability_export_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(vulnerability_export_part_upload_to_be_synced.id).organization_id) + .to eq(vulnerability_export_part_1.organization_id) + expect(find_partitioned_upload(vulnerability_export_part_upload_synced.id).organization_id) + .to eq(vulnerability_export_part_2.organization_id) + expect(find_partitioned_upload(vulnerability_export_part_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(vulnerability_remediation_upload_to_be_synced.id).project_id) + .to eq(vulnerability_remediation_1.project_id) + expect(find_partitioned_upload(vulnerability_remediation_upload_synced.id).project_id) + .to eq(vulnerability_remediation_2.project_id) + expect(find_partitioned_upload(vulnerability_remediation_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(vulnerability_archive_export_upload_to_be_synced.id).project_id) + .to eq(vulnerability_archive_export_1.project_id) + expect(find_partitioned_upload(vulnerability_archive_export_upload_synced.id).project_id) + .to eq(vulnerability_archive_export_2.project_id) + expect(find_partitioned_upload(vulnerability_archive_export_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(project_upload_to_be_synced.id).project_id).to eq(project_1.id) + expect(find_partitioned_upload(project_upload_synced.id).project_id).to eq(project_2.id) + expect(find_partitioned_upload(project_upload_to_be_synced.id).namespace_id).to be_nil + expect(find_partitioned_upload(project_upload_synced.id).namespace_id).to be_nil + expect(find_partitioned_upload(project_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(namespace_upload_to_be_synced.id).namespace_id).to eq(namespace_1.id) + expect(find_partitioned_upload(namespace_upload_synced.id).namespace_id).to eq(namespace_2.id) + expect(find_partitioned_upload(namespace_upload_to_be_synced.id).organization_id).to be_nil + expect(find_partitioned_upload(namespace_upload_synced.id).organization_id).to be_nil + expect(find_partitioned_upload(namespace_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(project_relation_export_upload_upload_to_be_synced.id).project_id) + .to eq(project_relation_export_upload_1.project_id) + expect(find_partitioned_upload(project_relation_export_upload_upload_synced.id).project_id) + .to eq(project_relation_export_upload_2.project_id) + expect(find_partitioned_upload(project_relation_export_upload_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(designs_version_upload_to_be_synced.id).namespace_id) + .to eq(designs_version_1.namespace_id) + expect(find_partitioned_upload(designs_version_upload_synced.id).namespace_id) + .to eq(designs_version_2.namespace_id) + expect(find_partitioned_upload(designs_version_upload_to_be_removed.id)).not_to be_truthy + + expect(find_partitioned_upload(bulk_import_export_upload_upload_to_be_synced.id).namespace_id) + .to eq(bulk_import_export_upload_1.group_id) + expect(find_partitioned_upload(bulk_import_export_upload_upload_synced.id).namespace_id) + .to eq(bulk_import_export_upload_2.group_id) + expect(find_partitioned_upload(bulk_import_export_upload_upload_to_be_removed.id)).not_to be_truthy + end + end + + def find_partitioned_upload(id) + partitioned_uploads_table.find_by_id(id) + end + + def create_organization + suffix = SecureRandom.base64 + organizations.create!(name: 'Organization', path: "organization-#{suffix}") + end + + def create_namespace + organization = create_organization + namespaces.create!( + name: 'gitlab-org', + path: 'gitlab-org', + type: 'Group', + organization_id: organization.id + ) + end + + def create_project + namespace = create_namespace + projects.create!( + namespace_id: namespace.id, + organization_id: namespace.organization_id, + project_namespace_id: namespace.id, + name: 'Project', + path: 'project' + ) + end + + def create_user + users.create!(username: SecureRandom.base64, email: "#{SecureRandom.base64}@gitlab.com", projects_limit: 1) + end + + def create_upload(model_type, model, delete_model: false) + model_id = Array.wrap(model.id).first + uploads_table.create!(model_type: model_type, model_id: model_id, size: 42, path: '/some/path', + uploader: 'FileUploader', created_at: Time.current).tap do + model.delete if delete_model + end + end + + def create_achievement + namespace = create_namespace + achievements.create!(namespace_id: namespace.id, name: SecureRandom.base64) + end + + def create_abuse_report_upload(delete_model: false) + model = abuse_reports.create! + create_upload('AbuseReport', model, delete_model: delete_model) + end + + def create_appearance_upload(delete_model: false) + model = appearances.create!(title: 'foo', description: 'bar') + create_upload('Appearance', model, delete_model: delete_model) + end + + def create_achievement_upload(model: nil, delete_model: false) + model ||= create_achievement + create_upload('Achievements::Achievement', model, delete_model: delete_model) + end + + def create_ai_vectorizable_file + project = create_project + ai_vectorizable_files.create!(project_id: project.id, name: 'ai_file', file: 'ai_file') + end + + def create_ai_vectorizable_file_upload(model: nil, delete_model: false) + model ||= create_ai_vectorizable_file + create_upload('Ai::VectorizableFile', model, delete_model: delete_model) + end + + def create_alert + project = create_project + alert_management_alerts.create!(started_at: Time.current, iid: 1, project_id: project.id, title: 'High Alert!') + end + + def create_alert_metric_image + alert = create_alert + alert_management_alert_metric_images.create!(alert_id: alert.id, project_id: alert.project_id, file: 'alert file') + end + + def create_alert_metric_image_upload(model: nil, delete_model: false) + model ||= create_alert_metric_image + create_upload('AlertManagement::MetricImage', model, delete_model: delete_model) + end + + def create_dependency_list_export + group = create_namespace + dependency_list_exports.create!(group_id: group.id) + end + + def create_dependency_list_export_upload(model: nil, delete_model: false) + model ||= create_dependency_list_export + create_upload('Dependencies::DependencyListExport', model, delete_model: delete_model) + end + + def create_dependency_list_export_part + organization = create_organization + dependency_list_export = create_dependency_list_export + dependency_list_export_parts.create!(organization_id: organization.id, + dependency_list_export_id: dependency_list_export.id, start_id: 1, end_id: 9) + end + + def create_dependency_list_export_part_upload(model: nil, delete_model: false) + model ||= create_dependency_list_export_part + create_upload('Dependencies::DependencyListExport::Part', model, delete_model: delete_model) + end + + def create_import_export_upload + group = create_namespace + import_export_uploads.create!(group_id: group.id) + end + + def create_import_export_upload_upload(model: nil, delete_model: false) + model ||= create_import_export_upload + create_upload('ImportExportUpload', model, delete_model: delete_model) + end + + def create_issuable_metric_image + namespace = create_namespace + issue_work_item_type_id = table(:work_item_types).find_by(name: 'Issue').id + issue = table(:issues).create!( + namespace_id: namespace.id, + lock_version: 1, + work_item_type_id: issue_work_item_type_id + ) + issuable_metric_images.create!(namespace_id: namespace.id, issue_id: issue.id, file: 'some_file') + end + + def create_issuable_metric_image_upload(model: nil, delete_model: false) + model ||= create_import_export_upload + create_upload('IssuableMetricImage', model, delete_model: delete_model) + end + + def create_organization_detail + organization = create_organization + organization_details.create!(organization_id: organization.id) + end + + def create_organization_detail_upload(model: nil, delete_model: false) + model ||= create_organization_detail + create_upload('Organizations::OrganizationDetail', model, delete_model: delete_model) + end + + def create_topic + organization = create_organization + topics.create!(organization_id: organization.id, name: SecureRandom.base64) + end + + def create_topic_upload(model: nil, delete_model: false) + model ||= create_topic + create_upload('Projects::Topic', model, delete_model: delete_model) + end + + def create_snippet + organization = create_organization + user = create_user + snippets.create!(organization_id: organization.id, author_id: user.id) + end + + def create_snippet_upload(model: nil, delete_model: false) + model ||= create_snippet + create_upload('Snippet', model, delete_model: delete_model) + end + + def create_user_permission_export_upload + user = create_user + user_permission_export_uploads.create!(user_id: user.id) + end + + def create_user_permission_export_upload_upload(model: nil, delete_model: false) + model ||= create_user_permission_export_upload + create_upload('UserPermissionExportUpload', model, delete_model: delete_model) + end + + def create_user_upload(model: nil, delete_model: false) + model ||= create_user + create_upload('User', model, delete_model: delete_model) + end + + def create_vulnerability_export + organization = create_organization + user = create_user + vulnerability_exports.create!(organization_id: organization.id, author_id: user.id, status: 'open') + end + + def create_vulnerability_export_upload(model: nil, delete_model: false) + model ||= create_vulnerability_export + create_upload('Vulnerabilities::Export', model, delete_model: delete_model) + end + + def create_vulnerability_export_part + organization = create_organization + vulnerability_export = create_vulnerability_export + vulnerability_export_parts.create!(organization_id: organization.id, + vulnerability_export_id: vulnerability_export.id, start_id: 1, end_id: 100) + end + + def create_vulnerability_export_part_upload(model: nil, delete_model: false) + model ||= create_vulnerability_export_part + create_upload('Vulnerabilities::Export::Part', model, delete_model: delete_model) + end + + def create_vulnerability_remediation + project = create_project + vulnerability_remediations.create!(project_id: project.id, summary: 'summary', file: 'some_file', checksum: '123') + end + + def create_vulnerability_remediation_upload(model: nil, delete_model: false) + model ||= create_vulnerability_remediation + create_upload('Vulnerabilities::Remediation', model, delete_model: delete_model) + end + + def create_vulnerability_archive_export + project = create_project + user = create_user + vulnerability_archive_exports.create!(project_id: project.id, author_id: user.id, + date_range: Time.current.yesterday..Time.current) + end + + def create_vulnerability_archive_export_upload(model: nil, delete_model: false) + model ||= create_vulnerability_archive_export + create_upload('Vulnerabilities::ArchiveExport', model, delete_model: delete_model) + end + + def create_project_upload(model: nil, delete_model: false) + model ||= create_project + create_upload('Project', model, delete_model: delete_model).tap do |u| + u.update!(namespace_id: model.namespace_id) + end + end + + def create_namespace_upload(model: nil, delete_model: false) + model ||= create_namespace + create_upload('Namespace', model, delete_model: delete_model).tap do |u| + u.update!(organization_id: model.organization_id) + end + end + + def create_project_relation_export_upload + project = create_project + project_export_job = project_export_jobs.create!(project_id: project.id, jid: SecureRandom.base64) + project_relation_export = project_relation_exports.create!(project_id: project.id, + project_export_job_id: project_export_job.id, relation: 'rel') + project_relation_export_uploads.create!(project_id: project.id, + project_relation_export_id: project_relation_export.id, export_file: 'export.file') + end + + def create_project_relation_export_upload_upload(model: nil, delete_model: false) + model ||= create_project_relation_export_upload + create_upload('Projects::ImportExport::RelationExportUpload', model, delete_model: delete_model) + end + + def create_designs_version + namespace = create_namespace + project = create_project + design = design_management_designs.create!(project_id: project.id, namespace_id: namespace.id, + filename: 'file.name', iid: 1) + version = design_management_versions.create!(namespace_id: namespace.id, sha: SecureRandom.base64) + + design_management_designs_versions.create!(namespace_id: namespace.id, design_id: design.id, version_id: version.id) + end + + def create_designs_version_upload(model: nil, delete_model: false) + model ||= create_designs_version + create_upload('DesignManagement::Action', model, delete_model: delete_model) + end + + def create_bulk_import_export + namespace = create_namespace + bulk_import_exports.create!(group_id: namespace.id, status: 1, relation: 'rel') + end + + def create_bulk_import_export_upload + export = create_bulk_import_export + bulk_import_export_uploads.create!(export_id: export.id, group_id: export.group_id) + end + + def create_bulk_import_export_upload_upload(model: nil, delete_model: false) + model ||= create_bulk_import_export_upload + create_upload('BulkImports::ExportUpload', model, delete_model: delete_model) + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/migrations/queue_backfill_partitioned_uploads_spec.rb b/spec/migrations/queue_backfill_partitioned_uploads_spec.rb new file mode 100644 index 00000000000000..5b916d76a7422a --- /dev/null +++ b/spec/migrations/queue_backfill_partitioned_uploads_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillPartitionedUploads, migration: :gitlab_main, feature_category: :database do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :uploads, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/models/bulk_imports/export_upload_spec.rb b/spec/models/bulk_imports/export_upload_spec.rb index a2a9ac8c2f9dca..bd5258344e63a9 100644 --- a/spec/models/bulk_imports/export_upload_spec.rb +++ b/spec/models/bulk_imports/export_upload_spec.rb @@ -39,6 +39,9 @@ def find_callback(callbacks, key) describe '#uploads_sharding_key' do it 'returns project_id or group_id' do + bulk_import_export.save! + bulk_import_export.reload + expect(bulk_import_export.uploads_sharding_key).to eq( project_id: export.project_id, namespace_id: export.group_id diff --git a/spec/models/design_management/action_spec.rb b/spec/models/design_management/action_spec.rb index 8ce67adf1a9195..ae046ffe113078 100644 --- a/spec/models/design_management/action_spec.rb +++ b/spec/models/design_management/action_spec.rb @@ -133,8 +133,7 @@ describe '#uploads_sharding_key' do it 'returns namespace_id' do namespace = build_stubbed(:namespace) - design = build_stubbed(:design, namespace_id: namespace.id) - design_action = build_stubbed(:design_action, design: design) + design_action = build_stubbed(:design_action, namespace_id: namespace.id) expect(design_action.uploads_sharding_key).to eq(namespace_id: namespace.id) end diff --git a/spec/models/projects/import_export/relation_export_upload_spec.rb b/spec/models/projects/import_export/relation_export_upload_spec.rb index 89b29f97a9cb4e..e6f38f41b45f4e 100644 --- a/spec/models/projects/import_export/relation_export_upload_spec.rb +++ b/spec/models/projects/import_export/relation_export_upload_spec.rb @@ -79,8 +79,7 @@ def find_callback(callbacks, key) describe '#uploads_sharding_key' do it 'returns project_id' do project = build_stubbed(:project) - export = build_stubbed(:project_relation_export, project_id: project.id) - export_upload = build_stubbed(:relation_export_upload, relation_export: export) + export_upload = build_stubbed(:relation_export_upload, project_id: project.id) expect(export_upload.uploads_sharding_key).to eq(project_id: project.id) end -- GitLab