diff --git a/changelogs/unreleased/212322-update-existing-vulnerabilities-occurrences-uuids.yml b/changelogs/unreleased/212322-update-existing-vulnerabilities-occurrences-uuids.yml new file mode 100644 index 0000000000000000000000000000000000000000..9df5ee1920e545dbd5b38bbf50352537c02a5469 --- /dev/null +++ b/changelogs/unreleased/212322-update-existing-vulnerabilities-occurrences-uuids.yml @@ -0,0 +1,5 @@ +--- +title: Recalculate UUID for all Vulnerability::Findings +merge_request: 47529 +author: +type: other diff --git a/db/post_migrate/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb b/db/post_migrate/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb new file mode 100644 index 0000000000000000000000000000000000000000..7bb97a39bda4c5d93fd59431a72ecfe45b0376b8 --- /dev/null +++ b/db/post_migrate/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid' + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 2_500 + + disable_ddl_transaction! + + class VulnerabilitiesFinding < ActiveRecord::Base + include ::EachBatch + + self.table_name = "vulnerability_occurrences" + end + + def up + # Make sure that RemoveDuplicateVulnerabilitiesFindings has finished running + # so that we don't run into duplicate UUID issues + Gitlab::BackgroundMigration.steal('RemoveDuplicateVulnerabilitiesFindings') + + say "Scheduling #{MIGRATION} jobs" + queue_background_migration_jobs_by_range_at_intervals( + VulnerabilitiesFinding, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20201112130715 b/db/schema_migrations/20201112130715 new file mode 100644 index 0000000000000000000000000000000000000000..b241214e15e24189412ff1a033c6d55293547529 --- /dev/null +++ b/db/schema_migrations/20201112130715 @@ -0,0 +1 @@ +fc821ffb2e2298bfdf3555bb255e2300f3722141cadc8f89e62646fd6f22faf0 \ No newline at end of file diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb new file mode 100644 index 0000000000000000000000000000000000000000..7b18e617c812fcc31c997d3fc49d8319b015bff5 --- /dev/null +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid + # rubocop: disable Gitlab/NamespacedClass + class VulnerabilitiesIdentifier < ActiveRecord::Base + self.table_name = "vulnerability_identifiers" + has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + REPORT_TYPES = { + sast: 0, + dependency_scanning: 1, + container_scanning: 2, + dast: 3, + secret_detection: 4, + coverage_fuzzing: 5, + api_fuzzing: 6 + }.with_indifferent_access.freeze + enum report_type: REPORT_TYPES + end + + class CalculateFindingUUID + FINDING_NAMESPACES_IDS = { + development: "a143e9e2-41b3-47bc-9a19-081d089229f4", + test: "a143e9e2-41b3-47bc-9a19-081d089229f4", + staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + }.freeze + + NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze + PACK_PATTERN = "NnnnnN".freeze + + def self.call(value) + Digest::UUID.uuid_v5(namespace_id, value) + end + + def self.namespace_id + namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) + # Digest::UUID is broken when using an UUID in namespace_id + # https://github.com/rails/rails/issues/37681#issue-520718028 + namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) + end + end + # rubocop: enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + findings = VulnerabilitiesFinding + .joins(:primary_identifier) + .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) + .where(id: start_id..end_id) + + mappings = findings.each_with_object({}) do |finding, hash| + hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) } + end + + ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + end + + private + + def calculate_uuid_v5_for_finding(vulnerability_finding) + return unless vulnerability_finding + + uuid_v5_name_components = { + report_type: vulnerability_finding.report_type, + primary_identifier_fingerprint: vulnerability_finding.fingerprint, + location_fingerprint: vulnerability_finding.location_fingerprint, + project_id: vulnerability_finding.project_id + } + + name = uuid_v5_name_components.values.join('-') + + CalculateFindingUUID.call(name) + end +end diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..990ef4fbe6a8a3143af099bcb1203e4e2ff1719e --- /dev/null +++ b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, schema: 20201110110454 do + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:users) { table(:users) } + let(:user) { create_user! } + let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } + let(:scanners) { table(:vulnerability_scanners) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v5', + external_id: 'uuid-v5', + fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + name: 'Identifier for UUIDv5') + end + + let(:different_vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v4', + external_id: 'uuid-v4', + fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89', + name: 'Identifier for UUIDv4') + end + + let!(:vulnerability_for_uuidv4) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:vulnerability_for_uuidv5) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let(:known_uuid_v5) { "77211ed6-7dff-5f6b-8c9a-da89ad0a9b60" } + let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } + let(:desired_uuid_v5) { "3ca8ad45-6344-508b-b5e3-306a3bd6c6ba" } + + subject { described_class.new.perform(finding.id, finding.id) } + + context "when finding has a UUIDv4" do + before do + @uuid_v4 = create_finding!( + vulnerability_id: vulnerability_for_uuidv4.id, + project_id: project.id, + scanner_id: different_scanner.id, + primary_identifier_id: different_vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e", + uuid: known_uuid_v4 + ) + end + + let(:finding) { @uuid_v4 } + + it "replaces it with UUIDv5" do + expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v4]) + + subject + + expect(vulnerabilities_findings.pluck(:uuid)).to eq([desired_uuid_v5]) + end + end + + context "when finding has a UUIDv5" do + before do + @uuid_v5 = create_finding!( + vulnerability_id: vulnerability_for_uuidv5.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a", + uuid: known_uuid_v5 + ) + end + + let(:finding) { @uuid_v5 } + + it "stays the same" do + expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v5]) + + subject + + expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v5]) + end + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + vulnerabilities_findings.create!( + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + ) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) + users.create!( + name: name, + email: email, + username: name, + projects_limit: 0, + user_type: user_type, + confirmed_at: confirmed_at + ) + end +end diff --git a/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..fce32be4683a35aeaec6a7cca9e6e6919893d7af --- /dev/null +++ b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb') + +RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences, :migration do + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:users) { table(:users) } + let(:user) { create_user! } + let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } + let(:scanners) { table(:vulnerability_scanners) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v5', + external_id: 'uuid-v5', + fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + name: 'Identifier for UUIDv5') + end + + let(:different_vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v4', + external_id: 'uuid-v4', + fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89', + name: 'Identifier for UUIDv4') + end + + let!(:vulnerability_for_uuidv4) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:vulnerability_for_uuidv5) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } + let!(:finding_with_uuid_v4) do + create_finding!( + vulnerability_id: vulnerability_for_uuidv4.id, + project_id: project.id, + scanner_id: different_scanner.id, + primary_identifier_id: different_vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e", + uuid: known_uuid_v4 + ) + end + + let(:known_uuid_v5) { "e7d3d99d-04bb-5771-bb44-d80a9702d0a2" } + let!(:finding_with_uuid_v5) do + create_finding!( + vulnerability_id: vulnerability_for_uuidv5.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a", + uuid: known_uuid_v5 + ) + end + + before do + stub_const("#{described_class}::BATCH_SIZE", 1) + end + + around do |example| + freeze_time { Sidekiq::Testing.fake! { example.run } } + end + + it 'schedules background migration' do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v4.id, finding_with_uuid_v4.id) + expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v5.id, finding_with_uuid_v5.id) + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + vulnerabilities_findings.create!( + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + ) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.now, confirmed_at: Time.now) + users.create!( + name: name, + email: email, + username: name, + projects_limit: 0, + user_type: user_type, + confirmed_at: confirmed_at + ) + end +end