diff --git a/changelogs/unreleased/213571-migrate-lm-to-ls-artifacts.yml b/changelogs/unreleased/213571-migrate-lm-to-ls-artifacts.yml new file mode 100644 index 0000000000000000000000000000000000000000..29e6668351973ba1062fa35e42577b4d537f127b --- /dev/null +++ b/changelogs/unreleased/213571-migrate-lm-to-ls-artifacts.yml @@ -0,0 +1,5 @@ +--- +title: Migrate license_management artifacts to license_scanning type +merge_request: 36817 +author: +type: changed diff --git a/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb b/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb new file mode 100644 index 0000000000000000000000000000000000000000..fce4ee095bde4e181a760d0a86e58bc6e18430d7 --- /dev/null +++ b/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexForLicenseComplianceArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_ci_job_artifacts_on_license_compliance_file_types' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, [:job_id, :file_type], where: 'file_type = 10 OR file_type = 101', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb new file mode 100644 index 0000000000000000000000000000000000000000..dcc8e4d11b262397562e7c517615ba8e34313d17 --- /dev/null +++ b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class MigrateLicenseManagementArtifactsToLicenseScanning < ActiveRecord::Migration[6.0] + DOWNTIME = false + LICENSE_MANAGEMENT_FILE_TYPE = 10 + LICENSE_SCANNING_FILE_TYPE = 101 + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_job_artifacts' + end + + # We're updating file_type of ci artifacts from license_management to license_scanning + # But before that we need to delete "rogue" artifacts for CI builds that have associated with them + # both license_scanning and license_management artifacts. It's an edge case and usually, we don't have + # such builds in the database. + def up + return unless Gitlab.ee? + + JobArtifact + .where("file_type = 10 OR file_type = 101") + .each_batch(column: :job_id, of: 1000) do |relation| + min, max = relation.pluck('MIN(job_id)', 'MAX(job_id)').flatten + + ActiveRecord::Base.connection.execute <<~SQL + WITH ci_job_artifacts_with_row_number as ( + SELECT job_id, id, ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY id ASC) as row_number + FROM ci_job_artifacts + WHERE (file_type = #{LICENSE_SCANNING_FILE_TYPE} OR file_type = #{LICENSE_MANAGEMENT_FILE_TYPE}) + AND job_id >= #{Integer(min)} AND job_id < #{Integer(max)} + ) + DELETE FROM ci_job_artifacts + WHERE ci_job_artifacts.id IN (SELECT id from ci_job_artifacts_with_row_number WHERE ci_job_artifacts_with_row_number.row_number > 1) + SQL + end + + JobArtifact.where(file_type: LICENSE_MANAGEMENT_FILE_TYPE).each_batch(of: 1000) do |relation| + relation.update_all(file_type: LICENSE_SCANNING_FILE_TYPE) + end + end + + def down + # no-op + # we're deleting duplicating artifacts and updating file_type for license_management artifacts + end +end diff --git a/db/schema_migrations/20200808221641 b/db/schema_migrations/20200808221641 new file mode 100644 index 0000000000000000000000000000000000000000..05e3769247e9f1a94a199ec0088c6b3ed9c79174 --- /dev/null +++ b/db/schema_migrations/20200808221641 @@ -0,0 +1 @@ +a24aa5052d37bff1bffc5df076d8422ea90f3781ae01ecf626bc59f3e299c90b \ No newline at end of file diff --git a/db/schema_migrations/20200809221641 b/db/schema_migrations/20200809221641 new file mode 100644 index 0000000000000000000000000000000000000000..4e43680663a4a467f2f40ab734681e677f22d64a --- /dev/null +++ b/db/schema_migrations/20200809221641 @@ -0,0 +1 @@ +4e360aa1b375e391ec1202f1fe2eb26d64895faf326ec9c7a9b8d8351b6f4dc3 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 86fa7a40734b78d65083c3374d4763f0f2165dfd..c215ba1eba525578801179c9f60e9582f4d0d93f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19202,6 +19202,8 @@ CREATE INDEX index_ci_job_artifacts_on_file_store ON public.ci_job_artifacts USI CREATE UNIQUE INDEX index_ci_job_artifacts_on_job_id_and_file_type ON public.ci_job_artifacts USING btree (job_id, file_type); +CREATE INDEX index_ci_job_artifacts_on_license_compliance_file_types ON public.ci_job_artifacts USING btree (job_id, file_type) WHERE ((file_type = 10) OR (file_type = 101)); + CREATE INDEX index_ci_job_artifacts_on_project_id ON public.ci_job_artifacts USING btree (project_id); CREATE INDEX index_ci_job_artifacts_on_project_id_for_security_reports ON public.ci_job_artifacts USING btree (project_id) WHERE (file_type = ANY (ARRAY[5, 6, 7, 8])); diff --git a/ee/spec/migrations/migrate_license_management_artifacts_to_license_scanning_spec.rb b/ee/spec/migrations/migrate_license_management_artifacts_to_license_scanning_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f72f94215322de624f5ec538391f8318745d1d78 --- /dev/null +++ b/ee/spec/migrations/migrate_license_management_artifacts_to_license_scanning_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20200809221641_migrate_license_management_artifacts_to_license_scanning.rb') + +RSpec.describe MigrateLicenseManagementArtifactsToLicenseScanning, :migration, :sidekiq do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:job_artifacts) { table(:ci_job_artifacts) } + let(:builds) { table(:ci_builds) } + let(:license_management_type) { Ci::JobArtifact.file_types[:license_management] } + let(:license_scanning_type) { Ci::JobArtifact.file_types[:license_scanning] } + + before do + namespaces.create!(id: 1, name: 'tanuki', path: 'tanuki') + projects.create!(id: 42, name: 'tanuki', path: 'tanuki', namespace_id: 1) + builds.create!(id: 1) + builds.create!(id: 2) + builds.create!(id: 3) + job_artifacts.create!(project_id: 42, job_id: 1, file_type: 10) + job_artifacts.create!(project_id: 42, job_id: 2, file_type: 9) + job_artifacts.create!(project_id: 42, job_id: 2, file_type: 10) + end + + context 'with two types of the report' do + before do + job_artifacts.create!(project_id: 42, job_id: 1, file_type: 101) + end + + it 'leaves only one artifact' do + migrate! + + expect(job_artifacts.where(file_type: 10).count).to eq 0 + expect(job_artifacts.where(file_type: 101).count).to eq 2 + expect(job_artifacts.where(file_type: 9).count).to eq 1 + end + end + + context 'with only license_management report' do + it 'changes license_management to license_scanning' do + migrate! + + expect(job_artifacts.where(file_type: 10).count).to eq 0 + expect(job_artifacts.where(file_type: 101).count).to eq 2 + expect(job_artifacts.where(file_type: 9).count).to eq 1 + end + end + + context 'with FOSS version of GitLab' do + before do + allow(Gitlab).to receive(:ee?).and_return(false) + end + + it 'skips this migration' do + migrate! + + expect(job_artifacts.where(file_type: 10).count).to eq 2 + expect(job_artifacts.where(file_type: 101).count).to eq 0 + end + end +end