diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 473087b7c2d126c5b7135f6db0fba82d371ffd19..3c4ffa4d6ab32019590672cd8b29f24580d5736a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -83,6 +83,7 @@ class Pipeline < ApplicationRecord has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id has_many :latest_builds_report_results, through: :latest_builds, source: :report_results + has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent accepts_nested_attributes_for :variables, reject_if: :persisted? diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..daa54aaafe37e3e6cdebfb8a2618abf6b68639b9 --- /dev/null +++ b/app/models/ci/pipeline_artifact.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# This class is being used to persist generated report consumable by gitlab frontend in a pipeline context. + +module Ci + class PipelineArtifact < ApplicationRecord + extend Gitlab::Ci::Model + + FILE_STORE_SUPPORTED = [ + ObjectStorage::Store::LOCAL, + ObjectStorage::Store::REMOTE + ].freeze + + FILE_SIZE_LIMIT = 10.megabytes.freeze + + belongs_to :project, class_name: "Project", inverse_of: :pipeline_artifacts + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_artifacts + + validates :pipeline, :project, :file_format, presence: true + validates :file_store, presence: true, inclusion: { in: FILE_STORE_SUPPORTED } + validates :size, presence: true, numericality: { less_than_or_equal_to: FILE_SIZE_LIMIT } + validates :file_type, presence: true, uniqueness: { scope: [:pipeline_id] } + + enum file_type: { + code_coverage: 1 + } + + enum file_format: { + raw: 1, + zip: 2, + gzip: 3 + }, _suffix: true + end +end diff --git a/app/models/project.rb b/app/models/project.rb index b37dc3ed3972b1b27ec8d44536202cdfd52945dc..fdca74af81d6086283e8475a38ea53ee65e41b17 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -299,6 +299,7 @@ class Project < ApplicationRecord has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :build_report_results, class_name: 'Ci::BuildReportResult', inverse_of: :project has_many :job_artifacts, class_name: 'Ci::JobArtifact' + has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :project has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' diff --git a/changelogs/unreleased/mo-introduct-pipeline-artifact.yml b/changelogs/unreleased/mo-introduct-pipeline-artifact.yml new file mode 100644 index 0000000000000000000000000000000000000000..22d44bd25fc77683dfe38f6fa10a897cc3658fe2 --- /dev/null +++ b/changelogs/unreleased/mo-introduct-pipeline-artifact.yml @@ -0,0 +1,5 @@ +--- +title: Add PipelineArtifact data model +merge_request: 37969 +author: +type: performance diff --git a/db/migrate/20200805150316_create_ci_pipeline_artifact.rb b/db/migrate/20200805150316_create_ci_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..01995972011440ea5ee78bd587b208bc2bb7901b --- /dev/null +++ b/db/migrate/20200805150316_create_ci_pipeline_artifact.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class CreateCiPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:ci_pipeline_artifacts) + create_table :ci_pipeline_artifacts do |t| + t.timestamps_with_timezone + t.bigint :pipeline_id, null: false, index: true + t.bigint :project_id, null: false, index: true + t.integer :size, null: false + t.integer :file_store, null: false, limit: 2 + t.integer :file_type, null: false, limit: 2 + t.integer :file_format, null: false, limit: 2 + t.text :file + + t.index [:pipeline_id, :file_type], unique: true + end + end + + add_text_limit :ci_pipeline_artifacts, :file, 255 + end + + def down + drop_table :ci_pipeline_artifacts + end +end diff --git a/db/migrate/20200805151001_add_foreign_key_to_pipeline_id_on_pipeline_artifact.rb b/db/migrate/20200805151001_add_foreign_key_to_pipeline_id_on_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..d6c3a4fe74203ecf0e9ec8bdc9d16d538dc96a58 --- /dev/null +++ b/db/migrate/20200805151001_add_foreign_key_to_pipeline_id_on_pipeline_artifact.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddForeignKeyToPipelineIdOnPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + add_foreign_key :ci_pipeline_artifacts, :ci_pipelines, column: :pipeline_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey + end + end + + def down + with_lock_retries do + remove_foreign_key :ci_pipeline_artifacts, column: :pipeline_id + end + end +end diff --git a/db/migrate/20200805151726_add_foreign_key_to_project_id_on_pipeline_artifact.rb b/db/migrate/20200805151726_add_foreign_key_to_project_id_on_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..367a2774d6280822689b31aee04d49d2b5518a63 --- /dev/null +++ b/db/migrate/20200805151726_add_foreign_key_to_project_id_on_pipeline_artifact.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddForeignKeyToProjectIdOnPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + add_foreign_key :ci_pipeline_artifacts, :projects, column: :project_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey + end + end + + def down + with_lock_retries do + remove_foreign_key :ci_pipeline_artifacts, column: :project_id + end + end +end diff --git a/db/schema_migrations/20200805150316 b/db/schema_migrations/20200805150316 new file mode 100644 index 0000000000000000000000000000000000000000..d7560ecc376c39143c338bde663d8339126fdc8b --- /dev/null +++ b/db/schema_migrations/20200805150316 @@ -0,0 +1 @@ +983fa89c574e41a7d38eb558779f46d6d42aee5fba92f4185307e1508c401298 \ No newline at end of file diff --git a/db/schema_migrations/20200805151001 b/db/schema_migrations/20200805151001 new file mode 100644 index 0000000000000000000000000000000000000000..54c569efbdc72656dcfe4a3a9a7a75f904e3540f --- /dev/null +++ b/db/schema_migrations/20200805151001 @@ -0,0 +1 @@ +8fdb1e994ca7a28f7e061fb80cf210c482bafbe2bd0dc19c631c8fe9e0e2bbaf \ No newline at end of file diff --git a/db/schema_migrations/20200805151726 b/db/schema_migrations/20200805151726 new file mode 100644 index 0000000000000000000000000000000000000000..677f44c63802a5dea8c0eea9c2f73ff7d7599864 --- /dev/null +++ b/db/schema_migrations/20200805151726 @@ -0,0 +1 @@ +e992135d6a4d10224b7e3deb304790735b6a35b5fb320670f9a7029e2924efb5 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 15b4df63c3c4347e48ef101b7cc4c64f897a9ff3..41b93b784cbde07b8d9683f4354fa2239848c48d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10045,6 +10045,29 @@ CREATE SEQUENCE public.ci_job_variables_id_seq ALTER SEQUENCE public.ci_job_variables_id_seq OWNED BY public.ci_job_variables.id; +CREATE TABLE public.ci_pipeline_artifacts ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + pipeline_id bigint NOT NULL, + project_id bigint NOT NULL, + size integer NOT NULL, + file_store smallint NOT NULL, + file_type smallint NOT NULL, + file_format smallint NOT NULL, + file text, + CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)) +); + +CREATE SEQUENCE public.ci_pipeline_artifacts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.ci_pipeline_artifacts_id_seq OWNED BY public.ci_pipeline_artifacts.id; + CREATE TABLE public.ci_pipeline_chat_data ( id bigint NOT NULL, pipeline_id integer NOT NULL, @@ -16706,6 +16729,8 @@ ALTER TABLE ONLY public.ci_job_artifacts ALTER COLUMN id SET DEFAULT nextval('pu ALTER TABLE ONLY public.ci_job_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_job_variables_id_seq'::regclass); +ALTER TABLE ONLY public.ci_pipeline_artifacts ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_artifacts_id_seq'::regclass); + ALTER TABLE ONLY public.ci_pipeline_chat_data ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_chat_data_id_seq'::regclass); ALTER TABLE ONLY public.ci_pipeline_messages ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_messages_id_seq'::regclass); @@ -17647,6 +17672,9 @@ ALTER TABLE ONLY public.ci_job_artifacts ALTER TABLE ONLY public.ci_job_variables ADD CONSTRAINT ci_job_variables_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT ci_pipeline_artifacts_pkey PRIMARY KEY (id); + ALTER TABLE ONLY public.ci_pipeline_chat_data ADD CONSTRAINT ci_pipeline_chat_data_pkey PRIMARY KEY (id); @@ -19107,6 +19135,12 @@ CREATE INDEX index_ci_job_variables_on_job_id ON public.ci_job_variables USING b CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON public.ci_job_variables USING btree (key, job_id); +CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON public.ci_pipeline_artifacts USING btree (pipeline_id); + +CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON public.ci_pipeline_artifacts USING btree (pipeline_id, file_type); + +CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON public.ci_pipeline_artifacts USING btree (project_id); + CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON public.ci_pipeline_chat_data USING btree (chat_name_id); CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON public.ci_pipeline_chat_data USING btree (pipeline_id); @@ -22138,6 +22172,9 @@ ALTER TABLE ONLY public.vulnerability_feedback ALTER TABLE ONLY public.user_custom_attributes ADD CONSTRAINT fk_rails_47b91868a8 FOREIGN KEY (user_id) REFERENCES public.users(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT fk_rails_4a70390ca6 FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.group_deletion_schedules ADD CONSTRAINT fk_rails_4b8c694a6c FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE; @@ -22597,6 +22634,9 @@ ALTER TABLE ONLY public.resource_milestone_events ALTER TABLE ONLY public.term_agreements ADD CONSTRAINT fk_rails_a88721bcdf FOREIGN KEY (term_id) REFERENCES public.application_setting_terms(id); +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT fk_rails_a9e811a466 FOREIGN KEY (pipeline_id) REFERENCES public.ci_pipelines(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.merge_request_user_mentions ADD CONSTRAINT fk_rails_aa1b2961b1 FOREIGN KEY (merge_request_id) REFERENCES public.merge_requests(id) ON DELETE CASCADE; diff --git a/spec/factories/ci/pipeline_artifacts.rb b/spec/factories/ci/pipeline_artifacts.rb new file mode 100644 index 0000000000000000000000000000000000000000..d3f3f9e92ac05e1c3f1054c9e214e56c29917100 --- /dev/null +++ b/spec/factories/ci/pipeline_artifacts.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ci_pipeline_artifact, class: 'Ci::PipelineArtifact' do + pipeline factory: :ci_pipeline + project { pipeline.project } + file_type { :code_coverage } + file_format { :raw } + file_store { Ci::PipelineArtifact::FILE_STORE_SUPPORTED.first } + size { 1.megabytes } + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 58cae672cf7b6f27f03b8bfd4fb591bf6cac7e83..bc51e8747d4021bf34b7420e704ac50c108b8cf1 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -230,6 +230,7 @@ ci_pipelines: - daily_report_results - latest_builds_report_results - messages +- pipeline_artifacts ci_refs: - project - ci_pipelines @@ -519,6 +520,7 @@ project: - vulnerability_statistic - vulnerability_historical_statistics - product_analytics_events +- pipeline_artifacts award_emoji: - awardable - user diff --git a/spec/models/ci/pipeline_artifact_spec.rb b/spec/models/ci/pipeline_artifact_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..930102f087c2ae059234bfcd6517aaff300d97a9 --- /dev/null +++ b/spec/models/ci/pipeline_artifact_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::PipelineArtifact, type: :model do + let_it_be(:coverage_report) { create(:ci_pipeline_artifact) } + + describe 'associations' do + it { is_expected.to belong_to(:pipeline) } + it { is_expected.to belong_to(:project) } + end + + it_behaves_like 'having unique enum values' + + describe 'validations' do + it { is_expected.to validate_presence_of(:pipeline) } + it { is_expected.to validate_presence_of(:project) } + it { is_expected.to validate_presence_of(:file_type) } + it { is_expected.to validate_presence_of(:file_format) } + it { is_expected.to validate_presence_of(:size) } + it { is_expected.to validate_uniqueness_of(:file_type).scoped_to([:pipeline_id]).ignoring_case_sensitivity } + + context 'when attributes are valid' do + it 'returns no errors' do + expect(coverage_report).to be_valid + end + end + + context 'when file_store is invalid' do + it 'returns errors' do + coverage_report.file_store = 0 + + expect(coverage_report).to be_invalid + expect(coverage_report.errors.full_messages).to eq(["File store is not included in the list"]) + end + end + + context 'when size is over 10 megabytes' do + it 'returns errors' do + coverage_report.size = 11.megabytes + + expect(coverage_report).to be_invalid + end + end + end +end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index c6d9dd8d555828dc1f722bdece440588773aad49..bb3d02eb69f9507aba0838b4d437d3210bf874b4 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -46,6 +46,7 @@ it { is_expected.to respond_to :git_author_email } it { is_expected.to respond_to :short_sha } it { is_expected.to delegate_method(:full_path).to(:project).with_prefix } + it { is_expected.to have_many(:pipeline_artifacts) } describe 'associations' do it 'has a bidirectional relationship with projects' do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 274e2021c0fed4ca2f8e60100961222308b34494..9dada39a7a044b115362a6b5e0e25050ac5ac73d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -122,6 +122,7 @@ it { is_expected.to have_many(:reviews).inverse_of(:project) } it { is_expected.to have_many(:packages).class_name('Packages::Package') } it { is_expected.to have_many(:package_files).class_name('Packages::PackageFile') } + it { is_expected.to have_many(:pipeline_artifacts) } it_behaves_like 'model with repository' do let_it_be(:container) { create(:project, :repository, path: 'somewhere') }