diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 76af1b49ddcd6db3fca02ac7a2c4e27288bdc71a..29b6ceebdc1a436b24f35b28bea559b639c1e781 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -83,6 +83,7 @@ class Pipeline < ApplicationRecord has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id has_many :latest_builds_report_results, through: :latest_builds, source: :report_results + has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline accepts_nested_attributes_for :variables, reject_if: :persisted? @@ -221,6 +222,10 @@ class Pipeline < ApplicationRecord AutoMergeProcessWorker.perform_async(merge_request.id) end + if pipeline.has_coverage_reports? + ::Ci::PipelineArtifactWorker.perform_async(pipeline.id) + end + if pipeline.auto_devops_source? self.class.auto_devops_pipelines_completed_total.increment(status: pipeline.status) end @@ -910,6 +915,10 @@ def has_reports?(reports_scope) complete? && latest_report_builds(reports_scope).exists? end + def has_coverage_reports? + self.has_reports?(Ci::JobArtifact.coverage_reports) + end + def test_report_summary Gitlab::Ci::Reports::TestReportSummary.new(latest_builds_report_results) end diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..87542c5e795a97c521801edd689f8bdc5d1f4927 --- /dev/null +++ b/app/models/ci/pipeline_artifact.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# This class is being used to persist generated report in a pipeline context + +module Ci + class PipelineArtifact < ApplicationRecord + include ObjectStorage::BackgroundMove + extend Gitlab::Ci::Model + + belongs_to :project, class_name: "Project", inverse_of: :pipeline_artifacts + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_artifacts + + mount_uploader :file, Ci::PipelineArtifactUploader + + enum file_type: { + coverage_report: 1, + } + + enum file_format: { + json: 1 + }, _suffix: true + + validates :pipeline, :project, presence: true + validates :file_store, inclusion: { in: [1, 2] } + + def hashed_path? + true + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 92d2c85e99ab66bc82cfdcdeb99bb64b9300182a..7a6e3e7356c403d7ca15783d4aec062bf45298da 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -301,6 +301,7 @@ class Project < ApplicationRecord has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :build_report_results, class_name: 'Ci::BuildReportResult', inverse_of: :project has_many :job_artifacts, class_name: 'Ci::JobArtifact' + has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :project has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' diff --git a/app/services/ci/generate_coverage_reports_service.rb b/app/services/ci/generate_coverage_reports_service.rb index ebd1eaf0bade177b6874f241ce42d49001752c23..ede8ee8d53afcb0f4b842ee8ba06a4c9314c67d0 100644 --- a/app/services/ci/generate_coverage_reports_service.rb +++ b/app/services/ci/generate_coverage_reports_service.rb @@ -9,11 +9,14 @@ module Ci class GenerateCoverageReportsService < CompareReportsBaseService def execute(base_pipeline, head_pipeline) merge_request = MergeRequest.find_by_id(params[:id]) - { - status: :parsed, - key: key(base_pipeline, head_pipeline), - data: head_pipeline.coverage_reports.pick(merge_request.new_paths) - } + head_pipeline.pipeline_artifacts.first.file.open do |file| + raw_coverage = Gitlab::Json.parse(file.read) + { + status: :parsed, + key: key(base_pipeline, head_pipeline), + data: { files: pick(raw_coverage, merge_request.new_paths) } + } + end rescue => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id) { @@ -26,5 +29,13 @@ def execute(base_pipeline, head_pipeline) def latest?(base_pipeline, head_pipeline, data) data&.fetch(:key, nil) == key(base_pipeline, head_pipeline) end + + private + + def pick(files, keys) + files.select do |key| + keys.include?(key) + end + end end end diff --git a/app/services/ci/pipeline_artifact_service.rb b/app/services/ci/pipeline_artifact_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..723d397eb2aee5d484c45a002cd529d9891306bf --- /dev/null +++ b/app/services/ci/pipeline_artifact_service.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Ci + class PipelineArtifactService + def execute(pipeline) + return unless Feature.enabled?(:coverage_report_view, pipeline.project) + return unless pipeline.has_coverage_reports? + + coverage_report = pipeline.job_artifacts.coverage_reports.first + + report = pipeline.pipeline_artifacts.create!( + project_id: pipeline.project_id, + file_type: 1, + file_format: 1, + file_store: coverage_report.file_store, + size: coverage_report.size, + file: CarrierWaveStringFile.new(pipeline.coverage_reports.to_json) + ) + end + end +end diff --git a/app/uploaders/ci/pipeline_artifact_uploader.rb b/app/uploaders/ci/pipeline_artifact_uploader.rb new file mode 100644 index 0000000000000000000000000000000000000000..334f942e0e362d99f0804a8243e316bea2be513e --- /dev/null +++ b/app/uploaders/ci/pipeline_artifact_uploader.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class PipelineArtifactUploader < JobArtifactUploader + def filename + "#{model.file_type}-#{model.pipeline_id}" + end + + def store_dir + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, + model.created_at.utc.strftime('%Y_%m_%d'), model.pipeline_id.to_s, model.id.to_s) + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3d243cfd00c3bde65e0f3b6102c6661276653191..b49b9543c670482b409b5885446cb97819b913be 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -883,6 +883,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: pipeline_background:ci_pipeline_artifact + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_background:ci_pipeline_success_unlock_artifacts :feature_category: :continuous_integration :has_external_dependencies: diff --git a/app/workers/ci/pipeline_artifact_worker.rb b/app/workers/ci/pipeline_artifact_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..80ea478500379612635f917c6fef383c16325f80 --- /dev/null +++ b/app/workers/ci/pipeline_artifact_worker.rb @@ -0,0 +1,14 @@ +module Ci + class PipelineArtifactWorker + include ApplicationWorker + include PipelineBackgroundQueue + + idempotent! + + def perform(pipeline_id) + Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + Ci::PipelineArtifactService.new.execute(pipeline) + end + end + end +end diff --git a/db/migrate/20200722152956_create_ci_pipeline_artifact.rb b/db/migrate/20200722152956_create_ci_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..147543ca89addda046f69df5a77aaf97b98652fb --- /dev/null +++ b/db/migrate/20200722152956_create_ci_pipeline_artifact.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class CreateCiPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:ci_pipeline_artifacts) + create_table :ci_pipeline_artifacts do |t| + t.references :pipeline, foreign_key: { to_table: :ci_pipelines, on_delete: :cascade }, index: true, null: false + t.references :project, foreign_key: { on_delete: :cascade }, index: true, null: false + t.timestamps_with_timezone + t.integer :file_type, null: false, limit: 2 + t.integer :size, null: false + t.integer :file_store, null: false, limit: 2 + t.integer :file_format, null: false, limit: 2 + t.text :file + end + end + + add_text_limit :ci_pipeline_artifacts, :file, 255 + end + + def down + drop_table(:ci_pipeline_artifacts) + end +end diff --git a/db/structure.sql b/db/structure.sql index e1c40897c4ef73adc86d9c8a8bc692f3147873b7..9e264b8ab14fabf849a2bc41362a02e819816d87 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9976,6 +9976,29 @@ CREATE SEQUENCE public.ci_job_variables_id_seq ALTER SEQUENCE public.ci_job_variables_id_seq OWNED BY public.ci_job_variables.id; +CREATE TABLE public.ci_pipeline_artifacts ( + id bigint NOT NULL, + pipeline_id bigint NOT NULL, + project_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + file_type integer NOT NULL, + size integer NOT NULL, + file_store integer DEFAULT 1 NOT NULL, + file_format integer NOT NULL, + file text, + CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)) +); + +CREATE SEQUENCE public.ci_pipeline_artifacts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.ci_pipeline_artifacts_id_seq OWNED BY public.ci_pipeline_artifacts.id; + CREATE TABLE public.ci_pipeline_chat_data ( id bigint NOT NULL, pipeline_id integer NOT NULL, @@ -16600,6 +16623,8 @@ ALTER TABLE ONLY public.ci_job_artifacts ALTER COLUMN id SET DEFAULT nextval('pu ALTER TABLE ONLY public.ci_job_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_job_variables_id_seq'::regclass); +ALTER TABLE ONLY public.ci_pipeline_artifacts ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_artifacts_id_seq'::regclass); + ALTER TABLE ONLY public.ci_pipeline_chat_data ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_chat_data_id_seq'::regclass); ALTER TABLE ONLY public.ci_pipeline_messages ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_messages_id_seq'::regclass); @@ -17533,6 +17558,9 @@ ALTER TABLE ONLY public.ci_job_artifacts ALTER TABLE ONLY public.ci_job_variables ADD CONSTRAINT ci_job_variables_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT ci_pipeline_artifacts_pkey PRIMARY KEY (id); + ALTER TABLE ONLY public.ci_pipeline_chat_data ADD CONSTRAINT ci_pipeline_chat_data_pkey PRIMARY KEY (id); @@ -18994,6 +19022,10 @@ CREATE INDEX index_ci_job_variables_on_job_id ON public.ci_job_variables USING b CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON public.ci_job_variables USING btree (key, job_id); +CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON public.ci_pipeline_artifacts USING btree (pipeline_id); + +CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON public.ci_pipeline_artifacts USING btree (project_id); + CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON public.ci_pipeline_chat_data USING btree (chat_name_id); CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON public.ci_pipeline_chat_data USING btree (pipeline_id); @@ -21993,6 +22025,9 @@ ALTER TABLE ONLY public.vulnerability_feedback ALTER TABLE ONLY public.user_custom_attributes ADD CONSTRAINT fk_rails_47b91868a8 FOREIGN KEY (user_id) REFERENCES public.users(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT fk_rails_4a70390ca6 FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.group_deletion_schedules ADD CONSTRAINT fk_rails_4b8c694a6c FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE; @@ -22449,6 +22484,9 @@ ALTER TABLE ONLY public.resource_milestone_events ALTER TABLE ONLY public.term_agreements ADD CONSTRAINT fk_rails_a88721bcdf FOREIGN KEY (term_id) REFERENCES public.application_setting_terms(id); +ALTER TABLE ONLY public.ci_pipeline_artifacts + ADD CONSTRAINT fk_rails_a9e811a466 FOREIGN KEY (pipeline_id) REFERENCES public.ci_pipelines(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.merge_request_user_mentions ADD CONSTRAINT fk_rails_aa1b2961b1 FOREIGN KEY (merge_request_id) REFERENCES public.merge_requests(id) ON DELETE CASCADE; @@ -24001,5 +24039,6 @@ COPY "schema_migrations" (version) FROM STDIN; 20200718040200 20200718040300 20200720154123 +20200722152956 \. diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_reports.rb index 31afb636d2f39d6640db7fb8390773e15359fb9a..2e616ddc0b4d88893d20e1585013d167f8a0d41f 100644 --- a/lib/gitlab/ci/reports/coverage_reports.rb +++ b/lib/gitlab/ci/reports/coverage_reports.rb @@ -18,6 +18,10 @@ def pick(keys) { files: coverage_files } end + def to_json + ::Gitlab::Json.generate(files) + end + def add_file(name, line_coverage) if files[name].present? line_coverage.each { |line, hits| combine_lines(name, line, hits) }