From f75938d231218165cc421435f8fcd0fb3f46fce6 Mon Sep 17 00:00:00 2001 From: Richard Chong Date: Fri, 12 Dec 2025 17:57:16 -0800 Subject: [PATCH] Consolidate E2E coverage artifacts and make export resilient Consolidate E2E artifact collection: - Merge process-backend-coverage and process-frontend-coverage into single process-e2e-coverage job - Consolidate two download scripts into one that fetches all E2E artifacts (coverage, test reports, test mappings) - Collect E2E test reports (rspec-*.json) for feature category association Make coverage export resilient to test failures: - Make needs: dependencies optional so export jobs run even when upstream test jobs fail - Add when: always to workhorse test artifacts so coverage is available even on test failure - Add file existence checks before export to gracefully skip when coverage report, test reports, or test mapping are missing --- .gitlab/ci/coverage.gitlab-ci.yml | 50 ++++++++++++++++--- .gitlab/ci/qa-common/main.gitlab-ci.yml | 47 ++++++++--------- .gitlab/ci/test-on-cng/main.gitlab-ci.yml | 4 -- .gitlab/ci/test-on-gdk/main.gitlab-ci.yml | 15 ++---- .gitlab/ci/test-on-omnibus/main.gitlab-ci.yml | 4 -- .gitlab/ci/workhorse.gitlab-ci.yml | 1 + Gemfile | 2 +- Gemfile.checksum | 2 +- Gemfile.lock | 4 +- Gemfile.next.checksum | 2 +- Gemfile.next.lock | 4 +- ...nload_e2e_artifacts_from_child_pipeline.rb | 29 +++++++++++ ...wnload_e2e_coverage_from_child_pipeline.rb | 24 --------- ...e_frontend_coverage_from_child_pipeline.rb | 24 --------- 14 files changed, 105 insertions(+), 107 deletions(-) create mode 100755 scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb delete mode 100755 scripts/coverage/download_e2e_coverage_from_child_pipeline.rb delete mode 100755 scripts/coverage/download_e2e_frontend_coverage_from_child_pipeline.rb diff --git a/.gitlab/ci/coverage.gitlab-ci.yml b/.gitlab/ci/coverage.gitlab-ci.yml index 2632051f48b899..b982c61f98d1f0 100644 --- a/.gitlab/ci/coverage.gitlab-ci.yml +++ b/.gitlab/ci/coverage.gitlab-ci.yml @@ -23,14 +23,27 @@ test-coverage:export-rspec-and-e2e: stage: post-qa needs: - job: e2e:test-on-gdk + optional: true - job: rspec:coverage + optional: true - job: update-tests-metadata + optional: true script: - - ruby scripts/coverage/download_e2e_coverage_from_child_pipeline.rb + - ruby scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb - ruby scripts/coverage/merge_backend_coverage.rb - echo "Merging E2E test mappings with Crystalball mappings..." - ruby scripts/coverage/merge_e2e_backend_test_mapping.rb - | + if [ ! -f "coverage-backend/coverage.lcov" ]; then + echo "Backend coverage report not found. Skipping export to ClickHouse..." + exit 0 + fi + + if ! ls rspec/rspec-*.json e2e-test-reports/rspec-*.json 2>/dev/null | head -1 | grep -q .; then + echo "No test reports found. Skipping export to ClickHouse..." + exit 0 + fi + if [ ! -f "crystalball/merged-mapping.json.gz" ]; then echo "Merged test mapping not found. Skipping export to ClickHouse..." exit 0 @@ -54,10 +67,7 @@ test-coverage:export-rspec-and-e2e: - e2e-test-reports/ expire_in: 7d -# Job is temporary disabled as it relies on `process-frontend-coverage` job that -# was causing master pipeline failures. -# Example https://gitlab.com/gitlab-org/gitlab/-/jobs/12319670339 -.test-coverage:export-jest-and-e2e: +test-coverage:export-jest-and-e2e: extends: - .coverage-export-base - .with-ci-node-image @@ -66,15 +76,27 @@ test-coverage:export-rspec-and-e2e: stage: post-qa needs: - job: e2e:test-on-gdk + optional: true - job: coverage-frontend + optional: true before_script: - !reference [.default-utils-before_script, before_script] - eval "bundle install ${BUNDLE_INSTALL_FLAGS}" - source scripts/utils.sh - yarn_install_script script: - - ruby scripts/coverage/download_e2e_frontend_coverage_from_child_pipeline.rb + - ruby scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb - | + if [ ! -f "coverage-frontend/lcov.info" ]; then + echo "Frontend coverage report not found. Skipping export to ClickHouse..." + exit 0 + fi + + if ! ls jest-reports/**/*.json e2e-test-reports/rspec-*.json 2>/dev/null | head -1 | grep -q .; then + echo "No test reports found. Skipping export to ClickHouse..." + exit 0 + fi + echo "Merging Jest + E2E coverage..." node scripts/frontend/merge_coverage_frontend.js - | @@ -109,7 +131,23 @@ test-coverage:export-workhorse: - .workhorse:rules:workhorse-coverage needs: - job: workhorse-coverage + optional: true script: + - | + if [ ! -f "workhorse/coverage.lcov" ]; then + echo "Workhorse coverage report not found. Skipping export to ClickHouse..." + exit 0 + fi + + if [ ! -f "workhorse-test-reports/workhorse-tests.json" ]; then + echo "Workhorse test report not found. Skipping export to ClickHouse..." + exit 0 + fi + + if [ ! -f "workhorse-test-mapping/workhorse-source-to-test.json" ]; then + echo "Workhorse test mapping not found. Skipping export to ClickHouse..." + exit 0 + fi - echo "Exporting Workhorse Go coverage to ClickHouse..." - bundle exec test-coverage --test-reports 'workhorse-test-reports/workhorse-tests.json' diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml index a83922c0c1a9b9..e8fcbb0b699933 100644 --- a/.gitlab/ci/qa-common/main.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml @@ -112,6 +112,7 @@ stages: - .qa-install stage: report when: always + allow_failure: true variables: QA_CODE_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-code-paths-mapping-*.json script: @@ -124,6 +125,7 @@ stages: - .qa-install stage: report when: always + allow_failure: true variables: QA_FRONTEND_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/js-coverage-by-example-*.json script: @@ -131,33 +133,12 @@ stages: rules: - if: '$BABEL_ENV == "istanbul"' -.process-backend-coverage: - extends: - - .qa-install - stage: report - before_script: - - cd $CI_PROJECT_DIR - script: - - echo "Collecting E2E Coverband coverage files..." - - mkdir -p coverage-e2e-backend - - cp qa/tmp/coverband-coverage-*.json coverage-e2e-backend/ 2>/dev/null || echo "No Coverband files found" - - echo "Collecting E2E test mapping files..." - - mkdir -p e2e-test-mapping - - cp qa/tmp/test-code-paths-mapping-*.json e2e-test-mapping/ 2>/dev/null || echo "No test mapping files found" - artifacts: - paths: - - coverage-e2e-backend/ - - e2e-test-mapping/ - expire_in: 7d - when: always - rules: - - if: '$COVERBAND_ENABLED == "true" && $GLCI_EXPORT_COVERAGE == "true"' - -.process-frontend-coverage: +.process-e2e-coverage: extends: - .with-ci-node-image - .yarn-cache stage: report + allow_failure: true variables: NODE_ENV: "development" # Need devDependencies for istanbul packages before_script: @@ -166,17 +147,29 @@ stages: - echo "Installing yarn packages with devDependencies..." - yarn install --frozen-lockfile --production=false script: - - echo "Processing E2E Istanbul coverage..." - - yarn node scripts/frontend/merge_e2e_coverage.js + - echo "Collecting E2E backend coverage files..." + - mkdir -p coverage-e2e-backend + - cp qa/tmp/coverband-coverage-*.json coverage-e2e-backend/ 2>/dev/null || echo "No Coverband files found" + - echo "Collecting E2E backend test mapping files..." + - mkdir -p e2e-test-mapping + - cp qa/tmp/test-code-paths-mapping-*.json e2e-test-mapping/ 2>/dev/null || echo "No backend test mapping files found" + - echo "Processing E2E frontend coverage..." + - NODE_OPTIONS="--max-old-space-size=4096" yarn node scripts/frontend/merge_e2e_coverage.js - echo "Collecting E2E frontend test mappings..." - - find qa/tmp -name 'js-coverage-by-example-*.json' -exec cp {} coverage-e2e-frontend/ \; 2>/dev/null || echo "No E2E test mapping files found" + - cp qa/tmp/js-coverage-by-example-*.json coverage-e2e-frontend/ 2>/dev/null || echo "No E2E frontend test mapping files found" + - echo "Collecting E2E test report files..." + - mkdir -p e2e-test-reports + - cp qa/tmp/rspec-*.json e2e-test-reports/ 2>/dev/null || echo "No E2E test report files found" artifacts: paths: + - coverage-e2e-backend/ - coverage-e2e-frontend/ + - e2e-test-mapping/ + - e2e-test-reports/ expire_in: 7d when: always rules: - - if: '$BABEL_ENV == "istanbul" && $GLCI_EXPORT_COVERAGE == "true"' + - if: '$COVERBAND_ENABLED == "true" && $BABEL_ENV == "istanbul" && $GLCI_EXPORT_COVERAGE == "true"' .notify-slack: extends: diff --git a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml index 0d91074ac3b876..af58d384615af3 100644 --- a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml @@ -227,10 +227,6 @@ e2e-test-report: variables: ALLURE_REPORT_RESULTS_GLOB: "qa/tmp/allure-results" -process-backend-coverage: - extends: - - .process-backend-coverage - notify-slack: extends: - .notify-slack diff --git a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml index a3376b95940474..18cf745f1d01c7 100644 --- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml @@ -151,18 +151,11 @@ export-frontend-code-paths-mapping: extends: - .export-frontend-code-paths-mapping -process-backend-coverage: +process-e2e-coverage: extends: - - .process-backend-coverage - -# Job is temporary disabled as it was causing master pipeline failures. -# Example https://gitlab.com/gitlab-org/gitlab/-/jobs/12319670339 -# -# process-frontend-coverage: -# extends: -# - .process-frontend-coverage -# needs: -# - job: gdk-instance + - .process-e2e-coverage + needs: + - job: gdk-instance notify-slack: extends: diff --git a/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml b/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml index aca028a0495120..4e4aab73880a10 100644 --- a/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml @@ -530,10 +530,6 @@ upload-test-runtime-data: variables: QA_JSON_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/gitlab-qa-run-*/gitlab-*-qa-*/rspec-*.json -process-backend-coverage: - extends: - - .process-backend-coverage - notify-slack: extends: - .notify-slack diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index e23621d66f130e..722408060dbe10 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -55,6 +55,7 @@ workhorse:test go: coverage: '/\d+.\d+%/' artifacts: expire_in: 30 days + when: always paths: - workhorse/coverage.html - workhorse/cover.out diff --git a/Gemfile b/Gemfile index 957461471f0062..9349c54c322253 100644 --- a/Gemfile +++ b/Gemfile @@ -599,7 +599,7 @@ group :test do # Moved in `test` because https://gitlab.com/gitlab-org/gitlab/-/issues/217527 gem 'derailed_benchmarks', require: false, feature_category: :shared # rubocop:todo Gemfile/MissingFeatureCategory -- https://gitlab.com/gitlab-org/gitlab/-/issues/581839 - gem 'gitlab_quality-test_tooling', '~> 3.1.0', require: false, feature_category: :tooling + gem 'gitlab_quality-test_tooling', '~> 3.2.1', require: false, feature_category: :tooling end gem 'octokit', '~> 9.0', feature_category: :importers diff --git a/Gemfile.checksum b/Gemfile.checksum index 4883a5d026fc52..65bfbed481fbfe 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -245,7 +245,7 @@ {"name":"gitlab-styles","version":"13.1.0","platform":"ruby","checksum":"46c7c5729616355868b7b40a4ffcd052b36346076042abe8cafaee1688cbf2c1"}, {"name":"gitlab_chronic_duration","version":"0.12.0","platform":"ruby","checksum":"0d766944d415b5c831f176871ee8625783fc0c5bfbef2d79a3a616f207ffc16d"}, {"name":"gitlab_omniauth-ldap","version":"2.3.0","platform":"ruby","checksum":"167036fe37c2711f2e1d2047260766e4c9b31ac37dfc873b101bcd4ea2a3a3b4"}, -{"name":"gitlab_quality-test_tooling","version":"3.1.0","platform":"ruby","checksum":"0c0cb0f24e556eb06181a6c1b971efd963675171b52ed46025c7091263ec56f4"}, +{"name":"gitlab_quality-test_tooling","version":"3.2.1","platform":"ruby","checksum":"32ad2d7e441afc63562e103fcb8a2485e1022830fcc393704700df7daef50247"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"aarch64-linux-gnu","checksum":"c75e9944e0edceb56078bec627458a31ddbe3cb6f69cde9f1cb7c0fb1c7ffb92"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"aarch64-linux-musl","checksum":"ad09ec1f86643ead038c213e7864ae69adb50ecc47486c81d8f812929f17758a"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"arm64-darwin","checksum":"fcf08ce327f1e81df25a24b7b454cc3607f1f784d6e7325b09ccc1f80c946fee"}, diff --git a/Gemfile.lock b/Gemfile.lock index 46c2c77bbd132e..a96679a3a8b79d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -821,7 +821,7 @@ GEM omniauth (>= 1.3, < 3) pyu-ruby-sasl (>= 0.0.3.3, < 0.1) rubyntlm (~> 0.5) - gitlab_quality-test_tooling (3.1.0) + gitlab_quality-test_tooling (3.2.1) activesupport (>= 7.0) amatch (~> 0.4.1) fog-google (~> 1.24, >= 1.24.1) @@ -2215,7 +2215,7 @@ DEPENDENCIES gitlab-utils! gitlab_chronic_duration (~> 0.12) gitlab_omniauth-ldap (~> 2.3.0) - gitlab_quality-test_tooling (~> 3.1.0) + gitlab_quality-test_tooling (~> 3.2.1) gitlab_query_language (~> 0.20.11) gon (~> 6.5.0) google-apis-androidpublisher_v3 (~> 0.86.0) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 4883a5d026fc52..65bfbed481fbfe 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -245,7 +245,7 @@ {"name":"gitlab-styles","version":"13.1.0","platform":"ruby","checksum":"46c7c5729616355868b7b40a4ffcd052b36346076042abe8cafaee1688cbf2c1"}, {"name":"gitlab_chronic_duration","version":"0.12.0","platform":"ruby","checksum":"0d766944d415b5c831f176871ee8625783fc0c5bfbef2d79a3a616f207ffc16d"}, {"name":"gitlab_omniauth-ldap","version":"2.3.0","platform":"ruby","checksum":"167036fe37c2711f2e1d2047260766e4c9b31ac37dfc873b101bcd4ea2a3a3b4"}, -{"name":"gitlab_quality-test_tooling","version":"3.1.0","platform":"ruby","checksum":"0c0cb0f24e556eb06181a6c1b971efd963675171b52ed46025c7091263ec56f4"}, +{"name":"gitlab_quality-test_tooling","version":"3.2.1","platform":"ruby","checksum":"32ad2d7e441afc63562e103fcb8a2485e1022830fcc393704700df7daef50247"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"aarch64-linux-gnu","checksum":"c75e9944e0edceb56078bec627458a31ddbe3cb6f69cde9f1cb7c0fb1c7ffb92"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"aarch64-linux-musl","checksum":"ad09ec1f86643ead038c213e7864ae69adb50ecc47486c81d8f812929f17758a"}, {"name":"gitlab_query_language","version":"0.20.11","platform":"arm64-darwin","checksum":"fcf08ce327f1e81df25a24b7b454cc3607f1f784d6e7325b09ccc1f80c946fee"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 46c2c77bbd132e..a96679a3a8b79d 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -821,7 +821,7 @@ GEM omniauth (>= 1.3, < 3) pyu-ruby-sasl (>= 0.0.3.3, < 0.1) rubyntlm (~> 0.5) - gitlab_quality-test_tooling (3.1.0) + gitlab_quality-test_tooling (3.2.1) activesupport (>= 7.0) amatch (~> 0.4.1) fog-google (~> 1.24, >= 1.24.1) @@ -2215,7 +2215,7 @@ DEPENDENCIES gitlab-utils! gitlab_chronic_duration (~> 0.12) gitlab_omniauth-ldap (~> 2.3.0) - gitlab_quality-test_tooling (~> 3.1.0) + gitlab_quality-test_tooling (~> 3.2.1) gitlab_query_language (~> 0.20.11) gon (~> 6.5.0) google-apis-androidpublisher_v3 (~> 0.86.0) diff --git a/scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb b/scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb new file mode 100755 index 00000000000000..aa0c10e9ce35b2 --- /dev/null +++ b/scripts/coverage/download_e2e_artifacts_from_child_pipeline.rb @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative 'child_pipeline_artifact_downloader' + +# Downloads all E2E artifacts (coverage + test reports + test mappings) from a child pipeline +# triggered by the e2e:test-on-gdk job. +# +# The artifacts downloaded include: +# - coverage-e2e-backend/ (Coverband coverage files) +# - coverage-e2e-frontend/ (Istanbul coverage + test mappings) +# - e2e-test-reports/ (RSpec JSON reports with feature_category metadata) +if __FILE__ == $PROGRAM_NAME + downloader = ChildPipelineArtifactDownloader.new( + bridge_name: 'e2e:test-on-gdk', + job_name: 'process-e2e-coverage', + coverage_type: 'e2e-artifacts' + ) + + begin + downloader.run + rescue StandardError => e + puts "Warning: #{e.message}" + end + + # Exit 0 even if artifacts not found (graceful skip) + # This allows the parent job to continue without E2E artifacts + exit 0 +end diff --git a/scripts/coverage/download_e2e_coverage_from_child_pipeline.rb b/scripts/coverage/download_e2e_coverage_from_child_pipeline.rb deleted file mode 100755 index 3ae75aefd8a73e..00000000000000 --- a/scripts/coverage/download_e2e_coverage_from_child_pipeline.rb +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative 'child_pipeline_artifact_downloader' - -# Downloads E2E backend coverage artifacts from a child pipeline -# triggered by the e2e:test-on-gdk job. -if __FILE__ == $PROGRAM_NAME - downloader = ChildPipelineArtifactDownloader.new( - bridge_name: 'e2e:test-on-gdk', - job_name: 'process-backend-coverage', - coverage_type: 'backend' - ) - - begin - downloader.run - rescue StandardError => e - puts "Warning: #{e.message}" - end - - # Exit 0 even if artifacts not found (graceful skip) - # This allows the parent job to continue without E2E coverage - exit 0 -end diff --git a/scripts/coverage/download_e2e_frontend_coverage_from_child_pipeline.rb b/scripts/coverage/download_e2e_frontend_coverage_from_child_pipeline.rb deleted file mode 100755 index 2d7753661eb4bb..00000000000000 --- a/scripts/coverage/download_e2e_frontend_coverage_from_child_pipeline.rb +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative 'child_pipeline_artifact_downloader' - -# Downloads E2E frontend coverage artifacts from a child pipeline -# triggered by the e2e:test-on-gdk job. -if __FILE__ == $PROGRAM_NAME - downloader = ChildPipelineArtifactDownloader.new( - bridge_name: 'e2e:test-on-gdk', - job_name: 'process-frontend-coverage', - coverage_type: 'frontend' - ) - - begin - downloader.run - rescue StandardError => e - puts "Warning: #{e.message}" - end - - # Exit 0 even if artifacts not found (graceful skip) - # This allows the parent job to continue without E2E coverage - exit 0 -end -- GitLab