diff --git a/.gitlab/ci/jobs/test/common.yml b/.gitlab/ci/jobs/test/common.yml deleted file mode 100644 index b66ce24785eeed84085c21426929f6d8653dec09..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/common.yml +++ /dev/null @@ -1,121 +0,0 @@ -.test_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - needs: - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - stage: test - before_script: - - . ./scripts/version.sh - - eval $(opam env) - -# We use the --job option to split tests into jobs of roughly the same -# duration. This is based on a file that contains timings of test results, -# generated with --record. To rebalance jobs, update this record with: -# -# make && dune exec tezt/tests/main.exe -- --record tezt/test-results.json - -.tezt_template: - variables: - JUNIT: "tezt-junit.xml" - TEZT_VARIANT: "" - # The variable TESTS controls test selection. It passed directly - # to tezt so can contain tags, their negation, or an arbitrary - # TSL expression. - TESTS: "" - artifacts: - reports: - junit: $JUNIT - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - # Record artifacts are fetched from the scheduled pipeline. - # If this pipeline is broken on a Friday and only gets fixed on Monday, - # the next pipeline runs on Tuesday, so artifacts are unavailable - # for 4 days. We thus set the expiration to 7 days for safety. - # It also allows to make statistics on MRs that are less than 1 week old. - # The tezt artifacts (including records and coverage) - # take up roughly 2MB / job. Total artifact storage - # becomes N*P*T*W where N is the days of retention (7 atm), P the - # number of pipelines per day (~200 atm), T the number of Tezt - # jobs per pipeline (60) and W the artifact size per tezt job - # (2MB). This makes 82GB which is less than 1.5% of our total - # artifact usage (https://gitlab.com/tezos/tezos/-/artifacts). - expire_in: 7 days - when: always - script: - # Print variables in a shell-friendly format. - - 'echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\""' - # Store the list of tests that have been scheduled for execution for later debugging. - # It is imperative this this first call to tezt receives any flags passed to the - # second call that affect test selection. - - ./scripts/ci/tezt.sh - ${TESTS} - --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - # For Tezt tests, there are multiple timeouts: - # - --global-timeout is the internal timeout of Tezt, which only works if tests - # are cooperative; - # - the "timeout" command, which we set to send SIGTERM to Tezt 60s after --global-timeout - # in case tests are not cooperative; - # - the "timeout" command also sends SIGKILL 60s after having sent SIGTERM in case - # Tezt is still stuck; - # - the CI timeout. - # The use of the "timeout" command is to make sure that Tezt eventually exits, - # because if the CI timeout is reached, there are no artefacts, - # and thus no logs to investigate. - # See also: https://gitlab.com/gitlab-org/gitlab/-/issues/19818 - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh - ${TESTS} - --color - --log-buffer-size 5000 - --log-file tezt.log - --global-timeout 1800 - --on-unknown-regression-files fail - --junit ${JUNIT} - --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} - # Only merge coverage traces if coverage output is enabled. - - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi - -# Common template for tezt jobs that use x86_64 binaries and produce -# coverage. -.tezt_tests: - extends: - - .test_template - - .image_template__runtime_e2etest_dependencies - - .oc.template__coverage_output_artifact - - .tezt_template - - .oc.template__retry_flaky - dependencies: - - "select_tezts" - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - - "oc.build_kernels" - - "oc.tezt:fetch-records" - needs: - - "select_tezts" - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - - "oc.build_kernels" - - "oc.tezt:fetch-records" - tags: - - gcp_tezt - # Add [BISECT_FILE] to the set of artifacts defined in [.tezt_template] - artifacts: - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE diff --git a/.gitlab/ci/jobs/test/tezt-before_merging.yml b/.gitlab/ci/jobs/test/tezt-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..76d56055de5f63a26fd0decbc8f96e382e327ef6 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-before_merging.yml @@ -0,0 +1,317 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 60 + +tezt-memory-4k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_4k + TESTS: memory_4k && /ci_disabled && /memory_3k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 4 + +tezt-memory-3k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_3k + TESTS: memory_3k && /ci_disabled && /memory_4k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt-time-sensitive: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -time_sensitive + TESTS: time_sensitive && /ci_disabled && /memory_3k && /memory_4k && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt:static-binaries: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + before_script: + - mv octez-binaries/x86_64/octez-* . + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && cli && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always diff --git a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml index 9a48f9e6709088074fe0a43d06f0521c0a5be3ad..7c3635f545cf3164392feac968b8fdcc0a124ff1 100644 --- a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml @@ -1,7 +1,67 @@ -include: .gitlab/ci/jobs/test/tezt-flaky.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# Runs flaky Tezts in the 'before_merging' pipeline. tezt-flaky: - extends: - - .tezt-flaky - - .rules__octez_changes_manual + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 diff --git a/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml b/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml deleted file mode 100644 index 59bbab4a46748770054212ef52aa835a102af47d..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml +++ /dev/null @@ -1,6 +0,0 @@ -include: .gitlab/ci/jobs/test/tezt-flaky.yml - -# Runs flaky Tezts in the 'schedule_extended_test' pipeline. -tezt-flaky: - extends: - - .tezt-flaky diff --git a/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..4420604e566455c5f93e90288e03fbf66efec499 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml @@ -0,0 +1,58 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt-flaky: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 diff --git a/.gitlab/ci/jobs/test/tezt-flaky.yml b/.gitlab/ci/jobs/test/tezt-flaky.yml deleted file mode 100644 index cbf15b6fa748e5bab3d25d5a1b01bac53137b6e0..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/tezt-flaky.yml +++ /dev/null @@ -1,15 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# Runs flaky Tezts in the 'schedule_extended_test' pipeline. -.tezt-flaky: - extends: - - .tezt_tests - variables: - # Run only flaky tests - TESTS: "/ci_disabled /memory_3k /memory_4k /time_sensitive /slow /cloud flaky" - # To handle flakiness, consider tweaking TEZT_PARALLEL (passed to - # Tezt's '--job-count'), and TEZT_RETRY (passed to Tezt's - # '--retry') - TEZT_RETRY: 3 - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-flaky" diff --git a/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..cc93511388f2cf102660038f12e41c9963c60a88 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml @@ -0,0 +1,277 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 60 + +tezt-memory-4k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_4k + TESTS: memory_4k && /ci_disabled && /memory_3k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 4 + +tezt-memory-3k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_3k + TESTS: memory_3k && /ci_disabled && /memory_4k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt-time-sensitive: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -time_sensitive + TESTS: time_sensitive && /ci_disabled && /memory_3k && /memory_4k && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt:static-binaries: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + before_script: + - mv octez-binaries/x86_64/octez-* . + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && cli && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always diff --git a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml index 4e9695004a1cae538c14e77cc873bef477941569..89c66d86fb3f96a8b26d3da222e6603c260978c7 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml @@ -1,6 +1,65 @@ -include: .gitlab/ci/jobs/test/tezt-slow.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. tezt-slow: - extends: - - .tezt-slow - - .rules__octez_changes_manual + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && + /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 10 diff --git a/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml deleted file mode 100644 index 26b845b05596ec44f5f081501c675f09ee91e990..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml +++ /dev/null @@ -1,5 +0,0 @@ -include: .gitlab/ci/jobs/test/tezt-slow.yml - -tezt-slow: - extends: - - .tezt-slow diff --git a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..a0cf7167b073ba56a9ae12f865195c4228621a95 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml @@ -0,0 +1,56 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt-slow: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && + /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 10 diff --git a/.gitlab/ci/jobs/test/tezt-slow.yml b/.gitlab/ci/jobs/test/tezt-slow.yml deleted file mode 100644 index fc19a9656f1d651c405cbdea446a32c865eb8637..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/tezt-slow.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# Runs Tezts tagged as 'slow' in the 'schedule_extended_test' -# pipeline. -.tezt-slow: - extends: - - .tezt_tests - variables: - # Run only slow tests - # TODO: https://gitlab.com/tezos/tezos/-/issues/7063 - # The deselection of Paris [test_adaptive_issuance_launch.ml] - # should be removed once the fixes to its slowness has been - # snapshotted from Alpha. - TESTS: "/ci_disabled /cloud slow --not-file src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml" - TEZT_PARALLEL: 3 - TEZT_VARIANT: "-slow" - parallel: 10 diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml deleted file mode 100644 index 155b83c70e84466a63960307aba7e411fe8a5f15..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/tezt.yml +++ /dev/null @@ -1,89 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -tezt: - extends: - - .tezt_tests - variables: - # Exclude tests with tags 'ci_disabled', 'flaky', 'memory_{3,4}k', - # 'time_sensitive' and 'slow'. - # See tezt/lib_tezos/tag.mli for more information. - TESTS: "/ci_disabled /flaky /memory_3k /memory_4k /time_sensitive /slow /cloud" - # the -j option of tezt - TEZT_PARALLEL: 3 - # WARNING: if you increase the number of parallel jobs, you need to - # update .gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml with the - # new list of jobs. - parallel: 60 - -# the following memory hungry tests are executed with -j 1 -tezt-memory-4k: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud memory_4k" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-memory_4k" - parallel: 4 - -tezt-memory-3k: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud memory_3k" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-memory_3k" - -# the following tests are executed with -j 1 to ensure that other -# tests do not affect their executions. However, these tests are not -# particularly cpu/memory-intensive hence they do not need to run on a -# particular machine contrary to performance regression tests. -tezt-time-sensitive: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud time_sensitive" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-time_sensitive" - -# these are tezt tests as above, but run using the static binaries -tezt:static-binaries: - stage: test - extends: - # Expansion of .integration_template but without coverage. - - .default_settings_template - - .image_template__runtime_e2etest_dependencies - - .tezt_template - - .rules__octez_changes - dependencies: - - select_tezts - # Fetch src/proto_*/parameters/*.json and tezt/tests/main.exe from - # oc.build_x86_64-exp-dev-extra - - oc.build_x86_64-exp-dev-extra - # And fetch the static executables from build:static - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - needs: - - select_tezts - - oc.build_x86_64-exp-dev-extra - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - variables: - TESTS: "/ci_disabled /cloud cli" - before_script: - - mv octez-binaries/x86_64/octez-* . - -# Note: if you reactivate this test and if you keep it manual, put it in the "manual" stage. -# -#tezt:manual:migration: -# extends: -# - .test_template -# before_script: -# - export TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=Y -# - curl -s https://api.github.com/repos/Phlogi/tezos-snapshots/releases/latest | jq -r ".assets[] | select(.name) | .browser_download_url" | grep roll | xargs wget -q -# - block_hash=$(echo mainnet.roll.* | sed -r 's/mainnet\.roll\.[0-9_-]+\.(.*)\.[0-9]+\.chain\.xz/\1/g') -# - cat mainnet.roll.* | xz -d -v -T0 > mainnet.rolling -# - scripts/prepare_migration_test.sh auto mainnet.rolling "$block_hash" -# script: -# - dune exec ./tezt/manual_tests/main.exe -- migration --color --log-buffer-size 5000 --log-file tezt-migration.log -# artifacts: -# when: always -# paths: -# - tezt-migration.log -# expire_in: 30 days diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 651c5a60154819c86be78b6e10fef1db524f0704..17bbdb1ba8415407d7a5c8f75609639c6fde79d3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -45,7 +45,7 @@ include: - .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml - .gitlab/ci/jobs/test/install_octez-before_merging.yml - - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt-before_merging.yml - .gitlab/ci/jobs/test/tezt-slow-before_merging.yml - .gitlab/ci/jobs/test/tezt-flaky-before_merging.yml - .gitlab/ci/jobs/test/test_kernels.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d646d102053ceb6c628ae695b935b0accb8bdb3d..88b0011f3a1e48277db943102f088293e3d1ee17 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -32,9 +32,9 @@ include: # Stage: test - .gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml # Flaky tezts - - .gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml + - .gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml # Slow tezts - - .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml + - .gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml # Tests that may not have been run in before_merging pipeline # because of absence of certain changes - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml @@ -50,7 +50,7 @@ include: - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml - .gitlab/ci/jobs/test/test_kernels.yml - .gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml - .gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 6463d2aad3e6e6cd0282973f7e34e52de7be676b..caa217a4f79d2d084efeaef5a5c1ac087892e9b8 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -39,6 +39,12 @@ open Common unconditional variant [schedule_extended_test]. *) type code_verification_pipeline = Before_merging | Schedule_extended_test +(** Manual trigger configuration for [make_rules] *) +type manual = + | No (** Do not add rule for manual trigger. *) + | Yes (** Add rule for manual trigger. *) + | On_changes of string list (** Add manual trigger on certain [changes:] *) + (* [make_rules] makes rules for jobs that are: - automatic in scheduled pipelines; - conditional in [before_merging] pipelines. @@ -47,19 +53,24 @@ type code_verification_pipeline = Before_merging | Schedule_extended_test set to [true] to ensure that we only run the job in case previous jobs succeeded (setting [when: on_success]). + If [label] is set, add rule that selects the job in + [Before_merging] pipelines for merge requests with the given + label. Rules for manual triggers can be configured using + [manual]. + If [label], [changes] and [manual] are omitted, then rules will enable the job [On_success] in the [before_merging] pipeline. This is safe, but prefer specifying a [changes] clause if possible. *) -let make_rules ?label ?changes ?(manual = false) ?(dependent = false) - pipeline_type = +let make_rules ?label ?changes ?(manual = No) ?(dependent = false) pipeline_type + = match pipeline_type with | Schedule_extended_test -> (* The scheduled pipeline always runs all jobs unconditionally -- unless they are dependent on a previous, non-trigger job, in the pipeline. *) [job_rule ~when_:(if dependent then On_success else Always) ()] - | Before_merging -> + | Before_merging -> ( (* MR labels can be used to force tests to run. *) (match label with | Some label -> @@ -70,7 +81,11 @@ let make_rules ?label ?changes ?(manual = false) ?(dependent = false) | None -> [] | Some changes -> [job_rule ~changes ~when_:On_success ()]) (* For some tests, it can be relevant to have a manual trigger. *) - @ if manual then [job_rule ~when_:Manual ()] else [] + @ + match manual with + | No -> [] + | Yes -> [job_rule ~when_:Manual ()] + | On_changes changes -> [job_rule ~when_:Manual ~changes ()]) type opam_package_group = Executable | All @@ -358,6 +373,142 @@ let image_of_distribution = function | Ubuntu_jammy -> Images.ubuntu_jammy | Fedora_37 -> Images.fedora_37 +let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name + ~(tezt_tests : Tezt_core.TSL_AST.t) ?(retry = 2) ?(tezt_retry = 1) + ?(tezt_parallel = 1) ?(tezt_variant = "") + ?(before_script = before_script ~source_version:true ~eval_opam:true []) + ~dependencies () : tezos_job = + let variables = + [ + ("JUNIT", "tezt-junit.xml"); + ("TEZT_VARIANT", tezt_variant); + ("TESTS", Tezt_core.TSL.show tezt_tests); + ("TEZT_RETRY", string_of_int tezt_retry); + ("TEZT_PARALLEL", string_of_int tezt_parallel); + ] + in + let artifacts = + artifacts + ~reports:(reports ~junit:"$JUNIT" ()) + [ + "selected_tezts.tsv"; + "tezt.log"; + "tezt-*.log"; + "tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json"; + "$JUNIT"; + ] + (* The record artifacts [tezt-results-$CI_NODE_INDEX.json] + should be stored for as long as a given commit on master is + expected to be HEAD in order to support auto-balancing. At + the time of writing, we have approximately 6 merges per day, + so 1 day should more than enough. However, we set it to 3 + days to keep records over the weekend. The tezt artifacts + (including records and coverage) take up roughly 2MB / + job. Total artifact storage becomes [N*P*T*W] where [N] is + the days of retention (3 atm), [P] the number of pipelines + per day (~200 atm), [T] the number of Tezt jobs per pipeline + (60) and [W] the artifact size per tezt job (2MB). This makes + 35GB which is less than 0.5% than our + {{:https://gitlab.com/tezos/tezos/-/artifacts}total artifact + usage}. *) + ~expire_in:(Duration (Days 7)) + ~when_:Always + in + let print_variables = + [ + "TESTS"; + "JUNIT"; + "CI_NODE_INDEX"; + "CI_NODE_TOTAL"; + "TEZT_PARALLEL"; + "TEZT_VARIANT"; + ] + in + let retry = if retry = 0 then None else Some retry in + job + ~__POS__ + ~image:Images.runtime_e2etest_dependencies + ~name + ?parallel + ~tags + ~stage:Stages.test + ?rules + ~artifacts + ~variables + ~dependencies + ?retry + ~before_script + [ + (* Print [print_variables] in a shell-friendly manner for easier debugging *) + "echo \"" + ^ String.concat + " " + (List.map (fun var -> sf {|%s=\"${%s}\"|} var var) print_variables) + ^ "\""; + (* Store the list of tests that have been scheduled for execution for later debugging. + It is imperative this this first call to tezt receives any flags passed to the + second call that affect test selection.Note that TESTS must be quoted (here and below) + since it will contain e.g. '&&' which we want to interpreted as TSL and not shell + syntax. *) + "./scripts/ci/tezt.sh \"${TESTS}\" --from-record tezt/records --job \ + ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv"; + (* For Tezt tests, there are multiple timeouts: + - --global-timeout is the internal timeout of Tezt, which only works if tests + are cooperative; + - the "timeout" command, which we set to send SIGTERM to Tezt 60s after --global-timeout + in case tests are not cooperative; + - the "timeout" command also sends SIGKILL 60s after having sent SIGTERM in case + Tezt is still stuck; + - the CI timeout. + The use of the "timeout" command is to make sure that Tezt eventually exits, + because if the CI timeout is reached, there are no artefacts, + and thus no logs to investigate. + See also: https://gitlab.com/gitlab-org/gitlab/-/issues/19818 *) + "./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh \ + \"${TESTS}\" --color --log-buffer-size 5000 --log-file tezt.log \ + --global-timeout 1800 --on-unknown-regression-files fail --junit \ + ${JUNIT} --from-record tezt/records --job \ + ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record \ + tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count \ + ${TEZT_PARALLEL} --retry ${TEZT_RETRY}"; + "if [ -n \"${BISECT_FILE:-}\" ]; then ./scripts/ci/merge_coverage.sh; fi"; + ] + +(** Tezt tag selector string. + + It returns a TSL expression that: + - always deselects tags with [ci_disabled]; + - selects, respectively deselects, the tests with the tags + [memory_3k], [memory_4k], [time_sensitive], [slow] or [cloud], + depending on the value of the corresponding function + argument. These arguments all default to false. + + See [src/lib_test/tag.mli] for a description of the above tags. + + The list of TSL expressions [and_] are appended to the final + selector, allowing to modify the selection further. *) +let tezt_tests ?(memory_3k = false) ?(memory_4k = false) + ?(time_sensitive = false) ?(slow = false) ?(cloud = false) + (and_ : Tezt_core.TSL_AST.t list) : Tezt_core.TSL_AST.t = + let tags = + [ + (false, "ci_disabled"); + (memory_3k, "memory_3k"); + (memory_4k, "memory_4k"); + (time_sensitive, "time_sensitive"); + (slow, "slow"); + (cloud, "cloud"); + ] + in + let positive, negative = List.partition fst tags in + let positive = List.map snd positive in + let negative = List.map snd negative in + Tezt_core.( + TSL.conjunction + @@ List.map (fun tag -> TSL_AST.Has_tag tag) positive + @ List.map (fun tag -> TSL_AST.Not (Has_tag tag)) negative + @ and_) + (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) let jobs pipeline_type = @@ -496,7 +647,7 @@ let jobs pipeline_type = let job_docker_rust_toolchain = job_docker_rust_toolchain ~__POS__ - ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:true ()) + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:Yes ()) ~dependencies:dependencies_needs_trigger () |> job_external_split @@ -545,7 +696,7 @@ let jobs pipeline_type = () |> job_external_split in - let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in + let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:Yes () in let job_build_arm64_release : Tezos_ci.tezos_job = job_build_arm64_release ~rules:build_arm_rules () |> job_external_split in @@ -553,20 +704,118 @@ let jobs pipeline_type = job_build_arm64_exp_dev_extra ~rules:build_arm_rules () |> job_external_split in + (* Used in [before_merging] and [schedule_extended_tests]. + + Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_select_tezts : tezos_job = + job + ~__POS__ + ~name:"select_tezts" + (* We need: + - Git (to run git diff) + - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.build + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + (script_propagate_exit_code "scripts/ci/select_tezts.sh") + ~allow_failure:(With_exit_codes [17]) + ~artifacts: + (artifacts + ~expire_in:(Duration (Days 3)) + ~when_:Always + ["selected_tezts.tsl"]) + |> job_external_once + in + let job_build_kernels : tezos_job = + job + ~__POS__ + ~name:"oc.build_kernels" + ~image:Images.rust_toolchain + ~stage:Stages.build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~dependent:true ()) + [ + "make -f kernels.mk build"; + "make -f etherlink.mk evm_kernel.wasm"; + "make -C src/risc_v risc-v-sandbox risc-v-dummy.elf"; + "make -C src/risc_v/tests/ build"; + ] + ~artifacts: + (artifacts + ~name:"build-kernels-$CI_COMMIT_REF_SLUG" + ~expire_in:(Duration (Days 1)) + ~when_:On_success + [ + "evm_kernel.wasm"; + "smart-rollup-installer"; + "sequenced_kernel.wasm"; + "tx_kernel.wasm"; + "tx_kernel_dal.wasm"; + "dal_echo_kernel.wasm"; + "src/risc_v/risc-v-sandbox"; + "src/risc_v/risc-v-dummy.elf"; + "src/risc_v/tests/inline_asm/rv64-inline-asm-tests"; + ]) + ~cache: + [ + {key = "kernels"; paths = ["cargo/"]}; + {key = "kernels-sccache"; paths = ["_sccache"]}; + ] + |> enable_kernels |> enable_sccache |> job_external_split + in + (* Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_tezt_fetch_records : tezos_job = + job + ~__POS__ + ~name:"oc.tezt:fetch-records" + ~image:Images.runtime_build_dependencies + ~stage:Stages.build + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~rules:(make_rules ~changes:changeset_octez ()) + [ + "dune exec scripts/ci/update_records/update.exe -- --log-file \ + tezt-fetch-records.log --from last-successful-schedule-extended-test \ + --info"; + ] + ~after_script:["./scripts/ci/filter_corrupted_records.sh"] + (* Allow failure of this job, since Tezt can use the records + stored in the repo as backup for balancing. *) + ~allow_failure:Yes + ~artifacts: + (artifacts + ~expire_in:(Duration (Hours 4)) + ~when_:Always + [ + "tezt-fetch-records.log"; + "tezt/records/*.json"; + (* Keep broken records for debugging *) + "tezt/records/*.json.broken"; + ]) + |> job_external_split + in + let job_static_x86_64_experimental = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration + of pipelines. So we start this job as early as possible, + without waiting for sanity_ci. *) + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + () + |> job_external_split + in let build = - let job_static_x86_64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration - of pipelines. So we start this job as early as possible, - without waiting for sanity_ci. *) - ~dependencies:dependencies_needs_trigger - ~rules:(make_rules ~changes:changeset_octez ()) - () - |> job_external_split - in (* TODO: The code is a bit convulted here because these jobs are either in the build or in the manual stage depending on the pipeline type. However, we can put them in the build stage on @@ -597,105 +846,6 @@ let jobs pipeline_type = ["dune build @check"] |> job_external_split in - let job_build_kernels : tezos_job = - job - ~__POS__ - ~name:"oc.build_kernels" - ~image:Images.rust_toolchain - ~stage:Stages.build - ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) - ~rules: - (make_rules ~changes:changeset_octez_or_kernels ~dependent:true ()) - [ - "make -f kernels.mk build"; - "make -f etherlink.mk evm_kernel.wasm"; - "make -C src/risc_v risc-v-sandbox risc-v-dummy.elf"; - "make -C src/risc_v/tests/ build"; - ] - ~artifacts: - (artifacts - ~name:"build-kernels-$CI_COMMIT_REF_SLUG" - ~expire_in:(Duration (Days 1)) - ~when_:On_success - [ - "evm_kernel.wasm"; - "smart-rollup-installer"; - "sequenced_kernel.wasm"; - "tx_kernel.wasm"; - "tx_kernel_dal.wasm"; - "dal_echo_kernel.wasm"; - "src/risc_v/risc-v-sandbox"; - "src/risc_v/risc-v-dummy.elf"; - "src/risc_v/tests/inline_asm/rv64-inline-asm-tests"; - ]) - ~cache: - [ - {key = "kernels"; paths = ["cargo/"]}; - {key = "kernels-sccache"; paths = ["_sccache"]}; - ] - |> enable_kernels |> enable_sccache |> job_external_split - in - (* Fetch records for Tezt generated on the last merge request pipeline - on the most recently merged MR and makes them available in artifacts - for future merge request pipelines. *) - let job_tezt_fetch_records : tezos_job = - job - ~__POS__ - ~name:"oc.tezt:fetch-records" - ~image:Images.runtime_build_dependencies - ~stage:Stages.build - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ~rules:(make_rules ~changes:changeset_octez ()) - [ - "dune exec scripts/ci/update_records/update.exe -- --log-file \ - tezt-fetch-records.log --from \ - last-successful-schedule-extended-test --info"; - ] - ~after_script:["./scripts/ci/filter_corrupted_records.sh"] - (* Allow failure of this job, since Tezt can use the records - stored in the repo as backup for balancing. *) - ~allow_failure:Yes - ~artifacts: - (artifacts - ~expire_in:(Duration (Hours 4)) - ~when_:Always - [ - "tezt-fetch-records.log"; - "tezt/records/*.json"; - (* Keep broken records for debugging *) - "tezt/records/*.json.broken"; - ]) - |> job_external_split - in - (* Used in [before_merging] and [schedule_extended_tests]. - - Fetch records for Tezt generated on the last merge request pipeline - on the most recently merged MR and makes them available in artifacts - for future merge request pipelines. *) - let job_select_tezts : tezos_job = - job - ~__POS__ - ~name:"select_tezts" - (* We need: - - Git (to run git diff) - - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) - ~image:Images.runtime_prebuild_dependencies - ~stage:Stages.build - ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) - ["scripts/ci/select_tezts.sh || exit $?"] - ~allow_failure:(With_exit_codes [17]) - ~artifacts: - (artifacts - ~expire_in:(Duration (Days 3)) - ~when_:Always - ["selected_tezts.tsl"]) - |> job_external_once - in [ job_docker_rust_toolchain; job_docker_client_libs_dependencies; @@ -1012,7 +1162,7 @@ let jobs pipeline_type = ["docs/introduction/install*.sh"; "docs/introduction/compile*.sh"] in let install_octez_rules = - make_rules ~changes:changeset_install_jobs ~manual:true () + make_rules ~changes:changeset_install_jobs ~manual:Yes () in let job_install_bin ~__POS__ ~name ?allow_failure ?(rc = false) distribution = @@ -1041,7 +1191,7 @@ let jobs pipeline_type = ~name:"oc.install_opam_focal" ~image:Images.opam_ubuntu_focal ~dependencies:dependencies_needs_trigger - ~rules:(make_rules ~manual:true ()) + ~rules:(make_rules ~manual:Yes ()) ~allow_failure:Yes ~stage:Stages.test (* The default behavior of opam is to use `nproc` to determine its level of @@ -1119,6 +1269,176 @@ let jobs pipeline_type = ] |> jobs_external_split ~path:"test/install_octez" in + (* Tezt jobs. + + The tezt jobs are split into a set of special-purpose jobs running the + tests of the corresponding tag: + - [tezt-memory-3k]: runs the jobs with tag [memory_3k], + - [tezt-memory-4k]: runs the jobs with tag [memory_4k], + - [tezt-time_sensitive]: runs the jobs with tag [time-sensitive], + - [tezt-slow]: runs the jobs with tag [slow]. + - [tezt-flaky]: runs the jobs with tag [flaky] and + none of the tags above. + + and a job [tezt] that runs all remaining tests (excepting those + that are tagged [ci_disabled], that are disabled in the CI.) + + There is an implicit rule that the Tezt tags [memory_3k], + [memory_4k], [time_sensitive], [slow] and [cloud] are mutually + exclusive. The [flaky] tag is not exclusive to these tags. If + e.g. a test has both tags [slow] and [flaky], it will run in + [tezt-slow], to prevent flaky tests to run in the [tezt-flaky] + job if they also have another special tag. Tests tagged [cloud] are + meant to be used with Tezt cloud (see [tezt/lib_cloud/README.md]) and + do not run in the CI. + + For more information on tags, see [src/lib_test/tag.mli]. *) + let tezt_dependencies = + Dependent + [ + Artifacts job_select_tezts; + Artifacts job_build_x86_64_release; + Artifacts job_build_x86_64_exp_dev_extra; + Artifacts job_build_kernels; + Artifacts job_tezt_fetch_records; + ] + in + let job_tezt_flaky : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-flaky" + ~tezt_tests:(tezt_tests [Has_tag "flaky"]) + ~tezt_variant:"-flaky" + (* To handle flakiness, consider tweaking [~tezt_parallel] (passed to + Tezt's '--job-count'), and [~tezt_retry] (passed to Tezt's + '--retry') *) + ~retry:2 + ~tezt_retry:3 + ~tezt_parallel:1 + ~dependencies:tezt_dependencies + ~rules: + (* This job can only be manually triggered when it's + artifact dependencies exists, which they do when + [changeset_octez] is changed. *) + (make_rules ~dependent:true ~manual:(On_changes changeset_octez) ()) + () + |> enable_coverage_output_artifact |> job_external_split + in + let job_tezt_slow : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-slow" + ~rules: + (* See comment for [job_tezt_flaky] *) + (make_rules ~dependent:true ~manual:(On_changes changeset_octez) ()) + ~tezt_tests: + (tezt_tests + ~slow:true + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7063 + The deselection of Paris [test_adaptive_issuance_launch.ml] + should be removed once the fixes to its slowness has been + snapshotted from Alpha. *) + [ + Not + (String_predicate + ( File, + Is + "src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml" + )); + ]) + ~tezt_variant:"-slow" + ~retry:2 + ~tezt_parallel:3 + ~parallel:(Vector 10) + ~dependencies:tezt_dependencies + () + |> job_external_split + in + let jobs_tezt = + let rules = make_rules ~dependent:true ~changes:changeset_octez () in + let coverage_expiry = Duration (Days 3) in + let tezt : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt" + (* Exclude all tests with tags in [tezt_tags_always_disable] or + [tezt_tags_exclusive_tags]. *) + ~tezt_tests:(tezt_tests [Not (Has_tag "flaky")]) + ~tezt_parallel:3 + ~parallel:(Vector 60) + ~rules + ~dependencies:tezt_dependencies + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_memory_4k : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-memory-4k" + ~tezt_tests:(tezt_tests ~memory_4k:true []) + ~tezt_variant:"-memory_4k" + ~parallel:(Vector 4) + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_memory_3k : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-memory-3k" + ~tezt_tests:(tezt_tests ~memory_3k:true []) + ~tezt_variant:"-memory_3k" + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_time_sensitive : tezos_job = + (* the following tests are executed with [~tezt_parallel:1] to ensure + that other tests do not affect their executions. However, these + tests are not particularly cpu/memory-intensive hence they do not + need to run on a particular machine contrary to performance + regression tests. *) + job_tezt + ~__POS__ + ~name:"tezt-time-sensitive" + ~tezt_tests:(tezt_tests ~time_sensitive:true []) + ~tezt_variant:"-time_sensitive" + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_static_binaries : tezos_job = + job_tezt + ~__POS__ + ~tags:["gcp"] + ~name:"tezt:static-binaries" + ~tezt_tests:(tezt_tests [Has_tag "cli"; Not (Has_tag "flaky")]) + ~tezt_parallel:3 + ~retry:0 + ~dependencies: + (Dependent + [ + Artifacts job_select_tezts; + Artifacts job_build_x86_64_exp_dev_extra; + Artifacts job_static_x86_64_experimental; + Artifacts job_tezt_fetch_records; + ]) + ~rules + ~before_script:(before_script ["mv octez-binaries/x86_64/octez-* ."]) + () + in + [ + tezt; + tezt_memory_4k; + tezt_memory_3k; + tezt_time_sensitive; + tezt_static_binaries; + ] + |> jobs_external_split ~path:"test/tezt" + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -1132,8 +1452,10 @@ let jobs pipeline_type = job_oc_script_test_release_versions; job_oc_script_b58_prefix; job_oc_test_liquidity_baking_scripts; + job_tezt_flaky; + job_tezt_slow; ] - @ jobs_unit @ jobs_install_octez + @ jobs_unit @ jobs_install_octez @ jobs_tezt @ match pipeline_type with | Before_merging -> @@ -1146,7 +1468,7 @@ let jobs pipeline_type = ~dependencies:dependencies_needs_trigger (* ./scripts/ci/check_commit_messages.sh exits with code 65 when a git history contains invalid commits titles in situations where that is allowed. *) - ["./scripts/ci/check_commit_messages.sh || exit $?"] + (script_propagate_exit_code "./scripts/ci/check_commit_messages.sh") ~allow_failure:(With_exit_codes [65]) |> job_external in diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 5abfd2be1f7d275193431eb070740b22e9d7868b..db162cbbb86b73dcd6eb53236c2d6ee0d7f5793f 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -214,6 +214,21 @@ let before_script ?(take_ownership = false) ?(source_version = false) @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" @ before_script +(** A [script:] that executes [script] and propagates its exit code. + + This might seem like a noop but is in fact necessary to please his + majesty GitLab. + + For more info, see: + - https://gitlab.com/tezos/tezos/-/merge_requests/9923#note_1538894754; + - https://gitlab.com/tezos/tezos/-/merge_requests/12141; and + - https://gitlab.com/groups/gitlab-org/-/epics/6074 + + TODO: replace this with [FF_USE_NEW_BASH_EVAL_STRATEGY=true], see + {{:https://docs.gitlab.com/runner/configuration/feature-flags.html}GitLab + Runner feature flags}. *) +let script_propagate_exit_code script = [script ^ " || exit $?"] + let opt_var name f = function Some value -> [(name, f value)] | None -> [] (** Add variable for bisect_ppx instrumentation. diff --git a/ci/bin/dune b/ci/bin/dune index 7438f8014a36eb66a5596a91e2fbb36da38b234e..5063369192dbfd2627bc7d943ffce457caff0f2d 100644 --- a/ci/bin/dune +++ b/ci/bin/dune @@ -6,7 +6,8 @@ (libraries gitlab_ci yaml - unix) + unix + tezt.core) (flags (:standard) -open Gitlab_ci.Base)) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index c1d817a1505e5c420b22ace46de03f5e5fa043d7..e1c852a15cbea2d25ec8e3549aec326360470b10 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -199,12 +199,6 @@ let () = | ".gitlab/ci/jobs/test/test_kernels.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-before_merging.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-flaky-before_merging.yml" - | ".gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-flaky.yml" - | ".gitlab/ci/jobs/test/tezt-slow-before_merging.yml" - | ".gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-slow.yml" | ".gitlab/ci/jobs/test/tezt.yml" | ".gitlab/ci/pipelines/before_merging.yml" | ".gitlab/ci/pipelines/schedule_extended_test.yml" -> true diff --git a/manifest/product_ciao.ml b/manifest/product_ciao.ml index 17099f0d09bef0aa5c33b818accd4a7673fda0ef..86c35968e9630f66823fe041d61ce476b959f0cf 100644 --- a/manifest/product_ciao.ml +++ b/manifest/product_ciao.ml @@ -30,5 +30,5 @@ let _ci_bin_main = ~opam:"" ~path:"ci/bin" ~bisect_ppx:No - ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"; yaml; unix] + ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"; yaml; unix; tezt_core_lib] ~release_status:Unreleased