diff --git a/.gitlab/ci/jobs/build/select_tezts.yml b/.gitlab/ci/jobs/build/select_tezts.yml index 23a2f139fe66d6f30533b1aae0d51a30e23c4e74..70fe7c9fe104cb6b1527f7601b4f1b2da1a452c0 100644 --- a/.gitlab/ci/jobs/build/select_tezts.yml +++ b/.gitlab/ci/jobs/build/select_tezts.yml @@ -1,20 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + select_tezts: - extends: - - .default_settings_template - # We need: - # - Git (to run git diff) - # - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) - - .image_template__runtime_prebuild_dependencies + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} stage: build - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - scripts/ci/select_tezts.sh || exit $? + tags: + - gcp + dependencies: [] allow_failure: exit_codes: 17 + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - scripts/ci/select_tezts.sh || exit $? artifacts: - when: always - paths: - - selected_tezts.tsl expire_in: 3 days + paths: + - selected_tezts.tsl + when: always diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 1525e775d9c936f7ba74beb900f3f89cde67b0c8..3c9fde318195a99a0519e3532a31d4980f5acebd 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -25,3 +25,5 @@ image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} .image_template__rust_toolchain: image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +.image_template__semgrep_agent: + image: returntocorp/semgrep-agent:sha-c6cd7cf diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 781bb20a2e8b6560f5defc30448e7361f72864d4..c831e40d43901115faaf185a23be6ffea07af80e 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -151,16 +151,6 @@ - .gitlab-ci.yml when: manual -.rules__octez_kaitai_e2e_changes: - rules: - - changes: - - images/**/* - - src/**/* - - client-libs/*kaitai*/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - # Add variable for bisect_ppx instrumentation. # # This template should be extended by jobs that build OCaml targets @@ -237,20 +227,6 @@ # Load the environment poetry previously created in the docker image. # Give access to the Python dependencies/executables - . $HOME/.venv/bin/activate - rules: - # The linting job runs over the set of [source_directories] - # defined in [scripts/lint.sh] that must be included here: - - changes: - - src/**/* - - tezt/**/* - - devtools/**/* - - scripts/**/* - - docs/**/* - - contrib/**/* - - client-libs/**/* - - etherlink/**/* - - .gitlab-ci.yml - - .gitlab/**/* .test_etherlink_kernel: extends: diff --git a/.gitlab/ci/jobs/test/commit_titles.yml b/.gitlab/ci/jobs/test/commit_titles.yml index b636b7a03760b688eae229cb9eb67135f99ad700..e2d913711e63ec58247c496119069016a897c7b3 100644 --- a/.gitlab/ci/jobs/test/commit_titles.yml +++ b/.gitlab/ci/jobs/test/commit_titles.yml @@ -1,14 +1,16 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + commit_titles: - extends: - - .default_settings_template - - .image_template__runtime_prebuild_dependencies - - .needs__trigger - stage: "test" - script: - # Check commit messages - - ./scripts/ci/check_commit_messages.sh || exit $? + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] allow_failure: - # ./scripts/ci/check_commit_messages.sh exits with code 65 when a - # git history contains invalid commits titles in situations where - # that is allowed. - exit_codes: [65] + exit_codes: 65 + script: + - ./scripts/ci/check_commit_messages.sh || exit $? diff --git a/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..b05c7cc774f104ccbceb03aa477143b92dc9eae6 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - images/**/* + - src/**/* + - client-libs/*kaitai*/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..5302066d9cf299b3553cfa02c3093cf20e4ccdc1 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml @@ -0,0 +1,18 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/jobs/test/kaitai_checks.yml b/.gitlab/ci/jobs/test/kaitai_checks.yml deleted file mode 100644 index 13b4f2c81bc5fef933878993b610123ddf6f71ef..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/kaitai_checks.yml +++ /dev/null @@ -1,10 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# check that ksy files are still up-to-date with octez -kaitai_checks: - extends: - - .test_template - - .needs__trigger - - .rules__octez_kaitai_e2e_changes - script: - - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..408cc9ab33001456d086225c87124bd19e9761e4 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} + stage: test + tags: + - gcp + rules: + - changes: + - images/**/* + - src/**/* + - client-libs/*kaitai*/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:client-libs-dependencies + - kaitai_checks + dependencies: + - oc.docker:client-libs-dependencies + before_script: + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + script: + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + client-libs/kaitai-struct-files/input 2>/dev/null diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..e5a1817cb557bb5dbbd87e34446f0b52fc55dcb8 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml @@ -0,0 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} + stage: test + tags: + - gcp + rules: + - when: on_success + needs: + - oc.docker:client-libs-dependencies + - kaitai_checks + dependencies: + - oc.docker:client-libs-dependencies + before_script: + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + script: + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + client-libs/kaitai-struct-files/input 2>/dev/null diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml deleted file mode 100644 index 9cfc357e31442384d31d9f635f1808409772849b..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml +++ /dev/null @@ -1,23 +0,0 @@ -kaitai_e2e_checks: - extends: - - .default_settings_template - - .image_template__client_libs_dependencies - - .rules__octez_kaitai_e2e_changes - stage: test - needs: [oc.docker:client-libs-dependencies, kaitai_checks] - dependencies: [oc.docker:client-libs-dependencies] - before_script: - - . ./scripts/version.sh - # TODO: https://gitlab.com/tezos/tezos/-/issues/5026 - # As observed for the `unit:js_components` running `npm i` - # everytime we run a job is inefficient. - # - # The benefit of this approach is that we specify node version - # and npm dependencies (package.json) in one place, and that the local - # environment is then the same as CI environment. - - . ./scripts/install_build_deps.js.sh - script: - - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh - client-libs/kaitai-struct-files/files - client-libs/kaitai-struct-files/input - 2>/dev/null diff --git a/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..0ae8eaff2337566a091a6a937ca7d4636ff6928e --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml @@ -0,0 +1,28 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +misc_opam_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/opam-check.sh + retry: 2 diff --git a/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..33cfd7d9cc02ec06dea985b0c3f9969e37430474 --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml @@ -0,0 +1,17 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +misc_opam_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/opam-check.sh + retry: 2 diff --git a/.gitlab/ci/jobs/test/misc_opam_checks.yml b/.gitlab/ci/jobs/test/misc_opam_checks.yml deleted file mode 100644 index ea65f24bebd7bc33792a9fdbf697e5eb0db67a4d..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/misc_opam_checks.yml +++ /dev/null @@ -1,10 +0,0 @@ -misc_opam_checks: - extends: - - .test_template - - .needs__trigger - # This job is flaky due to e.g. network issues. - - .oc.template__retry_flaky - - .rules__octez_changes - script: - # checks that all deps of opam packages are already installed - - ./scripts/opam-check.sh diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..98eb7dcfad50b98209ca222efa5635b4bacc9607 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.check_lift_limits_patch: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/bin_tps_evaluation/lift_limits.patch + - src/proto_alpha/lib_protocol/main.ml + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = + "src/proto_alpha/lib_protocol/main.ml" ]' + - git apply src/bin_tps_evaluation/lift_limits.patch + - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..3da2f721d70d7240f053b771d8aca6b344833e64 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml @@ -0,0 +1,19 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.check_lift_limits_patch: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = + "src/proto_alpha/lib_protocol/main.ml" ]' + - git apply src/bin_tps_evaluation/lift_limits.patch + - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml deleted file mode 100644 index a1794aae470a80c8ace94754f02f5ac8aa842e1e..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml +++ /dev/null @@ -1,18 +0,0 @@ -oc.check_lift_limits_patch: - extends: - - .test_template - - .needs__trigger - rules: - - changes: - - src/bin_tps_evaluation/lift_limits.patch - - src/proto_alpha/lib_protocol/main.ml - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - script: - # Check that the patch only modifies the - # src/proto_alpha/lib_protocol. If not, the rules above have to be - # updated. - - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = "src/proto_alpha/lib_protocol/main.ml" ]' - - git apply src/bin_tps_evaluation/lift_limits.patch - - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml index 7e808a0a7b7d2d29d7a7ab5c2b05ce5533424c37..0c71812921099568b0c3b5a42c657e9070b69f3d 100644 --- a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml @@ -1,7 +1,33 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + oc.misc_checks: - extends: - - .oc.misc_checks + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - tezt/**/* + - devtools/**/* + - scripts/**/* + - docs/**/* + - client-libs/**/* + - etherlink/**/* + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate script: - - ./scripts/ci/lint_misc_check.sh - - ./scripts/ci/lint_check_licenses.sh - - scripts/check_wasm_pvm_regressions.sh check + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - ./scripts/ci/lint_check_licenses.sh diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml b/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml deleted file mode 100644 index a21a310086f392eb77442ed64ad3dd152daeaf67..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml +++ /dev/null @@ -1,8 +0,0 @@ -# As 'oc.misc_checks' in 'oc.misc_checks:before_merging.yml' but does -# not check the license header of newly added OCaml files. -oc.misc_checks: - extends: - - .oc.misc_checks - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..a65b4c74d8cf594c3c3b3c8406f8b6b07e885308 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml @@ -0,0 +1,19 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.misc_checks: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check diff --git a/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..10121659decff87f26deb25a4433a964572c9817 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml @@ -0,0 +1,25 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - tezt/**/* + - devtools/**/* + - scripts/semgrep/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..439fcc6cd6ab8f7df91c775cd0e942757fa75ca8 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml @@ -0,0 +1,15 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/jobs/test/oc.semgrep.yml b/.gitlab/ci/jobs/test/oc.semgrep.yml deleted file mode 100644 index 53e0f3e4602bf5be7925b59942bf0e6fa069f296..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/test/oc.semgrep.yml +++ /dev/null @@ -1,26 +0,0 @@ -oc.semgrep: - extends: - - .default_settings_template - - .needs__trigger - rules: - - changes: - - src/**/* - - tezt/**/* - - devtools/**/* - - scripts/semgrep/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - # We specify the image by hash to avoid flakiness. Indeed, if we took the - # latest release, then an update in the parser or analyser could result in new - # errors being found even if the code doesn't change. This would place the - # burden for fixing the code on the wrong dev (the devs who happen to open an - # MR coinciding with the semgrep update rather than the dev who wrote the - # infringing code in the first place). - # Update the hash in scripts/semgrep/README.md too when updating it here - # Last update: 20212-01-03 - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index c746bdb1f29bd40312589591e7bd94b1551cb918..21e5caf41c295a62c84172fe768b96c55e25342e 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -30,13 +30,13 @@ include: - .gitlab/ci/jobs/build/select_tezts.yml # Stage: test - - .gitlab/ci/jobs/test/kaitai_checks.yml - - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml - - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml + - .gitlab/ci/jobs/test/kaitai_checks-before_merging.yml + - .gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml + - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml - - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml - .gitlab/ci/jobs/test/commit_titles.yml - - .gitlab/ci/jobs/test/oc.semgrep.yml + - .gitlab/ci/jobs/test/oc.semgrep-before_merging.yml - .gitlab/ci/jobs/test/oc.unit.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d62f61648b3145b767166fba3cb2e0828caa798f..2e4748ef15f6a7d1700d49b778296adb79515187 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -37,12 +37,12 @@ include: - .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml # Tests that may not have been run in before_merging pipeline # because of absence of certain changes - - .gitlab/ci/jobs/test/kaitai_checks.yml - - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml - - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml - - .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml - - .gitlab/ci/jobs/test/misc_opam_checks.yml - - .gitlab/ci/jobs/test/oc.semgrep.yml + - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml + - .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml + - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml + - .gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml + - .gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml + - .gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.unit.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 19ee2272f4d3821899ef6d47e430e85b8c36b43b..f84d462131ba0fcf9e5256af02581dafad5cb321 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -238,12 +238,11 @@ let jobs pipeline_type = ] |> job_external in - (* TODO: put job_trigger here when full pipeline is generated *) (* TODO: the dependency on job_trigger does not have to be optional *) - ([], Dependent [Optional job_trigger]) + ([job_trigger], Dependent [Optional job_trigger]) in let sanity = - let _job_sanity_ci : tezos_job = + let job_sanity_ci : tezos_job = job ~__POS__ ~name:"sanity_ci" @@ -261,7 +260,7 @@ let jobs pipeline_type = ] |> job_external_once in - let _job_docker_hadolint = + let job_docker_hadolint = job ~rules:(make_rules ~changes:changeset_hadolint_docker_files ()) ~__POS__ @@ -277,7 +276,7 @@ let jobs pipeline_type = ["hadolint build.Dockerfile"; "hadolint Dockerfile"] |> job_external in - [] + [job_sanity_ci; job_docker_hadolint] in let job_docker_rust_toolchain = job_docker_rust_toolchain @@ -287,7 +286,7 @@ let jobs pipeline_type = () |> job_external_split in - let _job_docker_client_libs_dependencies = + let job_docker_client_libs_dependencies = job_docker_authenticated ~__POS__ ~rules:(make_rules ~changes:changeset_kaitai_e2e_files ()) @@ -307,14 +306,14 @@ let jobs pipeline_type = in let build = let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in - let _job_build_arm64_release : Tezos_ci.tezos_job = + let job_build_arm64_release : Tezos_ci.tezos_job = job_build_arm64_release ~rules:build_arm_rules () |> job_external_split in - let _job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = + let job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = job_build_arm64_exp_dev_extra ~rules:build_arm_rules () |> job_external_split in - let _job_static_x86_64_experimental = + let job_static_x86_64_experimental = job_build_static_binaries ~__POS__ ~arch:Amd64 @@ -328,20 +327,22 @@ let jobs pipeline_type = |> job_external_split in (* TODO: The code is a bit convulted here because these jobs are - either in the build or in the manual stage depeneding on the + either in the build or in the manual stage depending on the pipeline type. However, we can put them in the build stage on [before_merging] pipelines as long as we're careful to put [allow_failure: true]. *) - (match pipeline_type with - | Schedule_extended_test -> - let _job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in - let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in - () - | Before_merging -> ()) ; + let bin_packages_jobs = + match pipeline_type with + | Schedule_extended_test -> + let job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in + let job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in + [job_build_dpkg_amd64; job_build_rpm_amd64] + | Before_merging -> [] + in (* The build_x86_64 jobs are split in two to keep the artifact size under the 1GB hard limit set by GitLab. *) - (* [_job_build_x86_64_release] builds the released executables. *) - let _job_build_x86_64_release = + (* [job_build_x86_64_release] builds the released executables. *) + let job_build_x86_64_release = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 @@ -354,7 +355,7 @@ let jobs pipeline_type = (* 'oc.build_x86_64-exp-dev-extra' builds the developer and experimental executables, as well as the tezt test suite used by the subsequent 'tezt' jobs and TPS evaluation tool. *) - let _job_build_x86_64_exp_dev_extra = + let job_build_x86_64_exp_dev_extra = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 @@ -364,7 +365,7 @@ let jobs pipeline_type = () |> job_external_split in - let _job_ocaml_check : tezos_job = + let job_ocaml_check : tezos_job = job ~__POS__ ~name:"ocaml-check" @@ -381,7 +382,7 @@ let jobs pipeline_type = ["dune build @check"] |> job_external_split in - let _job_build_kernels : tezos_job = + let job_build_kernels : tezos_job = job ~__POS__ ~name:"oc.build_kernels" @@ -422,7 +423,7 @@ let jobs pipeline_type = (* Fetch records for Tezt generated on the last merge request pipeline on the most recently merged MR and makes them available in artifacts for future merge request pipelines. *) - let _job_tezt_fetch_records : tezos_job = + let job_tezt_fetch_records : tezos_job = job ~__POS__ ~name:"oc.tezt:fetch-records" @@ -456,10 +457,41 @@ let jobs pipeline_type = ]) |> job_external_split in - (* TODO: include the jobs defined above when full pipeline is - generated, as well as rust tool chain and client libs docker - builds. *) - [] + (* Used in [before_merging] and [schedule_extended_tests]. + + Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_select_tezts : tezos_job = + job + ~__POS__ + ~name:"select_tezts" + (* We need: + - Git (to run git diff) + - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.build + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + ["scripts/ci/select_tezts.sh || exit $?"] + ~allow_failure:(With_exit_codes [17]) + ~artifacts: + (artifacts ~expire_in:(Days 3) ~when_:Always ["selected_tezts.tsl"]) + |> job_external_once + in + [ + job_docker_rust_toolchain; + job_docker_client_libs_dependencies; + job_build_arm64_release; + job_build_arm64_exp_dev_extra; + job_static_x86_64_experimental; + job_build_x86_64_release; + job_build_x86_64_exp_dev_extra; + job_ocaml_check; + job_build_kernels; + job_tezt_fetch_records; + job_select_tezts; + ] + @ bin_packages_jobs in let packaging = let job_opam_prepare : tezos_job = @@ -480,21 +512,178 @@ let jobs pipeline_type = ] |> job_external_once in - let (_jobs_opam_packages : tezos_job list) = + let (jobs_opam_packages : tezos_job list) = read_opam_packages |> List.map (job_opam_package ~dependencies:(Dependent [Artifacts job_opam_prepare])) |> jobs_external_once ~path:"packaging/opam_package.yml" in - [] + jobs_opam_packages + in + let test = + (* check that ksy files are still up-to-date with octez *) + let job_kaitai_checks : tezos_job = + job + ~__POS__ + ~name:"kaitai_checks" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_kaitai_e2e_files ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + "make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez \ + encodings and Kaitai files seem to be out of sync. You might need \ + to run `make check-kaitai-struct-files` and commit the resulting \ + diff.' ; false)"; + ] + |> job_external_split + in + let job_kaitai_e2e_checks = + job + ~__POS__ + ~name:"kaitai_e2e_checks" + ~image:Images.client_libs_dependencies + ~stage:Stages.test + ~dependencies: + (Dependent + [ + Artifacts job_docker_client_libs_dependencies; + Job job_kaitai_checks; + ]) + ~rules: + (make_rules ~changes:changeset_kaitai_e2e_files ~dependent:true ()) + ~before_script: + (before_script + ~source_version:true + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5026 + As observed for the `unit:js_components` running `npm i` + everytime we run a job is inefficient. + + The benefit of this approach is that we specify node version + and npm dependencies (package.json) in one place, and that the local + environment is then the same as CI environment. *) + ~install_js_deps:true + []) + [ + "./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh \ + client-libs/kaitai-struct-files/files \ + client-libs/kaitai-struct-files/input 2>/dev/null"; + ] + |> job_external_split + in + let job_oc_check_lift_limits_patch = + job + ~__POS__ + ~name:"oc.check_lift_limits_patch" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_lift_limits_patch ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + (* Check that the patch only modifies the + src/proto_alpha/lib_protocol. If not, the rules above have to be + updated. *) + "[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | \ + cut -f3) = \"src/proto_alpha/lib_protocol/main.ml\" ]"; + "git apply src/bin_tps_evaluation/lift_limits.patch"; + "dune build @src/proto_alpha/lib_protocol/check"; + ] + |> job_external_split + in + let job_oc_misc_checks : tezos_job = + job + ~__POS__ + ~name:"oc.misc_checks" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_lint_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ([ + "./scripts/ci/lint_misc_check.sh"; + "scripts/check_wasm_pvm_regressions.sh check"; + ] + @ + (* The license check only applies to new files (in the sense + of [git add]), so can only run in [before_merging] + pipelines. *) + if pipeline_type = Before_merging then + ["./scripts/ci/lint_check_licenses.sh"] + else []) + |> job_external_split + in + let job_misc_opam_checks : tezos_job = + job + ~__POS__ + ~name:"misc_opam_checks" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~retry:2 + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + (* checks that all deps of opam packages are already installed *) + "./scripts/opam-check.sh"; + ] + |> job_external_split + in + let job_semgrep : tezos_job = + job + ~__POS__ + ~name:"oc.semgrep" + ~image:Images.semgrep_agent + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_semgrep_files ()) + [ + "echo \"OCaml code linting. For information on how to reproduce \ + locally, check out scripts/semgrep/README.md\""; + "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; + ] + |> job_external_split + in + [ + job_kaitai_checks; + job_kaitai_e2e_checks; + job_oc_check_lift_limits_patch; + job_oc_misc_checks; + job_misc_opam_checks; + job_semgrep; + ] + @ + match pipeline_type with + | Before_merging -> + let job_commit_titles : tezos_job = + job + ~__POS__ + ~name:"commit_titles" + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + (* ./scripts/ci/check_commit_messages.sh exits with code 65 when a git history contains + invalid commits titles in situations where that is allowed. *) + ["./scripts/ci/check_commit_messages.sh || exit $?"] + ~allow_failure:(With_exit_codes [65]) + |> job_external + in + [job_commit_titles] + | Schedule_extended_test -> [] in - let test = [] in let doc = [] in let manual = match pipeline_type with | Before_merging -> - let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = + let job_docker_amd64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ ~external_:true @@ -502,7 +691,7 @@ let jobs pipeline_type = ~arch:Amd64 Test_manual in - let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = + let job_docker_arm64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ ~external_:true @@ -510,7 +699,7 @@ let jobs pipeline_type = ~arch:Arm64 Test_manual in - let _job_build_dpkg_amd64_manual = + let job_build_dpkg_amd64_manual = job_build_bin_package ~__POS__ ~name:"oc.build:dpkg:amd64" @@ -521,7 +710,7 @@ let jobs pipeline_type = () |> job_external ~directory:"build" ~filename_suffix:"manual" in - let _job_build_rpm_amd64_manual = + let job_build_rpm_amd64_manual = job_build_bin_package ~__POS__ ~rules:[job_rule ~when_:Manual ()] @@ -532,8 +721,12 @@ let jobs pipeline_type = () |> job_external ~directory:"build" ~filename_suffix:"manual" in - (* TODO: include the jobs defined above when full pipeline is generated *) - [] + [ + job_docker_amd64_test_manual; + job_docker_arm64_test_manual; + job_build_dpkg_amd64_manual; + job_build_rpm_amd64_manual; + ] (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] in @@ -545,4 +738,5 @@ let jobs pipeline_type = (using {!job_external} or {!jobs_external}) and included by hand in the files [.gitlab/ci/pipelines/before_merging.yml] and [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) - trigger @ sanity @ build @ packaging @ test @ doc @ manual + ignore (trigger @ sanity @ build @ packaging @ test @ doc @ manual) ; + [] diff --git a/ci/bin/common.ml b/ci/bin/common.ml index c001068df04592720befed63605cc1981d2839e1..a1d04823373e9c2fcabebe5e5eccd31a981675ef 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -154,6 +154,19 @@ module Images = struct let hadolint = Image.register ~name:"hadolint" ~image_path:"hadolint/hadolint:2.9.3-debian" + + (* We specify the semgrep image by hash to avoid flakiness. Indeed, if we took the + latest release, then an update in the parser or analyser could result in new + errors being found even if the code doesn't change. This would place the + burden for fixing the code on the wrong dev (the devs who happen to open an + MR coinciding with the semgrep update rather than the dev who wrote the + infringing code in the first place). + Update the hash in scripts/semgrep/README.md too when updating it here + Last update: 2022-01-03 *) + let semgrep_agent = + Image.register + ~name:"semgrep_agent" + ~image_path:"returntocorp/semgrep-agent:sha-c6cd7cf" end (** {2 Helpers} *) @@ -318,6 +331,39 @@ let changeset_kaitai_e2e_files = let changeset_ocaml_files = ["src/**/*"; "tezt/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"; "devtools/**/*"] +let changeset_lift_limits_patch = + [ + "src/bin_tps_evaluation/lift_limits.patch"; + "src/proto_alpha/lib_protocol/main.ml"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + +(* The linting job runs over the set of [source_directories] + defined in [scripts/lint.sh] that must be included here: *) +let changeset_lint_files = + [ + "src/**/*"; + "tezt/**/*"; + "devtools/**/*"; + "scripts/**/*"; + "docs/**/*"; + "client-libs/**/*"; + "etherlink/**/*"; + ".gitlab-ci.yml"; + ".gitlab/**/*"; + ] + +let changeset_semgrep_files = + [ + "src/**/*"; + "tezt/**/*"; + "devtools/**/*"; + "scripts/semgrep/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index a89c8d87bfef930077fb4e2d87d0460960c45f78..976e39b9fcf2ef221b7234116f6ccd4985e46cde 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,7 +174,6 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" | ".gitlab/ci/jobs/doc/documentation.yml" @@ -182,22 +181,13 @@ let () = | ".gitlab/ci/jobs/doc/oc.install_python.yml" | ".gitlab/ci/jobs/packaging/debian_repository.yml" | ".gitlab/ci/jobs/shared/images.yml" - | ".gitlab/ci/jobs/shared/templates.yml" - | ".gitlab/ci/jobs/test/commit_titles.yml" - | ".gitlab/ci/jobs/test/common.yml" + | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/kaitai_checks.yml" - | ".gitlab/ci/jobs/test/kaitai_e2e_checks.yml" - | ".gitlab/ci/jobs/test/misc_opam_checks.yml" - | ".gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" - | ".gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml" - | ".gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" - | ".gitlab/ci/jobs/test/oc.semgrep.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/oc.unit.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" diff --git a/scripts/lint.sh b/scripts/lint.sh index d51b60a1a0c1f3563db33e3ad0345e577c511acc..6b3895e4f02a10bd3c87071b970f84d7c64ae5f0 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -34,6 +34,8 @@ say() { declare -a source_directories +# Make sure that the set of source_directories here are also reflected in +# [changeset_lint_files] in [ci/bin/common.ml]. source_directories=(src docs/doc_gen tezt devtools contrib etherlink client-libs) # Set of newline-separated basic regular expressions to exclude from --check-licenses-git-new. license_check_exclude=$(