From 24afb8b46662d89f363e6ddbf9e7139cf8999836 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 19 Mar 2024 12:26:08 +0100 Subject: [PATCH 01/11] CIAO: rework scaffolding to have one big ignore Once the full pipeline is generated, we want to return a list containing all the jobs. In the mean time, we generate jobs and put their definition in external files. To stop CIAO from overwriting the pipelien definition file ([.gitlab/ci/pipelines/PIPELINE.yml]) we return the empty list. In this commit, I rework the code so that there will only be one place to change when we go from returning the empty list to returning the full list of jobs. --- ci/bin/code_verification.ml | 85 ++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 19ee2272f4d3..73243c2fe294 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -238,12 +238,11 @@ let jobs pipeline_type = ] |> job_external in - (* TODO: put job_trigger here when full pipeline is generated *) (* TODO: the dependency on job_trigger does not have to be optional *) - ([], Dependent [Optional job_trigger]) + ([job_trigger], Dependent [Optional job_trigger]) in let sanity = - let _job_sanity_ci : tezos_job = + let job_sanity_ci : tezos_job = job ~__POS__ ~name:"sanity_ci" @@ -261,7 +260,7 @@ let jobs pipeline_type = ] |> job_external_once in - let _job_docker_hadolint = + let job_docker_hadolint = job ~rules:(make_rules ~changes:changeset_hadolint_docker_files ()) ~__POS__ @@ -277,7 +276,7 @@ let jobs pipeline_type = ["hadolint build.Dockerfile"; "hadolint Dockerfile"] |> job_external in - [] + [job_sanity_ci; job_docker_hadolint] in let job_docker_rust_toolchain = job_docker_rust_toolchain @@ -287,7 +286,7 @@ let jobs pipeline_type = () |> job_external_split in - let _job_docker_client_libs_dependencies = + let job_docker_client_libs_dependencies = job_docker_authenticated ~__POS__ ~rules:(make_rules ~changes:changeset_kaitai_e2e_files ()) @@ -307,14 +306,14 @@ let jobs pipeline_type = in let build = let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in - let _job_build_arm64_release : Tezos_ci.tezos_job = + let job_build_arm64_release : Tezos_ci.tezos_job = job_build_arm64_release ~rules:build_arm_rules () |> job_external_split in - let _job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = + let job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = job_build_arm64_exp_dev_extra ~rules:build_arm_rules () |> job_external_split in - let _job_static_x86_64_experimental = + let job_static_x86_64_experimental = job_build_static_binaries ~__POS__ ~arch:Amd64 @@ -328,20 +327,22 @@ let jobs pipeline_type = |> job_external_split in (* TODO: The code is a bit convulted here because these jobs are - either in the build or in the manual stage depeneding on the + either in the build or in the manual stage depending on the pipeline type. However, we can put them in the build stage on [before_merging] pipelines as long as we're careful to put [allow_failure: true]. *) - (match pipeline_type with - | Schedule_extended_test -> - let _job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in - let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in - () - | Before_merging -> ()) ; + let bin_packages_jobs = + match pipeline_type with + | Schedule_extended_test -> + let job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in + let job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in + [job_build_dpkg_amd64; job_build_rpm_amd64] + | Before_merging -> [] + in (* The build_x86_64 jobs are split in two to keep the artifact size under the 1GB hard limit set by GitLab. *) - (* [_job_build_x86_64_release] builds the released executables. *) - let _job_build_x86_64_release = + (* [job_build_x86_64_release] builds the released executables. *) + let job_build_x86_64_release = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 @@ -354,7 +355,7 @@ let jobs pipeline_type = (* 'oc.build_x86_64-exp-dev-extra' builds the developer and experimental executables, as well as the tezt test suite used by the subsequent 'tezt' jobs and TPS evaluation tool. *) - let _job_build_x86_64_exp_dev_extra = + let job_build_x86_64_exp_dev_extra = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 @@ -364,7 +365,7 @@ let jobs pipeline_type = () |> job_external_split in - let _job_ocaml_check : tezos_job = + let job_ocaml_check : tezos_job = job ~__POS__ ~name:"ocaml-check" @@ -381,7 +382,7 @@ let jobs pipeline_type = ["dune build @check"] |> job_external_split in - let _job_build_kernels : tezos_job = + let job_build_kernels : tezos_job = job ~__POS__ ~name:"oc.build_kernels" @@ -422,7 +423,7 @@ let jobs pipeline_type = (* Fetch records for Tezt generated on the last merge request pipeline on the most recently merged MR and makes them available in artifacts for future merge request pipelines. *) - let _job_tezt_fetch_records : tezos_job = + let job_tezt_fetch_records : tezos_job = job ~__POS__ ~name:"oc.tezt:fetch-records" @@ -456,10 +457,19 @@ let jobs pipeline_type = ]) |> job_external_split in - (* TODO: include the jobs defined above when full pipeline is - generated, as well as rust tool chain and client libs docker - builds. *) - [] + [ + job_docker_rust_toolchain; + job_docker_client_libs_dependencies; + job_build_arm64_release; + job_build_arm64_exp_dev_extra; + job_static_x86_64_experimental; + job_build_x86_64_release; + job_build_x86_64_exp_dev_extra; + job_ocaml_check; + job_build_kernels; + job_tezt_fetch_records; + ] + @ bin_packages_jobs in let packaging = let job_opam_prepare : tezos_job = @@ -480,21 +490,21 @@ let jobs pipeline_type = ] |> job_external_once in - let (_jobs_opam_packages : tezos_job list) = + let (jobs_opam_packages : tezos_job list) = read_opam_packages |> List.map (job_opam_package ~dependencies:(Dependent [Artifacts job_opam_prepare])) |> jobs_external_once ~path:"packaging/opam_package.yml" in - [] + jobs_opam_packages in let test = [] in let doc = [] in let manual = match pipeline_type with | Before_merging -> - let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = + let job_docker_amd64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ ~external_:true @@ -502,7 +512,7 @@ let jobs pipeline_type = ~arch:Amd64 Test_manual in - let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = + let job_docker_arm64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ ~external_:true @@ -510,7 +520,7 @@ let jobs pipeline_type = ~arch:Arm64 Test_manual in - let _job_build_dpkg_amd64_manual = + let job_build_dpkg_amd64_manual = job_build_bin_package ~__POS__ ~name:"oc.build:dpkg:amd64" @@ -521,7 +531,7 @@ let jobs pipeline_type = () |> job_external ~directory:"build" ~filename_suffix:"manual" in - let _job_build_rpm_amd64_manual = + let job_build_rpm_amd64_manual = job_build_bin_package ~__POS__ ~rules:[job_rule ~when_:Manual ()] @@ -532,8 +542,12 @@ let jobs pipeline_type = () |> job_external ~directory:"build" ~filename_suffix:"manual" in - (* TODO: include the jobs defined above when full pipeline is generated *) - [] + [ + job_docker_amd64_test_manual; + job_docker_arm64_test_manual; + job_build_dpkg_amd64_manual; + job_build_rpm_amd64_manual; + ] (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] in @@ -545,4 +559,5 @@ let jobs pipeline_type = (using {!job_external} or {!jobs_external}) and included by hand in the files [.gitlab/ci/pipelines/before_merging.yml] and [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) - trigger @ sanity @ build @ packaging @ test @ doc @ manual + ignore (trigger @ sanity @ build @ packaging @ test @ doc @ manual) ; + [] -- GitLab From 9663d8df461f674a9053de117751484a9c359663 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 19 Mar 2024 12:30:09 +0100 Subject: [PATCH 02/11] CI: generate job [select_tezt] --- .gitlab/ci/jobs/build/select_tezts.yml | 29 +++++++++++++------------- ci/bin/code_verification.ml | 22 +++++++++++++++++++ ci/bin/main.ml | 1 - 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.gitlab/ci/jobs/build/select_tezts.yml b/.gitlab/ci/jobs/build/select_tezts.yml index 23a2f139fe66..70fe7c9fe104 100644 --- a/.gitlab/ci/jobs/build/select_tezts.yml +++ b/.gitlab/ci/jobs/build/select_tezts.yml @@ -1,20 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + select_tezts: - extends: - - .default_settings_template - # We need: - # - Git (to run git diff) - # - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) - - .image_template__runtime_prebuild_dependencies + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} stage: build - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - scripts/ci/select_tezts.sh || exit $? + tags: + - gcp + dependencies: [] allow_failure: exit_codes: 17 + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - scripts/ci/select_tezts.sh || exit $? artifacts: - when: always - paths: - - selected_tezts.tsl expire_in: 3 days + paths: + - selected_tezts.tsl + when: always diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 73243c2fe294..e9cd485c807a 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -457,6 +457,27 @@ let jobs pipeline_type = ]) |> job_external_split in + (* Used in [before_merging] and [schedule_extended_tests]. + + Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_select_tezts : tezos_job = + job + ~__POS__ + ~name:"select_tezts" + (* We need: + - Git (to run git diff) + - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.build + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + ["scripts/ci/select_tezts.sh || exit $?"] + ~allow_failure:(With_exit_codes [17]) + ~artifacts: + (artifacts ~expire_in:(Days 3) ~when_:Always ["selected_tezts.tsl"]) + |> job_external_once + in [ job_docker_rust_toolchain; job_docker_client_libs_dependencies; @@ -468,6 +489,7 @@ let jobs pipeline_type = job_ocaml_check; job_build_kernels; job_tezt_fetch_records; + job_select_tezts; ] @ bin_packages_jobs in diff --git a/ci/bin/main.ml b/ci/bin/main.ml index a89c8d87bfef..2e1885f518f8 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,7 +174,6 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" | ".gitlab/ci/jobs/doc/documentation.yml" -- GitLab From 7b34458fff8a333548183a7b52e9bc46e85df67f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 19 Mar 2024 12:14:21 +0100 Subject: [PATCH 03/11] CI: generate job [kaitai_checks] --- .../test/kaitai_checks-before_merging.yml | 27 +++++++++++++++++++ .../kaitai_checks-scheduled_extended_test.yml | 18 +++++++++++++ .gitlab/ci/jobs/test/kaitai_checks.yml | 10 ------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 22 ++++++++++++++- ci/bin/main.ml | 1 - 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 .gitlab/ci/jobs/test/kaitai_checks-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/kaitai_checks.yml diff --git a/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml new file mode 100644 index 000000000000..b05c7cc774f1 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - images/**/* + - src/**/* + - client-libs/*kaitai*/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml new file mode 100644 index 000000000000..5302066d9cf2 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml @@ -0,0 +1,18 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/jobs/test/kaitai_checks.yml b/.gitlab/ci/jobs/test/kaitai_checks.yml deleted file mode 100644 index 13b4f2c81bc5..000000000000 --- a/.gitlab/ci/jobs/test/kaitai_checks.yml +++ /dev/null @@ -1,10 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# check that ksy files are still up-to-date with octez -kaitai_checks: - extends: - - .test_template - - .needs__trigger - - .rules__octez_kaitai_e2e_changes - script: - - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` and commit the resulting diff.' ; false) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index c746bdb1f29b..4d62c148eb55 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -30,7 +30,7 @@ include: - .gitlab/ci/jobs/build/select_tezts.yml # Stage: test - - .gitlab/ci/jobs/test/kaitai_checks.yml + - .gitlab/ci/jobs/test/kaitai_checks-before_merging.yml - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d62f61648b31..9cca1f33322e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -37,7 +37,7 @@ include: - .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml # Tests that may not have been run in before_merging pipeline # because of absence of certain changes - - .gitlab/ci/jobs/test/kaitai_checks.yml + - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml - .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e9cd485c807a..e4766719f86e 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -521,7 +521,27 @@ let jobs pipeline_type = in jobs_opam_packages in - let test = [] in + let test = + (* check that ksy files are still up-to-date with octez *) + let job_kaitai_checks : tezos_job = + job + ~__POS__ + ~name:"kaitai_checks" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_kaitai_e2e_files ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + "make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez \ + encodings and Kaitai files seem to be out of sync. You might need \ + to run `make check-kaitai-struct-files` and commit the resulting \ + diff.' ; false)"; + ] + |> job_external_split + in + [job_kaitai_checks] + in let doc = [] in let manual = match pipeline_type with diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 2e1885f518f8..95619dd848de 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -185,7 +185,6 @@ let () = | ".gitlab/ci/jobs/test/commit_titles.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/kaitai_checks.yml" | ".gitlab/ci/jobs/test/kaitai_e2e_checks.yml" | ".gitlab/ci/jobs/test/misc_opam_checks.yml" | ".gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml" -- GitLab From 877defd1c9eb695a47bc92cbc47c5c289a97de5f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 14 Mar 2024 15:33:17 +0100 Subject: [PATCH 04/11] CI: generate [kaitai_e2e_checks] --- .gitlab/ci/jobs/shared/templates.yml | 10 ------ .../test/kaitai_e2e_checks-before_merging.yml | 27 ++++++++++++++ ...tai_e2e_checks-scheduled_extended_test.yml | 21 +++++++++++ .gitlab/ci/jobs/test/kaitai_e2e_checks.yml | 23 ------------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 35 ++++++++++++++++++- ci/bin/main.ml | 1 - 8 files changed, 84 insertions(+), 37 deletions(-) create mode 100644 .gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/kaitai_e2e_checks.yml diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 781bb20a2e8b..0fcdadd6adb7 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -151,16 +151,6 @@ - .gitlab-ci.yml when: manual -.rules__octez_kaitai_e2e_changes: - rules: - - changes: - - images/**/* - - src/**/* - - client-libs/*kaitai*/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - # Add variable for bisect_ppx instrumentation. # # This template should be extended by jobs that build OCaml targets diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml new file mode 100644 index 000000000000..408cc9ab3300 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} + stage: test + tags: + - gcp + rules: + - changes: + - images/**/* + - src/**/* + - client-libs/*kaitai*/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:client-libs-dependencies + - kaitai_checks + dependencies: + - oc.docker:client-libs-dependencies + before_script: + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + script: + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + client-libs/kaitai-struct-files/input 2>/dev/null diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml new file mode 100644 index 000000000000..e5a1817cb557 --- /dev/null +++ b/.gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml @@ -0,0 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} + stage: test + tags: + - gcp + rules: + - when: on_success + needs: + - oc.docker:client-libs-dependencies + - kaitai_checks + dependencies: + - oc.docker:client-libs-dependencies + before_script: + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + script: + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + client-libs/kaitai-struct-files/input 2>/dev/null diff --git a/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml b/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml deleted file mode 100644 index 9cfc357e3144..000000000000 --- a/.gitlab/ci/jobs/test/kaitai_e2e_checks.yml +++ /dev/null @@ -1,23 +0,0 @@ -kaitai_e2e_checks: - extends: - - .default_settings_template - - .image_template__client_libs_dependencies - - .rules__octez_kaitai_e2e_changes - stage: test - needs: [oc.docker:client-libs-dependencies, kaitai_checks] - dependencies: [oc.docker:client-libs-dependencies] - before_script: - - . ./scripts/version.sh - # TODO: https://gitlab.com/tezos/tezos/-/issues/5026 - # As observed for the `unit:js_components` running `npm i` - # everytime we run a job is inefficient. - # - # The benefit of this approach is that we specify node version - # and npm dependencies (package.json) in one place, and that the local - # environment is then the same as CI environment. - - . ./scripts/install_build_deps.js.sh - script: - - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh - client-libs/kaitai-struct-files/files - client-libs/kaitai-struct-files/input - 2>/dev/null diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 4d62c148eb55..1c930472084e 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -31,7 +31,7 @@ include: # Stage: test - .gitlab/ci/jobs/test/kaitai_checks-before_merging.yml - - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml + - .gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml - .gitlab/ci/jobs/test/misc_opam_checks.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 9cca1f33322e..9b5d36c28e48 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -38,7 +38,7 @@ include: # Tests that may not have been run in before_merging pipeline # because of absence of certain changes - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/kaitai_e2e_checks.yml + - .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml - .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml - .gitlab/ci/jobs/test/misc_opam_checks.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e4766719f86e..6b2d4f834581 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -540,7 +540,40 @@ let jobs pipeline_type = ] |> job_external_split in - [job_kaitai_checks] + let job_kaitai_e2e_checks = + job + ~__POS__ + ~name:"kaitai_e2e_checks" + ~image:Images.client_libs_dependencies + ~stage:Stages.test + ~dependencies: + (Dependent + [ + Artifacts job_docker_client_libs_dependencies; + Job job_kaitai_checks; + ]) + ~rules: + (make_rules ~changes:changeset_kaitai_e2e_files ~dependent:true ()) + ~before_script: + (before_script + ~source_version:true + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5026 + As observed for the `unit:js_components` running `npm i` + everytime we run a job is inefficient. + + The benefit of this approach is that we specify node version + and npm dependencies (package.json) in one place, and that the local + environment is then the same as CI environment. *) + ~install_js_deps:true + []) + [ + "./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh \ + client-libs/kaitai-struct-files/files \ + client-libs/kaitai-struct-files/input 2>/dev/null"; + ] + |> job_external_split + in + [job_kaitai_checks; job_kaitai_e2e_checks] in let doc = [] in let manual = diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 95619dd848de..cc3b0e0cec58 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -185,7 +185,6 @@ let () = | ".gitlab/ci/jobs/test/commit_titles.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/kaitai_e2e_checks.yml" | ".gitlab/ci/jobs/test/misc_opam_checks.yml" | ".gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" -- GitLab From 4e87f81157a3e5a636623c8f82dc1bf089cb421b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 14:08:25 +0100 Subject: [PATCH 05/11] CI: generate [oc.check_lift_limits_patch] jobs --- ...check_lift_limits_patch-before_merging.yml | 27 +++++++++++++++++++ ...t_limits_patch-scheduled_extended_test.yml | 19 +++++++++++++ .../jobs/test/oc.check_lift_limits_patch.yml | 18 ------------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 22 ++++++++++++++- ci/bin/common.ml | 8 ++++++ ci/bin/main.ml | 1 - 8 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml new file mode 100644 index 000000000000..98eb7dcfad50 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.check_lift_limits_patch: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/bin_tps_evaluation/lift_limits.patch + - src/proto_alpha/lib_protocol/main.ml + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = + "src/proto_alpha/lib_protocol/main.ml" ]' + - git apply src/bin_tps_evaluation/lift_limits.patch + - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml new file mode 100644 index 000000000000..3da2f721d70d --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml @@ -0,0 +1,19 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.check_lift_limits_patch: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = + "src/proto_alpha/lib_protocol/main.ml" ]' + - git apply src/bin_tps_evaluation/lift_limits.patch + - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml deleted file mode 100644 index a1794aae470a..000000000000 --- a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml +++ /dev/null @@ -1,18 +0,0 @@ -oc.check_lift_limits_patch: - extends: - - .test_template - - .needs__trigger - rules: - - changes: - - src/bin_tps_evaluation/lift_limits.patch - - src/proto_alpha/lib_protocol/main.ml - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - script: - # Check that the patch only modifies the - # src/proto_alpha/lib_protocol. If not, the rules above have to be - # updated. - - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = "src/proto_alpha/lib_protocol/main.ml" ]' - - git apply src/bin_tps_evaluation/lift_limits.patch - - dune build @src/proto_alpha/lib_protocol/check diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 1c930472084e..e1c9119aae49 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -32,7 +32,7 @@ include: # Stage: test - .gitlab/ci/jobs/test/kaitai_checks-before_merging.yml - .gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml - - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml + - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml - .gitlab/ci/jobs/test/misc_opam_checks.yml - .gitlab/ci/jobs/test/commit_titles.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 9b5d36c28e48..468f5efcf1ff 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -39,7 +39,7 @@ include: # because of absence of certain changes - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml + - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml - .gitlab/ci/jobs/test/misc_opam_checks.yml - .gitlab/ci/jobs/test/oc.semgrep.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 6b2d4f834581..af766ceb67e3 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -573,7 +573,27 @@ let jobs pipeline_type = ] |> job_external_split in - [job_kaitai_checks; job_kaitai_e2e_checks] + let job_oc_check_lift_limits_patch = + job + ~__POS__ + ~name:"oc.check_lift_limits_patch" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_lift_limits_patch ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + (* Check that the patch only modifies the + src/proto_alpha/lib_protocol. If not, the rules above have to be + updated. *) + "[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | \ + cut -f3) = \"src/proto_alpha/lib_protocol/main.ml\" ]"; + "git apply src/bin_tps_evaluation/lift_limits.patch"; + "dune build @src/proto_alpha/lib_protocol/check"; + ] + |> job_external_split + in + [job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch] in let doc = [] in let manual = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index c001068df045..9fe20d8e1038 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -318,6 +318,14 @@ let changeset_kaitai_e2e_files = let changeset_ocaml_files = ["src/**/*"; "tezt/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"; "devtools/**/*"] +let changeset_lift_limits_patch = + [ + "src/bin_tps_evaluation/lift_limits.patch"; + "src/proto_alpha/lib_protocol/main.ml"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index cc3b0e0cec58..36fe866968d3 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -186,7 +186,6 @@ let () = | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/misc_opam_checks.yml" - | ".gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" | ".gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml" | ".gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml" -- GitLab From d65a959243bcc310782afebeed31e43e3dc2fbe5 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 12:01:32 +0100 Subject: [PATCH 06/11] CI: refactor, script line order in [oc.misc_checks-before_merging] This ordering clarifies the difference between this job and the scheduled variant. --- .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml index 7e808a0a7b7d..5f61535758c8 100644 --- a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml @@ -3,5 +3,5 @@ oc.misc_checks: - .oc.misc_checks script: - ./scripts/ci/lint_misc_check.sh - - ./scripts/ci/lint_check_licenses.sh - scripts/check_wasm_pvm_regressions.sh check + - ./scripts/ci/lint_check_licenses.sh -- GitLab From 13dec21fcc261aba70376ac8227eed9903c3201f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 12:02:36 +0100 Subject: [PATCH 07/11] CI: refactor, only [changes:] rules in [oc.misc_checks-before_merging] The semantics of [changes:] in scheduled pipelines is less obvious and regardless we prefer to run jobs unconditionally in the scheduled pipeline. Also explicitly state the [when:] --- .gitlab/ci/jobs/shared/templates.yml | 14 -------------- .../ci/jobs/test/oc.misc_checks-before_merging.yml | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 0fcdadd6adb7..c831e40d4390 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -227,20 +227,6 @@ # Load the environment poetry previously created in the docker image. # Give access to the Python dependencies/executables - . $HOME/.venv/bin/activate - rules: - # The linting job runs over the set of [source_directories] - # defined in [scripts/lint.sh] that must be included here: - - changes: - - src/**/* - - tezt/**/* - - devtools/**/* - - scripts/**/* - - docs/**/* - - contrib/**/* - - client-libs/**/* - - etherlink/**/* - - .gitlab-ci.yml - - .gitlab/**/* .test_etherlink_kernel: extends: diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml index 5f61535758c8..cd2118c1b490 100644 --- a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml @@ -5,3 +5,17 @@ oc.misc_checks: - ./scripts/ci/lint_misc_check.sh - scripts/check_wasm_pvm_regressions.sh check - ./scripts/ci/lint_check_licenses.sh + rules: + # The linting job runs over the set of [source_directories] + # defined in [scripts/lint.sh] that must be included here: + - changes: + - src/**/* + - tezt/**/* + - devtools/**/* + - scripts/**/* + - docs/**/* + - client-libs/**/* + - etherlink/**/* + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success -- GitLab From f7d5c531667dd794bf8b00586ee3d2defe8a3ebc Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 14:15:13 +0100 Subject: [PATCH 08/11] CI: generate [oc.misc_checks] job --- .../test/oc.misc_checks-before_merging.yml | 50 ++++++++++++------- .../oc.misc_checks-schedule_extended_test.yml | 8 --- ...oc.misc_checks-scheduled_extended_test.yml | 19 +++++++ .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 35 ++++++++++++- ci/bin/common.ml | 15 ++++++ ci/bin/main.ml | 2 - scripts/lint.sh | 2 + 8 files changed, 102 insertions(+), 31 deletions(-) delete mode 100644 .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml create mode 100644 .gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml index cd2118c1b490..0c7181292109 100644 --- a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml @@ -1,21 +1,33 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + oc.misc_checks: - extends: - - .oc.misc_checks - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - ./scripts/ci/lint_check_licenses.sh + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp rules: - # The linting job runs over the set of [source_directories] - # defined in [scripts/lint.sh] that must be included here: - - changes: - - src/**/* - - tezt/**/* - - devtools/**/* - - scripts/**/* - - docs/**/* - - client-libs/**/* - - etherlink/**/* - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success + - changes: + - src/**/* + - tezt/**/* + - devtools/**/* + - scripts/**/* + - docs/**/* + - client-libs/**/* + - etherlink/**/* + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - ./scripts/ci/lint_check_licenses.sh diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml b/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml deleted file mode 100644 index a21a310086f3..000000000000 --- a/.gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml +++ /dev/null @@ -1,8 +0,0 @@ -# As 'oc.misc_checks' in 'oc.misc_checks:before_merging.yml' but does -# not check the license header of newly added OCaml files. -oc.misc_checks: - extends: - - .oc.misc_checks - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml new file mode 100644 index 000000000000..a65b4c74d8cf --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml @@ -0,0 +1,19 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.misc_checks: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 468f5efcf1ff..d93bbbbb2db0 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -40,7 +40,7 @@ include: - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml + - .gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/misc_opam_checks.yml - .gitlab/ci/jobs/test/oc.semgrep.yml - .gitlab/ci/jobs/test/oc.unit.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index af766ceb67e3..d3340f665871 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -593,7 +593,40 @@ let jobs pipeline_type = ] |> job_external_split in - [job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch] + let job_oc_misc_checks : tezos_job = + job + ~__POS__ + ~name:"oc.misc_checks" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_lint_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ([ + "./scripts/ci/lint_misc_check.sh"; + "scripts/check_wasm_pvm_regressions.sh check"; + ] + @ + (* The license check only applies to new files (in the sense + of [git add]), so can only run in [before_merging] + pipelines. *) + if pipeline_type = Before_merging then + ["./scripts/ci/lint_check_licenses.sh"] + else []) + |> job_external_split + in + [ + job_kaitai_checks; + job_kaitai_e2e_checks; + job_oc_check_lift_limits_patch; + job_oc_misc_checks; + ] in let doc = [] in let manual = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 9fe20d8e1038..cfaeee5fbb70 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -326,6 +326,21 @@ let changeset_lift_limits_patch = ".gitlab-ci.yml"; ] +(* The linting job runs over the set of [source_directories] + defined in [scripts/lint.sh] that must be included here: *) +let changeset_lint_files = + [ + "src/**/*"; + "tezt/**/*"; + "devtools/**/*"; + "scripts/**/*"; + "docs/**/*"; + "client-libs/**/*"; + "etherlink/**/*"; + ".gitlab-ci.yml"; + ".gitlab/**/*"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 36fe866968d3..fd3721895b96 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -187,8 +187,6 @@ let () = | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/misc_opam_checks.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" - | ".gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml" - | ".gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" diff --git a/scripts/lint.sh b/scripts/lint.sh index d51b60a1a0c1..6b3895e4f02a 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -34,6 +34,8 @@ say() { declare -a source_directories +# Make sure that the set of source_directories here are also reflected in +# [changeset_lint_files] in [ci/bin/common.ml]. source_directories=(src docs/doc_gen tezt devtools contrib etherlink client-libs) # Set of newline-separated basic regular expressions to exclude from --check-licenses-git-new. license_check_exclude=$( -- GitLab From 6570f388528f89c36b03e48d5dd5bb1d4b327f18 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 14:17:44 +0100 Subject: [PATCH 09/11] CI: generate [misc_opam_checks] job --- .../test/misc_opam_checks-before_merging.yml | 28 +++++++++++++++++++ ...sc_opam_checks-scheduled_extended_test.yml | 17 +++++++++++ .gitlab/ci/jobs/test/misc_opam_checks.yml | 10 ------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 17 +++++++++++ ci/bin/main.ml | 1 - 7 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 .gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/misc_opam_checks.yml diff --git a/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml new file mode 100644 index 000000000000..0ae8eaff2337 --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml @@ -0,0 +1,28 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +misc_opam_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/opam-check.sh + retry: 2 diff --git a/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml new file mode 100644 index 000000000000..33cfd7d9cc02 --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml @@ -0,0 +1,17 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +misc_opam_checks: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/opam-check.sh + retry: 2 diff --git a/.gitlab/ci/jobs/test/misc_opam_checks.yml b/.gitlab/ci/jobs/test/misc_opam_checks.yml deleted file mode 100644 index ea65f24bebd7..000000000000 --- a/.gitlab/ci/jobs/test/misc_opam_checks.yml +++ /dev/null @@ -1,10 +0,0 @@ -misc_opam_checks: - extends: - - .test_template - - .needs__trigger - # This job is flaky due to e.g. network issues. - - .oc.template__retry_flaky - - .rules__octez_changes - script: - # checks that all deps of opam packages are already installed - - ./scripts/opam-check.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index e1c9119aae49..d1ec2ac342a7 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -34,7 +34,7 @@ include: - .gitlab/ci/jobs/test/kaitai_e2e_checks-before_merging.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml - - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml - .gitlab/ci/jobs/test/commit_titles.yml - .gitlab/ci/jobs/test/oc.semgrep.yml - .gitlab/ci/jobs/test/oc.unit.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d93bbbbb2db0..2bb42466e1a1 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -41,7 +41,7 @@ include: - .gitlab/ci/jobs/test/kaitai_e2e_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.semgrep.yml - .gitlab/ci/jobs/test/oc.unit.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index d3340f665871..74e457d65268 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -621,11 +621,28 @@ let jobs pipeline_type = else []) |> job_external_split in + let job_misc_opam_checks : tezos_job = + job + ~__POS__ + ~name:"misc_opam_checks" + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~retry:2 + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + [ + (* checks that all deps of opam packages are already installed *) + "./scripts/opam-check.sh"; + ] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch; job_oc_misc_checks; + job_misc_opam_checks; ] in let doc = [] in diff --git a/ci/bin/main.ml b/ci/bin/main.ml index fd3721895b96..4cc269103abe 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -185,7 +185,6 @@ let () = | ".gitlab/ci/jobs/test/commit_titles.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/misc_opam_checks.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" -- GitLab From 7572c2f9796f4b7ea3d1d4470ca1f74e98528662 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 14:26:59 +0100 Subject: [PATCH 10/11] CI: generate [commit_titles] job --- .gitlab/ci/jobs/test/commit_titles.yml | 26 ++++++++++++++------------ ci/bin/code_verification.ml | 18 ++++++++++++++++++ ci/bin/main.ml | 4 +--- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/.gitlab/ci/jobs/test/commit_titles.yml b/.gitlab/ci/jobs/test/commit_titles.yml index b636b7a03760..e2d913711e63 100644 --- a/.gitlab/ci/jobs/test/commit_titles.yml +++ b/.gitlab/ci/jobs/test/commit_titles.yml @@ -1,14 +1,16 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + commit_titles: - extends: - - .default_settings_template - - .image_template__runtime_prebuild_dependencies - - .needs__trigger - stage: "test" - script: - # Check commit messages - - ./scripts/ci/check_commit_messages.sh || exit $? + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] allow_failure: - # ./scripts/ci/check_commit_messages.sh exits with code 65 when a - # git history contains invalid commits titles in situations where - # that is allowed. - exit_codes: [65] + exit_codes: 65 + script: + - ./scripts/ci/check_commit_messages.sh || exit $? diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 74e457d65268..ef3f48f48dd5 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -644,6 +644,24 @@ let jobs pipeline_type = job_oc_misc_checks; job_misc_opam_checks; ] + @ + match pipeline_type with + | Before_merging -> + let job_commit_titles : tezos_job = + job + ~__POS__ + ~name:"commit_titles" + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + (* ./scripts/ci/check_commit_messages.sh exits with code 65 when a git history contains + invalid commits titles in situations where that is allowed. *) + ["./scripts/ci/check_commit_messages.sh || exit $?"] + ~allow_failure:(With_exit_codes [65]) + |> job_external + in + [job_commit_titles] + | Schedule_extended_test -> [] in let doc = [] in let manual = diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 4cc269103abe..64f86d92437b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -181,9 +181,7 @@ let () = | ".gitlab/ci/jobs/doc/oc.install_python.yml" | ".gitlab/ci/jobs/packaging/debian_repository.yml" | ".gitlab/ci/jobs/shared/images.yml" - | ".gitlab/ci/jobs/shared/templates.yml" - | ".gitlab/ci/jobs/test/commit_titles.yml" - | ".gitlab/ci/jobs/test/common.yml" + | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" -- GitLab From 9b6b0ed8bed364b8b6dd280698de1f6608159954 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 15:21:31 +0100 Subject: [PATCH 11/11] CI: generate job [oc.semgrep] --- .gitlab/ci/jobs/shared/images.yml | 2 ++ .../jobs/test/oc.semgrep-before_merging.yml | 25 ++++++++++++++++++ .../oc.semgrep-scheduled_extended_test.yml | 15 +++++++++++ .gitlab/ci/jobs/test/oc.semgrep.yml | 26 ------------------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 16 ++++++++++++ ci/bin/common.ml | 23 ++++++++++++++++ ci/bin/main.ml | 1 - 9 files changed, 83 insertions(+), 29 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.semgrep-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.semgrep.yml diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 1525e775d9c9..3c9fde318195 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -25,3 +25,5 @@ image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} .image_template__rust_toolchain: image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +.image_template__semgrep_agent: + image: returntocorp/semgrep-agent:sha-c6cd7cf diff --git a/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml new file mode 100644 index 000000000000..10121659decf --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml @@ -0,0 +1,25 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - tezt/**/* + - devtools/**/* + - scripts/semgrep/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml new file mode 100644 index 000000000000..439fcc6cd6ab --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml @@ -0,0 +1,15 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/jobs/test/oc.semgrep.yml b/.gitlab/ci/jobs/test/oc.semgrep.yml deleted file mode 100644 index 53e0f3e4602b..000000000000 --- a/.gitlab/ci/jobs/test/oc.semgrep.yml +++ /dev/null @@ -1,26 +0,0 @@ -oc.semgrep: - extends: - - .default_settings_template - - .needs__trigger - rules: - - changes: - - src/**/* - - tezt/**/* - - devtools/**/* - - scripts/semgrep/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - # We specify the image by hash to avoid flakiness. Indeed, if we took the - # latest release, then an update in the parser or analyser could result in new - # errors being found even if the code doesn't change. This would place the - # burden for fixing the code on the wrong dev (the devs who happen to open an - # MR coinciding with the semgrep update rather than the dev who wrote the - # infringing code in the first place). - # Update the hash in scripts/semgrep/README.md too when updating it here - # Last update: 20212-01-03 - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d1ec2ac342a7..21e5caf41c29 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -36,7 +36,7 @@ include: - .gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml - .gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml - .gitlab/ci/jobs/test/commit_titles.yml - - .gitlab/ci/jobs/test/oc.semgrep.yml + - .gitlab/ci/jobs/test/oc.semgrep-before_merging.yml - .gitlab/ci/jobs/test/oc.unit.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 2bb42466e1a1..2e4748ef15f6 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -42,7 +42,7 @@ include: - .gitlab/ci/jobs/test/oc.check_lift_limits_patch-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.misc_checks-scheduled_extended_test.yml - .gitlab/ci/jobs/test/misc_opam_checks-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.semgrep.yml + - .gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.unit.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index ef3f48f48dd5..f84d462131ba 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -637,12 +637,28 @@ let jobs pipeline_type = ] |> job_external_split in + let job_semgrep : tezos_job = + job + ~__POS__ + ~name:"oc.semgrep" + ~image:Images.semgrep_agent + ~stage:Stages.test + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_semgrep_files ()) + [ + "echo \"OCaml code linting. For information on how to reproduce \ + locally, check out scripts/semgrep/README.md\""; + "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; + ] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch; job_oc_misc_checks; job_misc_opam_checks; + job_semgrep; ] @ match pipeline_type with diff --git a/ci/bin/common.ml b/ci/bin/common.ml index cfaeee5fbb70..a1d04823373e 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -154,6 +154,19 @@ module Images = struct let hadolint = Image.register ~name:"hadolint" ~image_path:"hadolint/hadolint:2.9.3-debian" + + (* We specify the semgrep image by hash to avoid flakiness. Indeed, if we took the + latest release, then an update in the parser or analyser could result in new + errors being found even if the code doesn't change. This would place the + burden for fixing the code on the wrong dev (the devs who happen to open an + MR coinciding with the semgrep update rather than the dev who wrote the + infringing code in the first place). + Update the hash in scripts/semgrep/README.md too when updating it here + Last update: 2022-01-03 *) + let semgrep_agent = + Image.register + ~name:"semgrep_agent" + ~image_path:"returntocorp/semgrep-agent:sha-c6cd7cf" end (** {2 Helpers} *) @@ -341,6 +354,16 @@ let changeset_lint_files = ".gitlab/**/*"; ] +let changeset_semgrep_files = + [ + "src/**/*"; + "tezt/**/*"; + "devtools/**/*"; + "scripts/semgrep/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 64f86d92437b..976e39b9fcf2 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -188,7 +188,6 @@ let () = | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" - | ".gitlab/ci/jobs/test/oc.semgrep.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/oc.unit.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" -- GitLab