From 13bfa7082a585a9d860c9716bd56cb366ade1138 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 4 Sep 2024 15:52:07 +0200 Subject: [PATCH 1/4] CI: Simplify naming of job [docker:hadolint] This job used to have different names depending on the pipeline type. There doesn't seem to be a good reason for this complication. --- .gitlab/ci/pipelines/before_merging.yml | 2 +- .gitlab/ci/pipelines/merge_train.yml | 2 +- .gitlab/ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 8 +------- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 5d34c60908df..066a9eb2e310 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -132,7 +132,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-before_merging: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 5d34c60908df..066a9eb2e310 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -132,7 +132,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-before_merging: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index ff4105a7a9d0..3d92b2b34437 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -109,7 +109,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-schedule_extended_test: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 459381f48f8a..42761e299753 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -349,13 +349,7 @@ let jobs pipeline_type = job ~rules:(make_rules ~changes:changeset_hadolint_docker_files ()) ~__POS__ - ~name: - (* TODO: I'm not sure it makes sense to have different names for the same job *) - ("docker:hadolint-" - ^ - match pipeline_type with - | Before_merging -> "before_merging" - | Schedule_extended_test -> "schedule_extended_test") + ~name:"docker:hadolint" ~image:Images.hadolint ~stage:Stages.sanity ["hadolint build.Dockerfile"; "hadolint Dockerfile"] -- GitLab From 856a1126fc237254b6b5ebd9de4846177bdf569d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 4 Sep 2024 15:56:33 +0200 Subject: [PATCH 2/4] CI: make [stage] and [dependencies] coherent in [sanity] stage This modifies the jobs [docker:hadolint] and [nix] such that they depend on [trigger], which is coherent with the other jobs in stage [sanity]. --- .gitlab/ci/pipelines/before_merging.yml | 4 ++++ .gitlab/ci/pipelines/merge_train.yml | 4 ++++ ci/bin/code_verification.ml | 12 ++++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 066a9eb2e310..fb05209f4fe1 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -142,6 +142,8 @@ docker:hadolint: - Dockerfile - build.Dockerfile when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes script: @@ -159,6 +161,8 @@ nix: - flake.lock - scripts/version.sh when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes cache: diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 066a9eb2e310..fb05209f4fe1 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -142,6 +142,8 @@ docker:hadolint: - Dockerfile - build.Dockerfile when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes script: @@ -159,6 +161,8 @@ nix: - flake.lock - scripts/version.sh when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes cache: diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 42761e299753..15d1fa46a91d 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -306,13 +306,15 @@ let jobs pipeline_type = (* Sanity jobs *) let sanity = + let stage = Stages.sanity in + let dependencies = dependencies_needs_start in let job_sanity_ci : tezos_job = job ~__POS__ ~name:"sanity_ci" ~image:Images.CI.build - ~stage:Stages.sanity - ~dependencies:dependencies_needs_start + ~stage + ~dependencies ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) [ "make -C manifest check"; @@ -329,7 +331,8 @@ let jobs pipeline_type = ~__POS__ ~name:"nix" ~image:Images.nix - ~stage:Stages.sanity + ~stage + ~dependencies ~artifacts:(artifacts ~when_:On_failure ["flake.lock"]) ~rules: (make_rules @@ -350,8 +353,9 @@ let jobs pipeline_type = ~rules:(make_rules ~changes:changeset_hadolint_docker_files ()) ~__POS__ ~name:"docker:hadolint" + ~dependencies ~image:Images.hadolint - ~stage:Stages.sanity + ~stage ["hadolint build.Dockerfile"; "hadolint Dockerfile"] in let mr_only_jobs = -- GitLab From 4ad6d3a25b545eff46716a621ca7e6af6a3a949c Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 4 Sep 2024 15:57:59 +0200 Subject: [PATCH 3/4] CI: move [oc.ocaml_fmt, oc.semgrep, oc.misc_checks] to stage [sanity] We promote these jobs such that they run earlier in the pipeline, ensuring that no staged jobs run unless these jobs execute successfully. --- .gitlab/ci/pipelines/before_merging.yml | 170 +++++++++--------- .gitlab/ci/pipelines/merge_train.yml | 170 +++++++++--------- .../ci/pipelines/schedule_extended_test.yml | 104 +++++------ ci/bin/code_verification.ml | 128 ++++++------- 4 files changed, 288 insertions(+), 284 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index fb05209f4fe1..0b69196c03d4 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -150,6 +150,91 @@ docker:hadolint: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - '**/*.ml' + - '**/*.mli' + - '**/.ocamlformat' + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - devtools/**/* + - scripts/semgrep/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - client-libs/**/* + - contrib/**/* + - devtools/**/* + - docs/**/* + - etherlink/**/* + - scripts/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + - ./scripts/ci/lint_check_licenses.sh + nix: image: nixos/nix:2.22.1 stage: sanity @@ -1807,41 +1892,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/**/* - - contrib/**/* - - devtools/**/* - - docs/**/* - - etherlink/**/* - - scripts/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - - ./scripts/ci/lint_check_licenses.sh - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1868,56 +1918,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - '**/.ocamlformat' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - scripts/semgrep/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index fb05209f4fe1..0b69196c03d4 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -150,6 +150,91 @@ docker:hadolint: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - '**/*.ml' + - '**/*.mli' + - '**/.ocamlformat' + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - devtools/**/* + - scripts/semgrep/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - client-libs/**/* + - contrib/**/* + - devtools/**/* + - docs/**/* + - etherlink/**/* + - scripts/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + - ./scripts/ci/lint_check_licenses.sh + nix: image: nixos/nix:2.22.1 stage: sanity @@ -1807,41 +1892,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/**/* - - contrib/**/* - - devtools/**/* - - docs/**/* - - etherlink/**/* - - scripts/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - - ./scripts/ci/lint_check_licenses.sh - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1868,56 +1918,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - '**/.ocamlformat' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - scripts/semgrep/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 3d92b2b34437..5b6c8a33d5c0 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -122,6 +122,58 @@ docker:hadolint: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + oc.build_arm64-released: image: ${ci_image_name}/build:${ci_image_tag} stage: build @@ -1398,26 +1450,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1435,38 +1467,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - when: always - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 15d1fa46a91d..5395206276d8 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -358,6 +358,64 @@ let jobs pipeline_type = ~stage ["hadolint build.Dockerfile"; "hadolint Dockerfile"] in + let job_oc_ocaml_fmt : tezos_job = + job + ~__POS__ + ~name:"oc.ocaml_fmt" + ~image:Images.CI.build + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_ocaml_fmt_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ["scripts/lint.sh --check-ocamlformat"; "dune build --profile=dev @fmt"] + in + let job_semgrep : tezos_job = + job + ~__POS__ + ~name:"oc.semgrep" + ~image:Images.semgrep_agent + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_semgrep_files ()) + [ + "echo \"OCaml code linting. For information on how to reproduce \ + locally, check out scripts/semgrep/README.md\""; + "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; + ] + in + let job_oc_misc_checks : tezos_job = + job + ~__POS__ + ~name:"oc.misc_checks" + ~image:Images.CI.test + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_lint_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ([ + "./scripts/ci/lint_misc_check.sh"; + "scripts/check_wasm_pvm_regressions.sh check"; + "etherlink/scripts/check_evm_store_migrations.sh check"; + ] + @ + (* The license check only applies to new files (in the sense + of [git add]), so can only run in [before_merging] + pipelines. *) + if pipeline_type = Before_merging then + ["./scripts/ci/lint_check_licenses.sh"] + else []) + in let mr_only_jobs = match pipeline_type with | Before_merging -> @@ -368,7 +426,14 @@ let jobs pipeline_type = ] | _ -> [] in - [job_sanity_ci; job_docker_hadolint] @ mr_only_jobs + [ + job_sanity_ci; + job_docker_hadolint; + job_oc_ocaml_fmt; + job_semgrep; + job_oc_misc_checks; + ] + @ mr_only_jobs in (* The build_x86_64 jobs are split in two to keep the artifact size under the 1GB hard limit set by GitLab. *) @@ -633,34 +698,6 @@ let jobs pipeline_type = ] |> enable_cargo_cache |> enable_sccache in - let job_oc_misc_checks : tezos_job = - job - ~__POS__ - ~name:"oc.misc_checks" - ~image:Images.CI.test - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_lint_files ()) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - ~init_python_venv:true - []) - ([ - "./scripts/ci/lint_misc_check.sh"; - "scripts/check_wasm_pvm_regressions.sh check"; - "etherlink/scripts/check_evm_store_migrations.sh check"; - ] - @ - (* The license check only applies to new files (in the sense - of [git add]), so can only run in [before_merging] - pipelines. *) - if pipeline_type = Before_merging then - ["./scripts/ci/lint_check_licenses.sh"] - else []) - in let job_oc_python_check : tezos_job = job ~__POS__ @@ -677,36 +714,6 @@ let jobs pipeline_type = []) ["./scripts/ci/lint_misc_python_check.sh"] in - let job_oc_ocaml_fmt : tezos_job = - job - ~__POS__ - ~name:"oc.ocaml_fmt" - ~image:Images.CI.build - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_ocaml_fmt_files ()) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ["scripts/lint.sh --check-ocamlformat"; "dune build --profile=dev @fmt"] - in - let job_semgrep : tezos_job = - job - ~__POS__ - ~name:"oc.semgrep" - ~image:Images.semgrep_agent - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_semgrep_files ()) - [ - "echo \"OCaml code linting. For information on how to reproduce \ - locally, check out scripts/semgrep/README.md\""; - "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; - ] - in let jobs_unit : tezos_job list = let build_dependencies = function | Amd64 -> @@ -1348,10 +1355,7 @@ let jobs pipeline_type = job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch; - job_oc_misc_checks; job_oc_python_check; - job_oc_ocaml_fmt; - job_semgrep; job_oc_integration_compiler_rejections; job_oc_script_test_gen_genesis; job_oc_script_snapshot_alpha_and_link; -- GitLab From 64fed439572dd8bbb15aadb2dbfecf5d5a41e1df Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 4 Sep 2024 16:09:04 +0200 Subject: [PATCH 4/4] CI: Put all sanity jobs on critical path for dependent jobs Jobs that execute out-of-stage-order (i.e. jobs that have [needs:]) will not wait on the jobs in the stage [sanity]. This commit ensures that jobs that depend only on [trigger] also depend on all sanity jobs. Since all dependent jobs should transitively depend on [trigger], this should ensure that all dependent jobs transitively depend on the sanity jobs as well. --- .gitlab/ci/pipelines/before_merging.yml | 266 +++++++++++++++++++++--- .gitlab/ci/pipelines/merge_train.yml | 266 +++++++++++++++++++++--- ci/bin/code_verification.ml | 25 ++- 3 files changed, 486 insertions(+), 71 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 0b69196c03d4..4052e02c76c9 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -395,8 +395,20 @@ oc.build:static-x86_64-linux-binaries: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -458,8 +470,20 @@ oc.build_x86_64-released: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -531,8 +555,20 @@ oc.build_x86_64-exp-dev-extra: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -597,8 +633,20 @@ ocaml-check: - tezt/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -902,8 +950,20 @@ opam:prepare: when: delayed start_in: 1 minute needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1789,8 +1849,20 @@ kaitai_checks: - src/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1863,8 +1935,20 @@ oc.check_lift_limits_patch: - src/proto_alpha/lib_protocol/main.ml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1906,8 +1990,20 @@ oc.python_check: - pyproject.toml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2009,8 +2105,20 @@ oc.script:test-gen-genesis: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2131,8 +2239,20 @@ oc.script:b58_prefix: - scripts/b58_prefix/test_b58_prefix.py when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2936,7 +3056,19 @@ oc.install_opam_jammy: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] allow_failure: true timeout: 2 hours @@ -2957,7 +3089,19 @@ oc.compile_sources_doc_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3560,8 +3704,20 @@ commit_titles: tags: - gcp needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 allow_failure: true @@ -3651,7 +3807,19 @@ oc.install_python_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3685,8 +3853,20 @@ documentation:odoc: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3743,8 +3923,20 @@ documentation:manuals: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3802,8 +3994,20 @@ documentation:docgen: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 0b69196c03d4..4052e02c76c9 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -395,8 +395,20 @@ oc.build:static-x86_64-linux-binaries: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -458,8 +470,20 @@ oc.build_x86_64-released: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -531,8 +555,20 @@ oc.build_x86_64-exp-dev-extra: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -597,8 +633,20 @@ ocaml-check: - tezt/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -902,8 +950,20 @@ opam:prepare: when: delayed start_in: 1 minute needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1789,8 +1849,20 @@ kaitai_checks: - src/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1863,8 +1935,20 @@ oc.check_lift_limits_patch: - src/proto_alpha/lib_protocol/main.ml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1906,8 +1990,20 @@ oc.python_check: - pyproject.toml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2009,8 +2105,20 @@ oc.script:test-gen-genesis: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2131,8 +2239,20 @@ oc.script:b58_prefix: - scripts/b58_prefix/test_b58_prefix.py when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2936,7 +3056,19 @@ oc.install_opam_jammy: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] allow_failure: true timeout: 2 hours @@ -2957,7 +3089,19 @@ oc.compile_sources_doc_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3560,8 +3704,20 @@ commit_titles: tags: - gcp needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 allow_failure: true @@ -3651,7 +3807,19 @@ oc.install_python_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3685,8 +3853,20 @@ documentation:odoc: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3743,8 +3923,20 @@ documentation:manuals: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3802,8 +3994,20 @@ documentation:docgen: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 5395206276d8..d97f1871ec71 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -296,18 +296,14 @@ let jobs pipeline_type = in ([job_start], make_dependencies) in - (* Short-cut for jobs that has no dependencies except [job_start] - on [Before_merging] pipelines. *) - let dependencies_needs_start = - make_dependencies - ~before_merging:(fun job_start -> Dependent [Job job_start]) - ~schedule_extended_test:(fun () -> Staged []) - in - (* Sanity jobs *) let sanity = let stage = Stages.sanity in - let dependencies = dependencies_needs_start in + let dependencies = + make_dependencies + ~before_merging:(fun job_start -> Dependent [Job job_start]) + ~schedule_extended_test:(fun () -> Staged []) + in let job_sanity_ci : tezos_job = job ~__POS__ @@ -435,6 +431,17 @@ let jobs pipeline_type = ] @ mr_only_jobs in + (* Short-cut for jobs that have no dependencies except [job_start] on + [Before_merging] pipelines. These jobs must also depend on all + jobs in the stage [sanity], such that they do not run if this + stage does not succeed. Since some sanity jobs are conditional, + we make these dependencies optional. *) + let dependencies_needs_start = + make_dependencies + ~before_merging:(fun job_start -> + Dependent ([Job job_start] @ List.map (fun job -> Optional job) sanity)) + ~schedule_extended_test:(fun () -> Staged []) + in (* The build_x86_64 jobs are split in two to keep the artifact size under the 1GB hard limit set by GitLab. *) (* [job_build_x86_64_release] builds the released executables. *) -- GitLab