diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 5d34c60908df7cc4e14c528b8995afcb0e5df48a..4052e02c76c9268ecf60252eb3b172e2656ef50c 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -132,7 +132,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-before_merging: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: @@ -142,12 +142,99 @@ docker:hadolint-before_merging: - Dockerfile - build.Dockerfile when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes script: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - '**/*.ml' + - '**/*.mli' + - '**/.ocamlformat' + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - devtools/**/* + - scripts/semgrep/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - client-libs/**/* + - contrib/**/* + - devtools/**/* + - docs/**/* + - etherlink/**/* + - scripts/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + - ./scripts/ci/lint_check_licenses.sh + nix: image: nixos/nix:2.22.1 stage: sanity @@ -159,6 +246,8 @@ nix: - flake.lock - scripts/version.sh when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes cache: @@ -306,8 +395,20 @@ oc.build:static-x86_64-linux-binaries: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -369,8 +470,20 @@ oc.build_x86_64-released: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -442,8 +555,20 @@ oc.build_x86_64-exp-dev-extra: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -508,8 +633,20 @@ ocaml-check: - tezt/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -813,8 +950,20 @@ opam:prepare: when: delayed start_in: 1 minute needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1700,8 +1849,20 @@ kaitai_checks: - src/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1774,8 +1935,20 @@ oc.check_lift_limits_patch: - src/proto_alpha/lib_protocol/main.ml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1803,41 +1976,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/**/* - - contrib/**/* - - devtools/**/* - - docs/**/* - - etherlink/**/* - - scripts/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - - ./scripts/ci/lint_check_licenses.sh - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1852,8 +1990,20 @@ oc.python_check: - pyproject.toml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1864,56 +2014,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - '**/.ocamlformat' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - scripts/semgrep/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test @@ -2005,8 +2105,20 @@ oc.script:test-gen-genesis: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2127,8 +2239,20 @@ oc.script:b58_prefix: - scripts/b58_prefix/test_b58_prefix.py when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2932,7 +3056,19 @@ oc.install_opam_jammy: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] allow_failure: true timeout: 2 hours @@ -2953,7 +3089,19 @@ oc.compile_sources_doc_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3556,8 +3704,20 @@ commit_titles: tags: - gcp needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 allow_failure: true @@ -3647,7 +3807,19 @@ oc.install_python_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3681,8 +3853,20 @@ documentation:odoc: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3739,8 +3923,20 @@ documentation:manuals: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3798,8 +3994,20 @@ documentation:docgen: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 5d34c60908df7cc4e14c528b8995afcb0e5df48a..4052e02c76c9268ecf60252eb3b172e2656ef50c 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -132,7 +132,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-before_merging: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: @@ -142,12 +142,99 @@ docker:hadolint-before_merging: - Dockerfile - build.Dockerfile when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes script: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - '**/*.ml' + - '**/*.mli' + - '**/.ocamlformat' + - .gitlab-ci.yml + - .gitlab/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - devtools/**/* + - scripts/semgrep/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - client-libs/**/* + - contrib/**/* + - devtools/**/* + - docs/**/* + - etherlink/**/* + - scripts/**/* + - src/**/* + - tezt/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + - ./scripts/ci/lint_check_licenses.sh + nix: image: nixos/nix:2.22.1 stage: sanity @@ -159,6 +246,8 @@ nix: - flake.lock - scripts/version.sh when: on_success + needs: + - trigger dependencies: [] timeout: 60 minutes cache: @@ -306,8 +395,20 @@ oc.build:static-x86_64-linux-binaries: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -369,8 +470,20 @@ oc.build_x86_64-released: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -442,8 +555,20 @@ oc.build_x86_64-exp-dev-extra: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -508,8 +633,20 @@ ocaml-check: - tezt/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -813,8 +950,20 @@ opam:prepare: when: delayed start_in: 1 minute needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1700,8 +1849,20 @@ kaitai_checks: - src/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1774,8 +1935,20 @@ oc.check_lift_limits_patch: - src/proto_alpha/lib_protocol/main.ml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1803,41 +1976,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/**/* - - contrib/**/* - - devtools/**/* - - docs/**/* - - etherlink/**/* - - scripts/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - - ./scripts/ci/lint_check_licenses.sh - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1852,8 +1990,20 @@ oc.python_check: - pyproject.toml when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -1864,56 +2014,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - '**/.ocamlformat' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - scripts/semgrep/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test @@ -2005,8 +2105,20 @@ oc.script:test-gen-genesis: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2127,8 +2239,20 @@ oc.script:b58_prefix: - scripts/b58_prefix/test_b58_prefix.py when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -2932,7 +3056,19 @@ oc.install_opam_jammy: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] allow_failure: true timeout: 2 hours @@ -2953,7 +3089,19 @@ oc.compile_sources_doc_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3556,8 +3704,20 @@ commit_titles: tags: - gcp needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 allow_failure: true @@ -3647,7 +3807,19 @@ oc.install_python_bookworm: - when: manual allow_failure: true needs: - - trigger + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: [] timeout: 60 minutes script: @@ -3681,8 +3853,20 @@ documentation:odoc: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3739,8 +3923,20 @@ documentation:manuals: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -3798,8 +3994,20 @@ documentation:docgen: - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger + - job: sanity_ci + optional: true + - job: docker:hadolint + optional: true + - job: oc.ocaml_fmt + optional: true + - job: oc.semgrep + optional: true + - job: oc.misc_checks + optional: true + - job: nix + optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index ff4105a7a9d0c050f99d1dfbee8d4b1c4e6b5910..5b6c8a33d5c0153a0290f24708425c931818f77b 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -109,7 +109,7 @@ sanity_ci: - ./scripts/ci/check_alpine_version.sh - make -C ci check -docker:hadolint-schedule_extended_test: +docker:hadolint: image: hadolint/hadolint:2.9.3-debian stage: sanity tags: @@ -122,6 +122,58 @@ docker:hadolint-schedule_extended_test: - hadolint build.Dockerfile - hadolint Dockerfile +oc.ocaml_fmt: + image: ${ci_image_name}/build:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - scripts/lint.sh --check-ocamlformat + - dune build --profile=dev @fmt + +oc.semgrep: + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: [] + timeout: 60 minutes + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out + scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh + +oc.misc_checks: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - ./scripts/ci/lint_misc_check.sh + - scripts/check_wasm_pvm_regressions.sh check + - etherlink/scripts/check_evm_store_migrations.sh check + oc.build_arm64-released: image: ${ci_image_name}/build:${ci_image_tag} stage: build @@ -1398,26 +1450,6 @@ oc.check_lift_limits_patch: CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - oc.python_check: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1435,38 +1467,6 @@ oc.python_check: script: - ./scripts/ci/lint_misc_python_check.sh -oc.ocaml_fmt: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - tags: - - gcp - rules: - - when: always - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - oc.integration:compiler-rejections: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 459381f48f8ae18c40e955c0c6fcbdd4ddbc676c..d97f1871ec7173ba1e0fb069e6c154ab3e67ea0e 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -296,23 +296,21 @@ let jobs pipeline_type = in ([job_start], make_dependencies) in - (* Short-cut for jobs that has no dependencies except [job_start] - on [Before_merging] pipelines. *) - let dependencies_needs_start = - make_dependencies - ~before_merging:(fun job_start -> Dependent [Job job_start]) - ~schedule_extended_test:(fun () -> Staged []) - in - (* Sanity jobs *) let sanity = + let stage = Stages.sanity in + let dependencies = + make_dependencies + ~before_merging:(fun job_start -> Dependent [Job job_start]) + ~schedule_extended_test:(fun () -> Staged []) + in let job_sanity_ci : tezos_job = job ~__POS__ ~name:"sanity_ci" ~image:Images.CI.build - ~stage:Stages.sanity - ~dependencies:dependencies_needs_start + ~stage + ~dependencies ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) [ "make -C manifest check"; @@ -329,7 +327,8 @@ let jobs pipeline_type = ~__POS__ ~name:"nix" ~image:Images.nix - ~stage:Stages.sanity + ~stage + ~dependencies ~artifacts:(artifacts ~when_:On_failure ["flake.lock"]) ~rules: (make_rules @@ -349,17 +348,70 @@ let jobs pipeline_type = job ~rules:(make_rules ~changes:changeset_hadolint_docker_files ()) ~__POS__ - ~name: - (* TODO: I'm not sure it makes sense to have different names for the same job *) - ("docker:hadolint-" - ^ - match pipeline_type with - | Before_merging -> "before_merging" - | Schedule_extended_test -> "schedule_extended_test") + ~name:"docker:hadolint" + ~dependencies ~image:Images.hadolint - ~stage:Stages.sanity + ~stage ["hadolint build.Dockerfile"; "hadolint Dockerfile"] in + let job_oc_ocaml_fmt : tezos_job = + job + ~__POS__ + ~name:"oc.ocaml_fmt" + ~image:Images.CI.build + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_ocaml_fmt_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ["scripts/lint.sh --check-ocamlformat"; "dune build --profile=dev @fmt"] + in + let job_semgrep : tezos_job = + job + ~__POS__ + ~name:"oc.semgrep" + ~image:Images.semgrep_agent + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_semgrep_files ()) + [ + "echo \"OCaml code linting. For information on how to reproduce \ + locally, check out scripts/semgrep/README.md\""; + "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; + ] + in + let job_oc_misc_checks : tezos_job = + job + ~__POS__ + ~name:"oc.misc_checks" + ~image:Images.CI.test + ~stage + ~dependencies + ~rules:(make_rules ~changes:changeset_lint_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ([ + "./scripts/ci/lint_misc_check.sh"; + "scripts/check_wasm_pvm_regressions.sh check"; + "etherlink/scripts/check_evm_store_migrations.sh check"; + ] + @ + (* The license check only applies to new files (in the sense + of [git add]), so can only run in [before_merging] + pipelines. *) + if pipeline_type = Before_merging then + ["./scripts/ci/lint_check_licenses.sh"] + else []) + in let mr_only_jobs = match pipeline_type with | Before_merging -> @@ -370,7 +422,25 @@ let jobs pipeline_type = ] | _ -> [] in - [job_sanity_ci; job_docker_hadolint] @ mr_only_jobs + [ + job_sanity_ci; + job_docker_hadolint; + job_oc_ocaml_fmt; + job_semgrep; + job_oc_misc_checks; + ] + @ mr_only_jobs + in + (* Short-cut for jobs that have no dependencies except [job_start] on + [Before_merging] pipelines. These jobs must also depend on all + jobs in the stage [sanity], such that they do not run if this + stage does not succeed. Since some sanity jobs are conditional, + we make these dependencies optional. *) + let dependencies_needs_start = + make_dependencies + ~before_merging:(fun job_start -> + Dependent ([Job job_start] @ List.map (fun job -> Optional job) sanity)) + ~schedule_extended_test:(fun () -> Staged []) in (* The build_x86_64 jobs are split in two to keep the artifact size under the 1GB hard limit set by GitLab. *) @@ -635,34 +705,6 @@ let jobs pipeline_type = ] |> enable_cargo_cache |> enable_sccache in - let job_oc_misc_checks : tezos_job = - job - ~__POS__ - ~name:"oc.misc_checks" - ~image:Images.CI.test - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_lint_files ()) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - ~init_python_venv:true - []) - ([ - "./scripts/ci/lint_misc_check.sh"; - "scripts/check_wasm_pvm_regressions.sh check"; - "etherlink/scripts/check_evm_store_migrations.sh check"; - ] - @ - (* The license check only applies to new files (in the sense - of [git add]), so can only run in [before_merging] - pipelines. *) - if pipeline_type = Before_merging then - ["./scripts/ci/lint_check_licenses.sh"] - else []) - in let job_oc_python_check : tezos_job = job ~__POS__ @@ -679,36 +721,6 @@ let jobs pipeline_type = []) ["./scripts/ci/lint_misc_python_check.sh"] in - let job_oc_ocaml_fmt : tezos_job = - job - ~__POS__ - ~name:"oc.ocaml_fmt" - ~image:Images.CI.build - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_ocaml_fmt_files ()) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ["scripts/lint.sh --check-ocamlformat"; "dune build --profile=dev @fmt"] - in - let job_semgrep : tezos_job = - job - ~__POS__ - ~name:"oc.semgrep" - ~image:Images.semgrep_agent - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules:(make_rules ~changes:changeset_semgrep_files ()) - [ - "echo \"OCaml code linting. For information on how to reproduce \ - locally, check out scripts/semgrep/README.md\""; - "sh ./scripts/semgrep/lint-all-ocaml-sources.sh"; - ] - in let jobs_unit : tezos_job list = let build_dependencies = function | Amd64 -> @@ -1350,10 +1362,7 @@ let jobs pipeline_type = job_kaitai_checks; job_kaitai_e2e_checks; job_oc_check_lift_limits_patch; - job_oc_misc_checks; job_oc_python_check; - job_oc_ocaml_fmt; - job_semgrep; job_oc_integration_compiler_rejections; job_oc_script_test_gen_genesis; job_oc_script_snapshot_alpha_and_link;