diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 4aed0eca2a1f16fbe01ea2e615f272cd59efc881..3860b660ed2c21c3317a2d4e662da17e26556ac3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -10,6 +10,7 @@ stages: - test_coverage - doc - manual +- consolidate oc.docker:ci:amd64: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 @@ -31,45 +32,6 @@ oc.docker:ci:amd64: reports: dotenv: ci_image_tag.env -oc.docker:client-libs-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: images - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: [] - script: - - ./scripts/ci/docker_client_libs_dependencies_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: client_libs_dependencies_image_tag.env - -oc.docker:jsonnet:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: images - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_jsonnet_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: jsonnet_image_tag.env - oc.docker:rust-toolchain:amd64: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -89,50 +51,6 @@ oc.docker:rust-toolchain:amd64: reports: dotenv: rust_toolchain_image_tag.env -oc.docker:rust-sdk-bindings:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: images - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_rust_sdk_bindings_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: rust_sdk_bindings_image_tag.env - -oc.docker:ci:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: images - tags: - - gcp_arm64 - dependencies: [] - timeout: 90 minutes - before_script: [] - script: - - ./images/ci_create_ci_images.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - ARCH: arm64 - artifacts: - reports: - dotenv: ci_image_tag.env - retry: - max: 1 - when: - - runner_system_failure - trigger: image: datadog/ci:v2.44.0 stage: start @@ -154,261 +72,50 @@ trigger: - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE --tags mr_number:$CI_MERGE_REQUEST_IID -sanity_ci: - image: ${ci_image_name}/build:${ci_image_tag} - stage: sanity - tags: - - gcp - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - make --silent -C manifest check - - make --silent -C ci check - -docker:hadolint: - image: hadolint/hadolint:2.12.0-alpine - stage: sanity - tags: - - gcp - rules: - - changes: - - Dockerfile - - build.Dockerfile - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - hadolint build.Dockerfile - - hadolint Dockerfile - -oc.ocaml_fmt: +oc.build_x86_64-released: image: ${ci_image_name}/build:${ci_image_tag} - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - '**/.ocamlformat' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: dune_cache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - scripts/lint.sh --check-ocamlformat - - dune build --profile=dev @fmt - after_script: - - eval $(opam env) - - dune cache trim --size=5GB - variables: - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - -oc.semgrep: - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: sanity - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - scripts/semgrep/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out - scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - -oc.misc_checks: - image: ${ci_image_name}/test:${ci_image_tag} - stage: sanity + stage: build tags: - - gcp + - gcp_high_cpu rules: - changes: - .gitlab-ci.yml - .gitlab/**/* + - CHANGES.rst + - LICENSES/**/* + - Makefile + - brassaia-eio/**/* + - brassaia/**/* - client-libs/**/* - - contrib/**/* - - devtools/**/* + - cohttp/**/* + - data-encoding/**/* - docs/**/* + - dune + - dune-project + - dune-workspace - etherlink/**/* + - irmin/**/* + - michelson_test_scripts/**/* + - opam/**/* + - opentelemetry/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - script-inputs/**/*/ - scripts/**/* + - scripts/**/*/ + - sdk/**/* - src/**/* - tezt/**/* + - tzt_reference_test_suite/**/* + - vendors/**/* when: on_success needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_check.sh - - scripts/check_wasm_pvm_regressions.sh check - - etherlink/scripts/check_evm_store_migrations.sh check - - ./scripts/check_rollup_node_sql_migrations.sh check - - ./src/bin_dal_node/scripts/check_dal_store_migrations.sh check - - ./scripts/ci/lint_check_licenses.sh - -check_jsonnet: - image: ${jsonnet_image_name}:${jsonnet_image_tag} - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.jsonnet' - when: on_success - needs: - - oc.docker:jsonnet:amd64 - - trigger - dependencies: - - oc.docker:jsonnet:amd64 - timeout: 60 minutes - before_script: - - cd grafazos/ - - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec - - cd ../ - script: - - scripts/lint.sh --check-jsonnet-format - - scripts/lint.sh --check-jsonnet-lint - -check_rust_fmt: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.rs' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - trigger - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - script: - - scripts/check-format-rust.sh - -documentation:rst-check: - image: ${ci_image_name}/test:${ci_image_tag} - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/**/* - when: on_success - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - . $HOME/.venv/bin/activate - script: - - make --silent -C docs sphinx-check - -nix: - image: nixos/nix:2.22.1 - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.nix' - - flake.lock - - scripts/version.sh - when: on_success - needs: - - trigger - dependencies: [] - timeout: 60 minutes - cache: - key: nix-store - paths: - - /nix/store - policy: pull-push - before_script: - - mkdir -p ~/.config/nix - - echo 'extra-experimental-features = flakes nix-command' > ~/.config/nix/nix.conf - script: - - nix run .#ci-check-version-sh-lock - artifacts: - paths: - - flake.lock - when: on_failure - -commit_titles: - image: ${ci_image_name}/prebuild:${ci_image_tag} - stage: sanity - tags: - - gcp - needs: - - oc.docker:ci:amd64 - - trigger + - job: oc.docker:ci:amd64 + - job: trigger dependencies: - oc.docker:ci:amd64 - allow_failure: - exit_codes: 65 - timeout: 60 minutes - script: - - ./scripts/ci/check_commit_messages.sh || exit $? - -oc.build_arm64-released: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_arm64 - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:arm64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG @@ -433,7 +140,7 @@ oc.build_arm64-released: after_script: - ./scripts/ci/sccache-stop.sh variables: - ARCH: arm64 + ARCH: x86_64 EXECUTABLE_FILES: script-inputs/released-executables CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache @@ -441,9 +148,10 @@ oc.build_arm64-released: OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime + COVERAGE_OPTIONS: --instrument-with bisect_ppx artifacts: name: build-$ARCH-$CI_COMMIT_REF_SLUG - expire_in: 1 day + expire_in: 3 days paths: - octez-* - octez-teztale-* @@ -453,71 +161,17 @@ oc.build_arm64-released: - _build/default/contrib/octez_injector_server/octez_injector_server.exe - etherlink-governance-observer when: on_success + retry: + max: 2 + when: + - stuck_or_timeout_failure + - runner_system_failure -oc.build_arm64-exp-dev-extra: - image: ${ci_image_name}/build:${ci_image_tag} +oc.build_dsn_node: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: build tags: - - gcp_arm64 - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/build_full_unreleased.sh - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables - BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe - tezt/tests/main.exe - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime - artifacts: - name: build-$ARCH-$CI_COMMIT_REF_SLUG - expire_in: 1 day - paths: - - octez-* - - octez-teztale-* - - src/proto_*/parameters/*.json - - _build/default/src/lib_protocol_compiler/bin/main_native.exe - - _build/default/tezt/tests/main.exe - - _build/default/contrib/octez_injector_server/octez_injector_server.exe - - etherlink-governance-observer - when: on_success - -oc.build:static-x86_64-linux-binaries: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu + - gcp rules: - changes: - .gitlab-ci.yml @@ -527,13 +181,18 @@ oc.build:static-x86_64-linux-binaries: - Makefile - brassaia-eio/**/* - brassaia/**/* + - client-libs/**/* - cohttp/**/* - data-encoding/**/* + - docs/**/* - dune - dune-project - dune-workspace + - etherlink.mk - etherlink/**/* + - images/**/* - irmin/**/* + - kernels.mk - michelson_test_scripts/**/* - opam/**/* - opentelemetry/**/* @@ -541,82 +200,52 @@ oc.build:static-x86_64-linux-binaries: - resto/**/* - rust-toolchain - script-inputs/**/* + - script-inputs/**/*/ - scripts/**/* + - scripts/**/*/ + - scripts/ci/**/* - sdk/**/* - src/**/* - tezt/**/* - tzt_reference_test_suite/**/* + - vendors/**/* when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - key: sccache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_sccache policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - - $CI_PROJECT_DIR/_target + - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/build_static_binaries.sh + - make -f etherlink.mk octez-dsn-node after_script: - ./scripts/ci/sccache-stop.sh variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/octez-released-executables script-inputs/octez-experimental-executables - CARGO_NET_OFFLINE: "false" + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 2G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime + SCCACHE_CACHE_SIZE: 5G + CARGO_NET_OFFLINE: "false" artifacts: + name: build-dsn-node-$CI_COMMIT_REF_SLUG + expire_in: 3 days paths: - - octez-binaries/$ARCH/* - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure + - octez-dsn-node + when: on_success -oc.build:static-arm64-linux-binaries: +oc.tezt:fetch-records: image: ${ci_image_name}/build:${ci_image_tag} stage: build tags: - - gcp_arm64 + - gcp rules: - changes: - .gitlab-ci.yml @@ -645,73 +274,33 @@ oc.build:static-arm64-linux-binaries: - src/**/* - tezt/**/* - tzt_reference_test_suite/**/* - when: manual - allow_failure: true - needs: - - job: oc.docker:ci:arm64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true + when: on_success dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 + allow_failure: true timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push before_script: - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/build_static_binaries.sh + - dune exec scripts/ci/update_records/update.exe -- --log-file tezt-fetch-records.log + --from last-successful-schedule-extended-test --info after_script: - - ./scripts/ci/sccache-stop.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/octez-released-executables script-inputs/octez-experimental-executables - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 2G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime + - ./scripts/ci/filter_corrupted_records.sh artifacts: + expire_in: 3 days paths: - - octez-binaries/$ARCH/* + - tezt-fetch-records.log + - tezt/records/*.json + - tezt/records/*.json.broken + when: always -oc.build_x86_64-released: - image: ${ci_image_name}/build:${ci_image_tag} +select_tezts: + image: ${ci_image_name}/prebuild:${ci_image_tag} stage: build tags: - - gcp_high_cpu + - gcp rules: - changes: - .gitlab-ci.yml @@ -721,10 +310,8 @@ oc.build_x86_64-released: - Makefile - brassaia-eio/**/* - brassaia/**/* - - client-libs/**/* - cohttp/**/* - data-encoding/**/* - - docs/**/* - dune - dune-project - dune-workspace @@ -737,90 +324,27 @@ oc.build_x86_64-released: - resto/**/* - rust-toolchain - script-inputs/**/* - - script-inputs/**/*/ - scripts/**/* - - scripts/**/*/ - sdk/**/* - src/**/* - tezt/**/* - tzt_reference_test_suite/**/* - - vendors/**/* when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true dependencies: - oc.docker:ci:amd64 + allow_failure: + exit_codes: 17 timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push before_script: - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - eval $(opam env) - - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/build_full_unreleased.sh - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/released-executables - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime - COVERAGE_OPTIONS: --instrument-with bisect_ppx + - scripts/ci/select_tezts.sh || exit $? artifacts: - name: build-$ARCH-$CI_COMMIT_REF_SLUG - expire_in: 1 day + expire_in: 3 days paths: - - octez-* - - octez-teztale-* - - src/proto_*/parameters/*.json - - _build/default/src/lib_protocol_compiler/bin/main_native.exe - - _build/default/tezt/tests/main.exe - - _build/default/contrib/octez_injector_server/octez_injector_server.exe - - etherlink-governance-observer - when: on_success - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure + - selected_tezts.tsl + when: always oc.build_x86_64-exp-dev-extra: image: ${ci_image_name}/build:${ci_image_tag} @@ -864,26 +388,6 @@ oc.build_x86_64-exp-dev-extra: needs: - job: oc.docker:ci:amd64 - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -932,7 +436,7 @@ oc.build_x86_64-exp-dev-extra: DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: name: build-$ARCH-$CI_COMMIT_REF_SLUG - expire_in: 1 day + expire_in: 3 days paths: - octez-* - octez-teztale-* @@ -948,8 +452,8 @@ oc.build_x86_64-exp-dev-extra: - stuck_or_timeout_failure - runner_system_failure -wasm-runtime-check: - image: ${ci_image_name}/build:${ci_image_tag} +oc.build_kernels: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: build tags: - gcp @@ -957,3866 +461,330 @@ wasm-runtime-check: - changes: - .gitlab-ci.yml - .gitlab/**/* - - src/lib_wasm_runtime/**/*.rs + - CHANGES.rst + - LICENSES/**/* + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - client-libs/**/* + - cohttp/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink.mk + - etherlink/**/* + - images/**/* + - irmin/**/* + - kernels.mk + - michelson_test_scripts/**/* + - opam/**/* + - opentelemetry/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - script-inputs/**/*/ + - scripts/**/* + - scripts/**/*/ + - scripts/ci/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + - vendors/**/* when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + - key: kernels-sccache paths: - - $CI_PROJECT_DIR/.cargo/registry/cache + - $CI_PROJECT_DIR/_sccache policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - - $CI_PROJECT_DIR/_sccache + - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - . ./scripts/ci/sccache-start.sh script: - - etherlink/lib_wasm_runtime/lint.sh + - make -f kernels.mk build + - make -f etherlink.mk evm_kernel.wasm + - make -C src/riscv riscv-dummy.elf after_script: - ./scripts/ci/sccache-stop.sh variables: - CARGO_NET_OFFLINE: "false" + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G - -ocaml-check: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu - rules: - - changes: - - '**/*.ml' - - '**/*.mli' - - .gitlab-ci.yml - - .gitlab/**/* - - devtools/**/* - - src/**/* - - tezt/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune_cache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - dune build @check --stop-on-first-error - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -oc.build_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - client-libs/**/* - - cohttp/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink.mk - - etherlink/**/* - - images/**/* - - irmin/**/* - - kernels.mk - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - script-inputs/**/*/ - - scripts/**/* - - scripts/**/*/ - - scripts/ci/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - - vendors/**/* - when: on_success - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: kernels-sccache - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -f kernels.mk build - - make -f etherlink.mk evm_kernel.wasm - - make -C src/riscv riscv-dummy.elf - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - CARGO_NET_OFFLINE: "false" - artifacts: - name: build-kernels-$CI_COMMIT_REF_SLUG - expire_in: 1 day - paths: - - evm_kernel.wasm - - smart-rollup-installer - - sequenced_kernel.wasm - - tx_kernel.wasm - - tx_kernel_dal.wasm - - dal_echo_kernel.wasm - - src/riscv/riscv-dummy.elf - when: on_success - -oc.build_dsn_node: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - client-libs/**/* - - cohttp/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink.mk - - etherlink/**/* - - images/**/* - - irmin/**/* - - kernels.mk - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - script-inputs/**/*/ - - scripts/**/* - - scripts/**/*/ - - scripts/ci/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - - vendors/**/* - when: on_success - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -f etherlink.mk octez-dsn-node - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - CARGO_NET_OFFLINE: "false" - artifacts: - name: build-dsn-node-$CI_COMMIT_REF_SLUG - expire_in: 1 day - paths: - - octez-dsn-node - when: on_success - -oc.tezt:fetch-records: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - dependencies: - - oc.docker:ci:amd64 - allow_failure: true - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - dune exec scripts/ci/update_records/update.exe -- --log-file tezt-fetch-records.log - --from last-successful-schedule-extended-test --info - after_script: - - ./scripts/ci/filter_corrupted_records.sh - artifacts: - expire_in: 4 hours - paths: - - tezt-fetch-records.log - - tezt/records/*.json - - tezt/records/*.json.broken - when: always - -build_octez_source: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu - rules: - - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/restrict_export_to_octez_source.sh - - ./scripts/ci/create_octez_tarball.sh octez - - mv octez.tar.bz2 ../ - - cd ../ - - tar xf octez.tar.bz2 - - cd octez/ - - eval $(opam env) - - make octez - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -build_grafazos_dashboards: - image: ${jsonnet_image_name}:${jsonnet_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - grafazos/**/* - when: always - - when: manual - allow_failure: true - dependencies: - - oc.docker:jsonnet:amd64 - timeout: 60 minutes - before_script: - - cd grafazos/ - - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec - script: - - make - artifacts: - name: grafazos-dashboards - expire_in: 1 day - paths: - - grafazos/output/**/*.json - when: on_success - -teztale.build:static-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - teztale/**/* - when: on_success - - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make teztale - after_script: - - mkdir -p ./teztale-binaries/x86_64 - - mv octez-teztale-* ./teztale-binaries/x86_64/ - - ./scripts/ci/sccache-stop.sh - variables: - PROFILE: static - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - name: teztale-binaries - expire_in: 1 day - paths: - - teztale-binaries/x86_64/octez-teztale-* - when: on_success - -teztale.build:static-arm64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_arm64 - rules: - - changes: - - teztale/**/* - when: on_success - - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make teztale - after_script: - - mkdir -p ./teztale-binaries/arm64 - - mv octez-teztale-* ./teztale-binaries/arm64/ - - ./scripts/ci/sccache-stop.sh - variables: - PROFILE: static - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - name: teztale-binaries - expire_in: 1 day - paths: - - teztale-binaries/arm64/octez-teztale-* - when: on_success - -etherlink.build:static-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - Makefile - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make evm-node-static - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 2G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime - artifacts: - name: evm-binaries - paths: - - octez-evm-* - - etherlink-* - when: on_success - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -etherlink.build:static-arm64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_arm64 - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - Makefile - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make evm-node-static - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 2G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime - artifacts: - name: evm-binaries - paths: - - octez-evm-* - - etherlink-* - when: on_success - -build-layer1-profiling: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - scripts/slim-mode.sh on - - make build OCTEZ_EXECUTABLES?=octez-node - - mkdir -p octez-binaries/x86_64/ - - mv octez-node octez-binaries/x86_64/ - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - TEZOS_PPX_PROFILER: profiling - PROFILE: static - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - expire_in: 1 day - paths: - - ./octez-binaries/x86_64/octez-node - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -select_tezts: - image: ${ci_image_name}/prebuild:${ci_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - dependencies: - - oc.docker:ci:amd64 - allow_failure: - exit_codes: 17 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - scripts/ci/select_tezts.sh || exit $? - artifacts: - expire_in: 3 days - paths: - - selected_tezts.tsl - when: always - -trigger:debian_repository_partial_auto: - stage: test - variables: - PIPELINE_TYPE: debian_repository_partial_auto - rules: - - changes: - - debian-deps-build.Dockerfile - - docs/introduction/install-bin-deb.sh - - docs/introduction/upgrade-bin-deb.sh - - manifest/**/*.ml* - - scripts/ci/build-debian-packages.sh - - scripts/ci/build-debian-packages_current.sh - - scripts/ci/build-packages-dependencies.sh - - scripts/ci/create_debian_repo.sh - - scripts/ci/prepare-apt-repo.sh - - scripts/packaging/Release.conf - - scripts/packaging/build-deb-local.sh - - scripts/packaging/octez/debian/* - - scripts/version.sh - when: on_success - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - trigger: - include: .gitlab/ci/pipelines/debian_repository_partial_auto.yml - strategy: depend - -trigger:rpm_repository_partial_auto: - stage: test - variables: - PIPELINE_TYPE: rpm_repository_partial_auto - rules: - - changes: - - manifest/**/*.ml* - - rpm-deps-build.Dockerfile - - scripts/ci/build-packages-dependencies.sh - - scripts/ci/build-rpm-packages.sh - - scripts/ci/create_rpm_repo.sh - - scripts/ci/prepare-apt-rpm-repo.sh - - scripts/packaging/build-deb-local.sh - - scripts/packaging/octez/rpm/* - - scripts/packaging/tests/rpm/* - - scripts/version.sh - when: on_success - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - trigger: - include: .gitlab/ci/pipelines/rpm_repository_partial_auto.yml - strategy: depend - -trigger:homebrew_auto: - stage: test - variables: - PIPELINE_TYPE: homebrew_auto - rules: - - changes: - - manifest/**/*.ml* - - scripts/ci/install-gsutil.sh - - scripts/packaging/Formula/* - - scripts/packaging/homebrew_install.sh - - scripts/packaging/homebrew_release.sh - - scripts/packaging/test_homebrew_install.sh - - scripts/version.sh - when: on_success - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - trigger: - include: .gitlab/ci/pipelines/homebrew_auto.yml - strategy: depend - -kaitai_checks: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/*kaitai*/**/* - - images/**/* - - scripts/ci/**/* - - src/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings - and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` - and commit the resulting diff.' ; false) - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - expire_in: 1 hour - paths: - - _build/default/client-libs/bin_codec_kaitai/codec.exe - when: on_success - -kaitai_e2e_checks: - image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/*kaitai*/**/* - - images/**/* - - scripts/ci/**/* - - src/**/* - when: on_success - needs: - - oc.docker:client-libs-dependencies - - kaitai_checks - dependencies: - - oc.docker:client-libs-dependencies - - kaitai_checks - timeout: 60 minutes - before_script: - - . ./scripts/version.sh - - . ./scripts/install_build_deps.js.sh - script: - - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files - 2>/dev/null - -oc.check_lift_limits_patch: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - src/bin_tps_evaluation/lift_limits.patch - - src/proto_alpha/lib_protocol/main.ml - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - '[ $(git apply --numstat src/bin_tps_evaluation/lift_limits.patch | cut -f3) = - "src/proto_alpha/lib_protocol/main.ml" ]' - - git apply src/bin_tps_evaluation/lift_limits.patch - - dune build @src/proto_alpha/lib_protocol/check - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -oc.python_check: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - '**/*.py' - - .gitlab-ci.yml - - .gitlab/**/* - - poetry.lock - - pyproject.toml - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - . $HOME/.venv/bin/activate - script: - - ./scripts/ci/lint_misc_python_check.sh - -oc.integration:compiler-rejections: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - dune build @runtest_rejections - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -oc.script:test-gen-genesis: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - eval $(opam env) - - cd scripts/gen-genesis - script: - - dune build gen_genesis.exe - -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - scripts/snapshot_alpha.sh - - scripts/snapshot_alpha_and_link.sh - - scripts/user_activated_upgrade.sh - - src/proto_alpha/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: oc.build_x86_64-released - optional: true - - job: oc.build_x86_64-exp-dev-extra - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/script:snapshot_alpha_and_link.sh - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -oc.script:test_octez_release_versions: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - ./scripts/test_octez_release_version.sh - -oc.script:b58_prefix: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - scripts/b58_prefix/b58_prefix.py - - scripts/b58_prefix/test_b58_prefix.py - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - . ./scripts/version.sh - - . $HOME/.venv/bin/activate - script: - - poetry run pylint scripts/b58_prefix/b58_prefix.py --disable=missing-docstring - --disable=invalid-name - - poetry run pytest scripts/b58_prefix/test_b58_prefix.py - -oc.test-liquidity-baking-scripts: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - scripts/check-liquidity-baking-scripts.sh - - scripts/ci/test_liquidity_baking_scripts.sh - - src/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - timeout: 60 minutes - before_script: - - . ./scripts/version.sh - - eval $(opam env) - script: - - ./scripts/ci/test_liquidity_baking_scripts.sh - -oc:scripts:release_script_values: - image: ${ci_image_name}/prebuild:${ci_image_tag} - stage: test - tags: - - gcp - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - script: - - scripts/ci/test_release_values.sh - -mir_unit: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - contrib/mir/**/* - - images/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - script: - - cargo test --manifest-path contrib/mir/Cargo.toml - variables: - CARGO_NET_OFFLINE: "false" - -mir_tzt: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - contrib/mir/**/* - - images/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - script: - - cargo run --manifest-path contrib/mir/Cargo.toml --bin tzt_runner tzt_reference_test_suite/*.tzt - variables: - CARGO_NET_OFFLINE: "false" - -test_sdk_rust: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - when: on_success - needs: - - job: oc.docker:rust-toolchain:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C sdk/rust check - - make -C sdk/rust test - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_sdk_bindings: - image: ${rust_sdk_bindings_image_name}:${rust_sdk_bindings_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - contrib/sdk-bindings - - images/**/* - - sdk/rust/**/* - when: on_success - needs: - - job: oc.docker:rust-sdk-bindings:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:rust-sdk-bindings:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . $HOME/.venv/bin/activate - - . ./scripts/ci/sccache-start.sh - script: - - make -C contrib/sdk-bindings check - - make -C contrib/sdk-bindings test - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - kernels.mk - - sdk/rust/**/* - - src/kernel_*/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - oc.build_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -f kernels.mk check - - make -f kernels.mk test - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_etherlink_kernel: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - etherlink.mk - - etherlink/**/*.rs - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - oc.build_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -f etherlink.mk check - - make -f etherlink.mk test - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_etherlink_firehose: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - etherlink/firehose/**/* - - etherlink/tezt/tests/evm_kernel_inputs/erc20tok.* - - images/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - oc.build_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C etherlink/firehose check - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -audit_riscv_deps: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: sanity - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - - src/riscv/**/* - when: on_success - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C src/riscv audit - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -check_riscv_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - - src/riscv/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - audit_riscv_deps - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C src/riscv CHECK_FLAGS= EXTRA_FLAGS='--no-default-features --features ci' - check - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_riscv_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - - src/riscv/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - check_riscv_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C src/riscv EXTRA_FLAGS='--no-default-features --features ci' test - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_long_riscv_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - - src/riscv/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - check_riscv_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C src/riscv EXTRA_FLAGS='--no-default-features --features ci' test-long - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_miri_riscv_kernels: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - images/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - - src/riscv/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - check_riscv_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -C src/riscv EXTRA_FLAGS='--no-default-features --features ci' test-miri - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -test_evm_compatibility: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - etherlink.mk - - etherlink/kernel_latest/evm_evaluation/**/* - - etherlink/kernel_latest/evm_execution/**/* - - images/**/* - when: on_success - needs: - - oc.docker:rust-toolchain:amd64 - - oc.build_kernels - dependencies: - - oc.docker:rust-toolchain:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/ci/sccache-start.sh - script: - - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs evm-evaluation-assessor - - git clone --depth 1 --branch v14.1@etherlink https://github.com/functori/tests - ethereum_tests - - ./evm-evaluation-assessor --eth-tests ./ethereum_tests/ --resources ./etherlink/kernel_latest/evm_evaluation/resources/ - -c - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - -oc.unit:non-proto-x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make $MAKE_TARGETS - - ./scripts/ci/merge_coverage.sh - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - ARCH: x86_64 - MAKE_TARGETS: test-nonproto-unit - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - COVERAGE_OPTIONS: --instrument-with bisect_ppx - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 1 day - paths: - - test_results - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: test_results/*.xml - when: always - retry: 2 - -oc.unit:etherlink-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - Makefile - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make $MAKE_TARGETS - - ./scripts/ci/merge_coverage.sh - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - ARCH: x86_64 - MAKE_TARGETS: test-etherlink-unit - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - COVERAGE_OPTIONS: --instrument-with bisect_ppx - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 1 day - paths: - - test_results - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: test_results/*.xml - when: always - retry: 2 - -oc.unit:other-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_high_cpu - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make $MAKE_TARGETS - - ./scripts/ci/merge_coverage.sh - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - ARCH: x86_64 - MAKE_TARGETS: test-other-unit - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - COVERAGE_OPTIONS: --instrument-with bisect_ppx - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 1 day - paths: - - test_results - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: test_results/*.xml - when: always - retry: 2 - -oc.unit:proto-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make $MAKE_TARGETS - - ./scripts/ci/merge_coverage.sh - after_script: - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - ARCH: x86_64 - MAKE_TARGETS: test-proto-unit - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - COVERAGE_OPTIONS: --instrument-with bisect_ppx - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 1 day - paths: - - test_results - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: test_results/*.xml - when: always - retry: 2 - -oc.unit:non-proto-arm64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:arm64 - - oc.build_arm64-released - - oc.build_arm64-exp-dev-extra - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make $MAKE_TARGETS - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - ARCH: arm64 - MAKE_TARGETS: test-nonproto-unit test-webassembly - DISTRIBUTE_TESTS_TO_PARALLELS: "true" - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-${ARCH} - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 - parallel: 2 - -oc.unit:webassembly-x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 20 minutes - before_script: - - . ./scripts/version.sh - - eval $(opam env) - script: - - make test-webassembly - -oc.unit:protocol_compiles: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - dune build @runtest_compile_protocol - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -de.unit:x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - data-encoding/** - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - eval $(opam env) - script: - - dune runtest data-encoding - -de.unit:arm64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - data-encoding/** - when: on_success - needs: - - oc.docker:ci:arm64 - - oc.build_arm64-released - - oc.build_arm64-exp-dev-extra - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - before_script: - - eval $(opam env) - script: - - dune runtest data-encoding - -resto.unit:x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - resto/** - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - before_script: - - eval $(opam env) - script: - - dune runtest resto - -resto.unit:arm64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - resto/** - when: on_success - needs: - - oc.docker:ci:arm64 - - oc.build_arm64-released - - oc.build_arm64-exp-dev-extra - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - before_script: - - eval $(opam env) - script: - - dune runtest resto - -oc.install_opam_noble: - image: ocaml/opam:ubuntu-24.04 - stage: test - tags: - - gcp - rules: - - when: manual - allow_failure: true - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: [] - allow_failure: true - timeout: 2 hours - script: - - ./docs/introduction/install-opam.sh - variables: - OPAMJOBS: "4" - -oc.compile_sources_doc_bookworm: - image: ocaml/opam:debian-12 - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - docs/introduction/compile-sources.sh - when: on_success - - when: manual - allow_failure: true - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: [] - timeout: 60 minutes - script: - - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - variables: - CARGO_HOME: /home/opam/.cargo - CARGO_NET_OFFLINE: "false" - retry: - max: 2 - when: - - stuck_or_timeout_failure - - runner_system_failure - -tezt: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - timeout: 40 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - - ./scripts/ci/merge_coverage.sh - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: "" - TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && - /cloud && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "3" - TEZT_NO_NPX: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 100 - -tezt-memory-3k: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt_memory_3k - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - - ./scripts/ci/merge_coverage.sh - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -memory_3k - TESTS: memory_3k && /ci_disabled && /memory_4k && /time_sensitive && /slow && - /cloud - TEZT_RETRY: "1" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 12 - -tezt-memory-4k: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt_memory_4k - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - - ./scripts/ci/merge_coverage.sh - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -memory_4k - TESTS: memory_4k && /ci_disabled && /memory_3k && /time_sensitive && /slow && - /cloud - TEZT_RETRY: "1" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 4 - -tezt-time-sensitive: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - - ./scripts/ci/merge_coverage.sh - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -time_sensitive - TESTS: time_sensitive && /ci_disabled && /memory_3k && /memory_4k && /slow && - /cloud - TEZT_RETRY: "1" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: $JUNIT - when: always - retry: 2 - -tezt-slow: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: manual - allow_failure: true - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -slow - TESTS: slow && /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && - /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) - TEZT_RETRY: "1" - TEZT_PARALLEL: "3" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 20 - -tezt-flaky: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp_tezt - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: manual - allow_failure: true - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - - oc.tezt:fetch-records - allow_failure: true - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - - ./scripts/ci/merge_coverage.sh - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -flaky - TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && - /cloud && flaky - TEZT_RETRY: "3" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - artifacts: - name: coverage-files-$CI_JOB_ID - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE/$CI_JOB_NAME_SLUG.* - reports: - junit: $JUNIT - when: always - retry: 2 - -tezt:static-binaries: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-exp-dev-extra - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - dependencies: - - oc.docker:ci:amd64 - - select_tezts - - oc.build_x86_64-exp-dev-extra - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - timeout: 60 minutes - before_script: - - mv octez-binaries/x86_64/octez-* . - script: - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record - tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: "" - TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && - /cloud && cli && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "3" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - reports: - junit: $JUNIT - when: always - -oc.unified_coverage: - image: ${ci_image_name}/e2etest:${ci_image_tag} - stage: test_coverage - tags: - - gcp - rules: - - if: $GITLAB_USER_LOGIN == "nomadic-margebot" || $CI_MERGE_REQUEST_ASSIGNEES =~ - /nomadic-margebot/ || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" - when: never - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - CHANGES.rst - - LICENSES/**/* - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - cohttp/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - michelson_test_scripts/**/* - - opam/**/* - - opentelemetry/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - dependencies: - - oc.docker:ci:amd64 - - oc.unit:non-proto-x86_64 - - oc.unit:etherlink-x86_64 - - oc.unit:other-x86_64 - - oc.unit:proto-x86_64 - - tezt - - tezt-memory-3k - - tezt-memory-4k - - tezt-time-sensitive - - tezt-flaky - allow_failure: - exit_codes: 64 - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - script: - - ./scripts/ci/report_coverage.sh || exit $? - variables: - TEZOS_WITHOUT_OPAM: "true" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - SLACK_COVERAGE_CHANNEL: C02PHBE7W73 - artifacts: - expire_in: 15 days - paths: - - _coverage_report/ - - $BISECT_FILE - reports: - coverage_report: - coverage_format: cobertura - path: _coverage_report/cobertura.xml - when: always - expose_as: Coverage report - coverage: '/Coverage: ([^%]+%)/' - -oc.install_python_bookworm: - image: debian:bookworm - stage: doc - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - docs/developer/install-python-debian-ubuntu.sh - - poetry.lock - - pyproject.toml - when: on_success - - when: manual - allow_failure: true - needs: - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: [] - timeout: 60 minutes - script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - -documentation:odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: doc - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - brassaia/**/* - - client-libs/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/* - - tezt/**/* - - vendors/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs odoc-lite - after_script: - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - artifacts: - expire_in: 1 hour - paths: - - docs/_build/api/odoc/ - - docs/odoc.log - when: always - -documentation:manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: doc - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - brassaia/**/* - - client-libs/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/* - - tezt/**/* - - vendors/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - dependencies: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_x86_64-exp-dev-extra - - oc.build_kernels - - oc.build_dsn_node - timeout: 60 minutes - before_script: - - eval $(opam env) - script: - - make -C docs -j octez-gen - artifacts: - expire_in: 1 week - paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json - -documentation:docgen: - image: ${ci_image_name}/test:${ci_image_tag} - stage: doc - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - brassaia/**/* - - client-libs/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/* - - tezt/**/* - - vendors/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: sanity_ci - optional: true - - job: docker:hadolint - optional: true - - job: oc.ocaml_fmt - optional: true - - job: oc.semgrep - optional: true - - job: oc.misc_checks - optional: true - - job: check_jsonnet - optional: true - - job: check_rust_fmt - optional: true - - job: documentation:rst-check - optional: true - - job: nix - optional: true - - job: commit_titles - optional: true - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - before_script: - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs -j docexes-gen - after_script: - - ./scripts/ci/sccache-stop.sh - variables: CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 1 week + name: build-kernels-$CI_COMMIT_REF_SLUG + expire_in: 3 days paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - evm_kernel.wasm + - smart-rollup-installer + - sequenced_kernel.wasm + - tx_kernel.wasm + - tx_kernel_dal.wasm + - dal_echo_kernel.wasm + - src/riscv/riscv-dummy.elf + when: on_success -documentation:build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: doc +build_memstat: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - brassaia/**/* - - client-libs/**/* - - data-encoding/**/* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/* - - tezt/**/* - - vendors/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen + - gcp_dev dependencies: - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen timeout: 60 minutes before_script: + - ./scripts/ci/take_ownership.sh - eval $(opam env) - - . $HOME/.venv/bin/activate script: - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + - dune build memstat artifacts: - expire_in: 1 week + expire_in: 7 days paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - _build/default/memstat/main.exe + when: always -documentation:linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} - stage: doc +tezt_official: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt_dev rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - .gitlab-ci.yml - .gitlab/**/* + - CHANGES.rst + - LICENSES/**/* + - Makefile + - brassaia-eio/**/* - brassaia/**/* - - client-libs/**/* + - cohttp/**/* - data-encoding/**/* - - docs/**/* - dune - dune-project - dune-workspace - etherlink/**/* - irmin/**/* - - script-inputs/**/*/ - - scripts/**/*/ + - michelson_test_scripts/**/* + - opam/**/* + - opentelemetry/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* - src/**/* - tezt/**/* - - vendors/**/* - when: manual - allow_failure: true + - tzt_reference_test_suite/**/* + when: on_success needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - allow_failure: true - timeout: 60 minutes + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + timeout: 40 minutes before_script: + - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate script: - - make -C docs redirectcheck - - make -C docs linkcheck - -trigger:homebrew: - stage: manual - variables: - PIPELINE_TYPE: homebrew - rules: - - when: manual - allow_failure: true - needs: [] - trigger: - include: .gitlab/ci/pipelines/homebrew.yml - strategy: depend - -trigger:rpm_repository_partial: - stage: manual - variables: - PIPELINE_TYPE: rpm_repository_partial - rules: - - when: manual - allow_failure: true - needs: [] - trigger: - include: .gitlab/ci/pipelines/rpm_repository_partial.yml - strategy: depend - -trigger:debian_repository_partial: - stage: manual + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --with-select-tezts "${TESTS}" --from-record tezt/records + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + --with-select-tezts "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record + tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} + - ./scripts/ci/merge_coverage.sh variables: - PIPELINE_TYPE: debian_repository_partial - rules: - - when: manual - allow_failure: true - needs: [] - trigger: - include: .gitlab/ci/pipelines/debian_repository_partial.yml - strategy: depend + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + TEZT_NO_NPX: "true" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE/$CI_JOB_NAME_SLUG.* + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 100 -oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: manual +tezt_fixed_list: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_very_high_cpu_dev rules: - - changes: - - Dockerfile - - build.Dockerfile - when: on_success - when: manual - allow_failure: true needs: - - oc.docker:rust-toolchain:amd64 - oc.docker:ci:amd64 + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat dependencies: - - oc.docker:rust-toolchain:amd64 - oc.docker:ci:amd64 - timeout: 60 minutes + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat + timeout: 7 days before_script: - - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/version.sh script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind + - tezt_analysis/v2/run_tezt_by_group_index.sh 1 1 ${REPORT_OUTPUT} tezt_analysis/v2/input_tests/tests.txt variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - DOCKER_BUILD_TARGET: with-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + REPORT_OUTPUT: tezt_analysis/reports/memory_stats_fixed.csv + artifacts: + expire_in: 7 days + paths: + - tezt_analysis/reports/*.csv + - tezt_analysis/reports/*.log -oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: manual +tezt_experimental: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp_arm64 + - gcp_tezt_dev rules: - - changes: - - Dockerfile - - build.Dockerfile - when: on_success - when: manual - allow_failure: true needs: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes + - oc.docker:ci:amd64 + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat + timeout: 7 days before_script: - - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/version.sh script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind + #- sleep 10800 # 3 hours + - tezt_analysis/v2/run_experimental.sh variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + REPORT_FOLDER: tezt_analysis/reports + artifacts: + expire_in: 7 days + paths: + - tezt_analysis/reports/*.csv + - tezt_analysis/reports/*.log -oc.script.docker_verify_image_arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: manual + +tezt_full: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp - rules: - - when: manual - allow_failure: true + - gcp_very_high_cpu_dev + #rules: + #- when: manual needs: - - oc.docker:arm64 - dependencies: [] - timeout: 60 minutes + - oc.docker:ci:amd64 + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat + dependencies: + - oc.docker:ci:amd64 + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.build_dsn_node + - oc.tezt:fetch-records + - build_memstat + timeout: 7 days before_script: - - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/version.sh script: - - ./scripts/ci/docker_verify_signature.sh - services: - - docker:${DOCKER_VERSION}-dind + #- sleep 10800 # 3 hours + - tezt_analysis/v2/run_tezt_by_group_index.sh ${CI_NODE_INDEX} ${CI_NODE_TOTAL} ${REPORT_OUTPUT} + #- tezt_analysis/v2/run_tezt_by_group_index.sh ${CI_NODE_INDEX} ${CI_NODE_TOTAL} ${REPORT_OUTPUT} tezt_analysis/v2/input_tests/tests.txt variables: - DOCKER_VERSION: 24.0.7 - IMAGE_ARCH_PREFIX: arm64_ + REPORT_OUTPUT: tezt_analysis/reports/memory_stats_part_${CI_NODE_INDEX}_of_${CI_NODE_TOTAL}.csv + artifacts: + expire_in: 7 days + paths: + - tezt_analysis/reports/*.csv + parallel: 49 -oc.script.docker_verify_image_amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: manual +consolidate_tezt_reports: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: consolidate tags: - - gcp - rules: - - when: manual - allow_failure: true + - gcp_dev needs: - - oc.docker:amd64 - dependencies: [] + - oc.docker:ci:amd64 + - tezt_full + dependencies: + - oc.docker:ci:amd64 + - tezt_full timeout: 60 minutes before_script: - - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/version.sh script: - - ./scripts/ci/docker_verify_signature.sh - services: - - docker:${DOCKER_VERSION}-dind + - tezt_analysis/v2/consolidate_tezt_reports.sh ${REPORT_OUTPUT} variables: - DOCKER_VERSION: 24.0.7 - IMAGE_ARCH_PREFIX: amd64_ + REPORT_OUTPUT: tezt_analysis/reports/memory_stats_part_${CI_NODE_INDEX}_of_${CI_NODE_TOTAL}.csv + artifacts: + expire_in: 7 days + paths: + - tezt_analysis/reports/memory_stats_merged.csv + - tezt_analysis/reports/memory_stats_sorted.csv diff --git a/memstat/dune b/memstat/dune new file mode 100644 index 0000000000000000000000000000000000000000..d8d38e1481d64f951728815bbb9b367bbed12667 --- /dev/null +++ b/memstat/dune @@ -0,0 +1,3 @@ +(executable + (name main) + (libraries clap unix)) diff --git a/memstat/main.ml b/memstat/main.ml new file mode 100644 index 0000000000000000000000000000000000000000..b066d30812fbfe9949fa1fea7b0f68e22bacf9cf --- /dev/null +++ b/memstat/main.ml @@ -0,0 +1,360 @@ +let sf = Printf.sprintf + +let ( let* ) = Result.bind + +let echo x = Printf.ksprintf print_endline x + +let echo_e x = Printf.ksprintf prerr_endline x + +let list_map_r l f = + let rec loop acc = function + | [] -> Ok (List.rev acc) + | head :: tail -> + let* head = f head in + loop (head :: acc) tail + in + loop [] l + +type peak_mode = No_peak | Peak | Only_peak + +(* Shared memory is divided proportional to the number of processes that share it. *) +(* https://stackoverflow.com/questions/131303/how-can-i-measure-the-actual-memory-usage-of-an-application-or-process *) +let get_memory pid = + try + let ch = open_in ("/proc/" ^ string_of_int pid ^ "/smaps") in + Fun.protect ~finally:(fun () -> close_in ch) @@ fun () -> + let rec sum_pss acc = + match input_line ch with + | exception End_of_file -> acc + | line -> + let parse_error () = + failwith (Printf.sprintf "failed to parse %S" line) + in + let line = String.trim line in + let value = + if not (String.starts_with ~prefix:"Pss:" line) then 0 + else if not (String.ends_with ~suffix:"kB" line) then parse_error () + else + let value = + String.sub + line + (String.length "Pss:") + (String.length line - String.length "Pss:" + - String.length "kB") + |> String.trim + in + match int_of_string_opt value with + | None -> parse_error () + | Some value -> value + in + sum_pss (acc + value) + in + Ok (sum_pss 0) + with Sys_error message | Failure message -> Error message + +type process_info = { + pid : int; + command : string; + memory_kb : int; + children : process_info list; + total_memory_kb : int; +} + +let merge_into_peak_info info peak_info_ref = + match !peak_info_ref with + | None -> peak_info_ref := Some info + | Some old -> + if info.total_memory_kb > old.total_memory_kb then + peak_info_ref := Some info + +let rec input_lines ?(acc = []) ch = + match input_line ch with + | exception End_of_file -> List.rev acc + | line -> input_lines ~acc:(line :: acc) ch + +let get_children pid = + try + (* I think a task is the same as a thread. *) + let task_path = "/proc/" ^ string_of_int pid ^ "/task" in + let parse_tid line = + match int_of_string_opt (String.trim line) with + | None -> + failwith (sf "failed to parse %S from %s as a TID" line task_path) + | Some pid -> pid + in + let task_ids = + Sys.readdir task_path |> Array.to_list |> List.map parse_tid + in + let get_task_children tid = + let task_children_path = + task_path ^ "/" ^ string_of_int tid ^ "/children" + in + let ch = open_in task_children_path in + Fun.protect ~finally:(fun () -> close_in ch) @@ fun () -> + let parse_pid line = + match int_of_string_opt line with + | None -> + failwith + (sf "failed to parse %S from %s as a PID" line task_children_path) + | Some pid -> pid + in + input_lines ch + |> List.map (String.split_on_char ' ') + |> List.flatten |> List.map String.trim + |> List.filter (( <> ) "") + |> List.map parse_pid + in + Ok (List.flatten (List.map get_task_children task_ids)) + with Sys_error message | Failure message -> Error message + +let get_command pid = + let path = "/proc/" ^ string_of_int pid ^ "/comm" in + try + let ch = open_in path in + Fun.protect ~finally:(fun () -> close_in ch) @@ fun () -> Ok (input_line ch) + with Sys_error message -> Error message + +let rec get_process_info ~recursively pid = + let* command = get_command pid in + let* memory_kb = get_memory pid in + let* children = + if recursively then + let* children = get_children pid in + list_map_r children (get_process_info ~recursively) + else Ok [] + in + let total_memory_kb = + List.fold_left + (fun acc child -> acc + child.total_memory_kb) + memory_kb + children + in + Ok {pid; command; memory_kb; children; total_memory_kb} + +let show_bytes ~bytes kb = + if bytes then + (* TODO: is it 1024? Should be 1000 if kB, 1024 would be kiB, but need to check. *) + Printf.sprintf "%d" (kb * 1000) + else Printf.sprintf "%d kB" kb + +let output_process_info ~verbose ~bytes info = + (if verbose then + let rec loop level info = + let indent = String.make (level * 2) ' ' in + echo + "%s[%d] %s (%s)" + indent + info.pid + (show_bytes ~bytes info.memory_kb) + info.command ; + List.iter (loop (level + 1)) info.children + in + loop 0 info) ; + let bytes_string = show_bytes ~bytes info.total_memory_kb in + if bytes then echo "%s" bytes_string else echo "Total: %s" bytes_string + +let output_peak_info ~verbose ~peak ~bytes peak_info_opt = + match (peak, peak_info_opt) with + | No_peak, _ | _, None -> () + | Peak, Some peak_info -> + echo "Peak:" ; + output_process_info ~verbose ~bytes peak_info + | Only_peak, Some peak_info -> output_process_info ~verbose ~bytes peak_info + +let with_openfile path flags f = + match Unix.openfile path flags 0o600 with + | exception Unix.Unix_error (code, _, _) -> Error (Unix.error_message code) + | file -> Fun.protect ~finally:(fun () -> Unix.close file) @@ fun () -> f file + +let watch_child ~recursively ~verbose ~frequency ~peak ~bytes pid = + let peak_info = ref None in + let rec loop () = + match Unix.waitpid [WNOHANG] pid with + | exception Unix.Unix_error (code, _, _) -> Error (Unix.error_message code) + | pid', status -> + if pid' = pid then ( + match status with + | WEXITED n -> + if verbose then echo "Process exited with code %d" n ; + Ok () + | WSIGNALED _ -> + if verbose then echo "Process was killed by signal" ; + Ok () + | WSTOPPED _ -> + if verbose then echo "Process was stopped by signal" ; + Ok ()) + else ( + (match get_process_info ~recursively pid with + | Error message -> echo_e "Failed to get process info: %s" message + | Ok info -> ( + merge_into_peak_info info peak_info ; + match peak with + | No_peak | Peak -> output_process_info ~verbose ~bytes info + | Only_peak -> ())) ; + Unix.sleepf (1. /. frequency) ; + loop ()) + in + let result = loop () in + output_peak_info ~verbose ~peak ~bytes !peak_info ; + result + +let watch_pid ~recursively ~verbose ~frequency ~bytes pid = + let peak_info = ref None in + let rec loop () = + (match get_process_info ~recursively pid with + | Error message -> echo_e "Failed to get process info: %s" message + | Ok info -> + merge_into_peak_info info peak_info ; + output_process_info ~verbose ~bytes info) ; + Unix.sleepf (1. /. frequency) ; + loop () + in + loop () + +let run_and_watch ~recursively ~verbose ~frequency ~peak ~bytes ~stdout_path + ~stderr_path executable arguments = + with_openfile stdout_path [O_WRONLY] @@ fun process_stdout -> + with_openfile stderr_path [O_WRONLY] @@ fun process_stderr -> + let argv = Array.of_list (executable :: arguments) in + match + Unix.create_process executable argv Unix.stdin process_stdout process_stderr + with + | exception Unix.Unix_error (code, _, _) -> Error (Unix.error_message code) + | pid -> watch_child ~recursively ~verbose ~frequency ~peak ~bytes pid + +let () = + Clap.description "Measure a process memory usage." ; + let verbose = + Clap.flag + ~set_long:"verbose" + ~set_short:'v' + ~description: + "Output more information, such as information about child processes." + false + in + let recursively = + Clap.flag + ~set_long:"recursively" + ~set_short:'r' + ~description: + "Measure memory usage of child processes as well, recursively, to \ + output the total." + false + in + let frequency = + Clap.default_float + ~long:"frequency" + ~short:'f' + ~description: + "How many times to measure per second, if measuring more than once." + 10. + in + let peak = + Clap.flag_enum + [ + (["no-peak"], [], No_peak); + (["peak"], [], Peak); + (["only-peak"], [], Only_peak); + ] + ~description: + "Whether to output peak memory usage at the end (--peak or \ + --only-peak), or not (--no-peak). With --only-peak, do not output \ + values other than peak memory usage, and do not output the 'Peak:' \ + line, making it easier for scripts. Only works with the 'run' command \ + for now." + Peak + in + let bytes = + Clap.flag + ~set_long:"bytes" + ~description: + "Output memory usage in bytes instead of kilobytes, and without the \ + unit. Also, do not prefix the total with 'Total:'. Useful for scripts \ + that just want the value. Usually used without --verbose, but can be \ + used with --recursively to output the total, including children \ + memory usage." + false + in + let command = + Clap.subcommand + [ + ( Clap.case + "get" + ~description: + "Measure memory usage of a process that is already running." + @@ fun () -> + let watch = + Clap.flag + ~set_long:"watch" + ~set_short:'w' + ~description:"Repeat the measurement forever." + false + in + let pid = + Clap.mandatory_int + ~placeholder:"PID" + ~description:"PID of the process to measure." + () + in + `get (pid, watch) ); + ( Clap.case + "run" + ~description: + "Run a program and measure its memory usage until it exists." + @@ fun () -> + let stdout_path = + Clap.default_string + ~long:"redirect-process-stdout-to" + ~short:'O' + ~placeholder:"PATH" + ~description: + "Redirect the standard output channel of the process to PATH." + "/dev/null" + in + let stderr_path = + Clap.default_string + ~long:"redirect-process-stderr-to" + ~short:'E' + ~placeholder:"PATH" + ~description: + "Redirect the standard error channel of the process to PATH." + "/dev/null" + in + let executable = + Clap.mandatory_string + ~placeholder:"EXECUTABLE" + ~description:"Name of the executable to run." + () + in + let arguments = + Clap.list_string + ~placeholder:"ARGUMENTS" + ~description:"Arguments to pass to EXECUTABLE." + () + in + `run (executable, arguments, stdout_path, stderr_path) ); + ] + in + Clap.close () ; + + let result = + match command with + | `get (pid, watch) -> + if watch then watch_pid ~recursively ~verbose ~frequency ~bytes pid + else + let* info = get_process_info ~recursively pid in + output_process_info ~verbose ~bytes info ; + Ok () + | `run (executable, arguments, stdout_path, stderr_path) -> + run_and_watch + ~recursively + ~verbose + ~frequency + ~peak + ~bytes + ~stdout_path + ~stderr_path + executable + arguments + in + match result with Ok () -> () | Error message -> echo_e "Error: %s" message diff --git a/tezt_analysis/v1/memory_tracker.sh b/tezt_analysis/v1/memory_tracker.sh new file mode 100755 index 0000000000000000000000000000000000000000..0194fa9661a7cda95e10f5672dd157e21b6f6f5c --- /dev/null +++ b/tezt_analysis/v1/memory_tracker.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +sudo apk add procps bash findutils + +MAIN_PID=$1 +OUTPUT_FILE=$2 +INTERVAL=0.5 # Check every half second +MAX_MEM=0 +MAX_CPU=0 + +echo "Starting monitoring for PID $MAIN_PID and children" > $OUTPUT_FILE + +while kill -0 $MAIN_PID 2> /dev/null; do + # Get all child PIDs + ALL_PIDS=$(pgrep -P $MAIN_PID | tr '\n' ' ') + ALL_PIDS="$MAIN_PID $ALL_PIDS" + + # Get total memory and CPU usage + TOTAL_MEM=0 + TOTAL_CPU=0 + + for pid in $ALL_PIDS; do + if [ -e /proc/$pid/status ]; then + # Get memory in KB + MEM=$(grep VmRSS /proc/$pid/status | awk '{print $2}') + if [ ! -z "$MEM" ]; then + TOTAL_MEM=$((TOTAL_MEM + MEM)) + fi + + # Get CPU usage + CPU=$(ps -p $pid -o %cpu= 2> /dev/null || echo "0") + TOTAL_CPU=$(echo "$TOTAL_CPU + $CPU" | bc) + fi + done + + # Update max values + if [ $TOTAL_MEM -gt $MAX_MEM ]; then + MAX_MEM=$TOTAL_MEM + echo "$(date +%H:%M:%S) - New peak memory: $MAX_MEM KB" >> $OUTPUT_FILE + fi + + if ( ($(echo "$TOTAL_CPU > $MAX_CPU" | bc -l))); then + MAX_CPU=$TOTAL_CPU + echo "$(date +%H:%M:%S) - New peak CPU: $MAX_CPU%" >> $OUTPUT_FILE + fi + + sleep $INTERVAL +done + +echo "=== SUMMARY ===" >> $OUTPUT_FILE +echo "Peak memory usage: $MAX_MEM KB ($(echo "scale=2; $MAX_MEM/1024" | bc) MB)" >> $OUTPUT_FILE +echo "Peak CPU usage: $MAX_CPU%" >> $OUTPUT_FILE diff --git a/tezt_analysis/v1/run_tezt.sh b/tezt_analysis/v1/run_tezt.sh new file mode 100755 index 0000000000000000000000000000000000000000..69abb15636ed6f8eb7a02419bf5cc541e7f7d60c --- /dev/null +++ b/tezt_analysis/v1/run_tezt.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# Run tezt test in background +_build/default/tezt/tests/main.exe -t 'Alpha: Sequencer produces an empty block in case of upgrade. (sequencer, latest, without dal, single chain)' & +TEST_PID=$! + +# Monitor it +touch memory_stats.txt +tezt_analysis/memory_tracker.sh $TEST_PID memory_stats.txt & +MONITOR_PID=$! + +# Wait for test to complete +wait $TEST_PID +TEST_EXIT_CODE=$? + +# Give monitor a moment to finish and kill it +sleep 1 +kill $MONITOR_PID 2> /dev/null + +# Display results +echo "Test completed with exit code: $TEST_EXIT_CODE" +cat memory_stats.txt diff --git a/tezt_analysis/v1/run_tezt_sequential.sh b/tezt_analysis/v1/run_tezt_sequential.sh new file mode 100755 index 0000000000000000000000000000000000000000..c545411dced66cec10f6f503f9a4636f632c36b8 --- /dev/null +++ b/tezt_analysis/v1/run_tezt_sequential.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# Install needed packages +sudo apk add procps grep bc + +# Run test and capture PID +_build/default/tezt/tests/main.exe -t 'Alpha: Sequencer produces an empty block in case of upgrade. (sequencer, latest, without dal, single chain)' & +MAIN_PID=$! + +# Initialize peak values +peak_mem=0 +peak_cpu=0 + +# Monitor resource usage until process completes +while kill -0 $MAIN_PID 2> /dev/null; do + # Get all PIDs (parent and children) + pids="$MAIN_PID $(pgrep -P $MAIN_PID | tr '\n' ' ')" + + # Calculate total memory usage + total_mem=0 + for pid in $pids; do + if [ -e /proc/$pid/status ]; then + mem=$(grep VmRSS /proc/$pid/status | awk '{print $2}' || echo "0") + total_mem=$((total_mem + mem)) + fi + done + + # Update peak memory if needed + if [ $total_mem -gt $peak_mem ]; then + peak_mem=$total_mem + echo "New peak memory: $peak_mem KB ($(echo "scale=2; $peak_mem/1024" | bc) MB)" + fi + + sleep 1 +done + +# Display summary +echo "=======================" +echo "FINAL RESULTS:" +echo "Peak memory usage: $peak_mem KB ($(echo "scale=2; $peak_mem/1024" | bc) MB)" +echo "=======================" + +# Wait for the main process to complete +wait $MAIN_PID diff --git a/tezt_analysis/v2/consolidate_tezt_reports.sh b/tezt_analysis/v2/consolidate_tezt_reports.sh new file mode 100755 index 0000000000000000000000000000000000000000..93df970d3dd8ff7e917e3c9dbe5e294051c54189 --- /dev/null +++ b/tezt_analysis/v2/consolidate_tezt_reports.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# Check if REPORT_OUTPUT is provided +if [ -z "$1" ]; then + echo "Error: REPORT_OUTPUT parameter is required" + echo "Usage: $0 " + exit 1 +fi + +REPORT_OUTPUT="$1" +REPORT_DIR=$(dirname "$REPORT_OUTPUT") +GLOBAL_REPORT="${REPORT_DIR}/memory_stats_merged.csv" + +echo "Merging all CSV files in ${REPORT_DIR} into ${GLOBAL_REPORT} ..." + +# Check if there are any CSV files in the directory +if [ ! "$(ls -A ${REPORT_DIR}/*.csv 2>/dev/null)" ]; then + echo "Error: No CSV files found in ${REPORT_DIR}" + exit 1 +fi + +# Clear the global report +> "$GLOBAL_REPORT" + +# Merge all CSV files tezt reports +for csv_file in "${REPORT_DIR}"/*.csv; do + # Skip the global report itself and sorted variant if it exists + if [ "$csv_file" == "$GLOBAL_REPORT" ]; then + continue + fi + + if [ "$csv_file" == "${REPORT_DIR}/memory_stats_sorted.csv" ]; then + continue + fi + + echo "Processing ${csv_file}..." + cat "$csv_file" >> "$GLOBAL_REPORT" +done + +# Count total tests in the merged report +TOTAL_TESTS=$(wc -l < "$GLOBAL_REPORT") + +echo "--------------------------------------------------" +echo "āœ… Merged report created successfully at ${GLOBAL_REPORT}" +echo "šŸ“Š Total tests analyzed: ${TOTAL_TESTS}" +echo "--------------------------------------------------" + +# Sort the merged report by memory usage (descending) +SORTED_REPORT="${REPORT_DIR}/memory_stats_sorted.csv" + +if [ -f "$GLOBAL_REPORT" ] && [ -s "$GLOBAL_REPORT" ]; then + echo "Creating sorted report (by memory usage)..." + + # Sort by memory usage (2nd field) in descending order + # Format is: test_title;memory_value + sort -t ';' -k 2 -n -r "$GLOBAL_REPORT" > "$SORTED_REPORT" + + echo "āœ… Sorted report created at ${SORTED_REPORT}" + +fi + +exit 0 \ No newline at end of file diff --git a/tezt_analysis/v2/input_tests/tests.txt b/tezt_analysis/v2/input_tests/tests.txt new file mode 100644 index 0000000000000000000000000000000000000000..53e249edd2f139acb58780fa615e665a0b268526 --- /dev/null +++ b/tezt_analysis/v2/input_tests/tests.txt @@ -0,0 +1,21 @@ +Alpha: Rollup-node kernel upgrade is applied to the sequencer state. (mainnet -> ghostnet) (sequencer, mainnet, without dal, single chain) +Alpha: Rollup-node kernel upgrade is applied to the sequencer state. (mainnet -> ghostnet) (sequencer, mainnet, without dal, multichain) +Alpha: Rollup-node kernel upgrade is applied to the sequencer state. (ghostnet -> latest) (sequencer, ghostnet, without dal, single chain) +Alpha: Rollup-node kernel upgrade is applied to the sequencer state. (ghostnet -> latest) (sequencer, ghostnet, without dal, multichain) +Alpha: Force kernel upgrade fail too early (latest -> ghostnet) (sequencer, latest, without dal, single chain) +Alpha: Force kernel upgrade fail too early (latest -> ghostnet) (sequencer, latest, without dal, multichain) +Alpha: Force kernel upgrade fail too early (latest -> ghostnet) (sequencer, latest, with dal, single chain) +Alpha: Force kernel upgrade fail too early (latest -> ghostnet) (sequencer, latest, with dal, multichain) +Alpha: Ensures EVM kernel's upgrade succeeds with potential migration(s). (mainnet -> ghostnet) +Alpha: Ensures EVM kernel's upgrade succeeds with potential migration(s) (mainnet -> latest). +Alpha: Ensures EVM kernel's upgrade succeeds with potential migration(s) (ghostnet -> latest). +Alpha: Agnostic baker --keep-alive +Alpha: After kernel upgrade a legacy deposit from delayed inbox can be decoded and processed. (mainnet -> latest) (sequencer, mainnet, without dal, single chain) +Alpha: After kernel upgrade a legacy deposit from delayed inbox can be decoded and processed. (mainnet -> latest) (sequencer, mainnet, without dal, multichain) +Alpha: After kernel upgrade a legacy deposit from delayed inbox can be decoded and processed. (ghostnet -> latest) (sequencer, ghostnet, without dal, single chain) +Alpha: After kernel upgrade a legacy deposit from delayed inbox can be decoded and processed. (ghostnet -> latest) (sequencer, ghostnet, without dal, multichain) +Alpha: --finalized-view returns the latest final block of the sequencer (sequencer, latest, without dal, single chain) +Alpha: --finalized-view returns the latest final block of the sequencer (sequencer, latest, without dal, multichain) +Alpha: --finalized-view forwards transactions to the sequencer without waiting (sequencer, latest, without dal, single chain) +Alpha: --finalized-view forwards transactions to the sequencer without waiting (sequencer, latest, without dal, multichain) +(R022 -> Alpha) constant migration diff --git a/tezt_analysis/v2/run_experimental.sh b/tezt_analysis/v2/run_experimental.sh new file mode 100755 index 0000000000000000000000000000000000000000..2d51287f6b0eedb4e4a93c1b55c06298dc4d9048 --- /dev/null +++ b/tezt_analysis/v2/run_experimental.sh @@ -0,0 +1,24 @@ +#!/bin/sh + + +if [ -z "${REPORT_FOLDER}" ]; then + echo "Error: REPORT_FOLDER environment variable is not set or is empty" + echo "Usage: REPORT_FOLDER=path/to/folder $0" + exit 1 +fi + +mkdir -p ${REPORT_FOLDER} + +test_title='Alpha: Sequencer produces an empty block in case of upgrade. (sequencer, latest, without dal, single chain)' + +# without log_file +echo "------------- WITHOUT --log-file ---------------" +_build/default/memstat/main.exe run -rv _build/default/tezt/tests/main.exe \\-t \\"${test_title}" + +# with log_file +echo "------------- WITH --log-file ---------------" +valid_title_as_filename=$(echo "$test_title" | tr -c 'a-zA-Z0-9_' '_' | tr -s '_') +log_file="${REPORT_FOLDER}/${valid_title_as_filename}.log" +_build/default/memstat/main.exe run -rv _build/default/tezt/tests/main.exe \\-t \\"${test_title}" \\--log-file "$log_file" +echo "-------------- cat $log_file -------------" +cat "$log_file" diff --git a/tezt_analysis/v2/run_tezt.sh b/tezt_analysis/v2/run_tezt.sh new file mode 100755 index 0000000000000000000000000000000000000000..a0fd25ae85d83c5e07ba3c6cc86951507c4cd87a --- /dev/null +++ b/tezt_analysis/v2/run_tezt.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +REPORT_OUTPUT="tezt_analysis/memory_stats.csv" + +# List tezt tests and filter for the ones that are not disabled +#tezt_output=$(_build/default/tezt/tests/main.exe --list-tsv -f basic.ml /ci_disabled) +tezt_output=$(_build/default/tezt/tests/main.exe --list-tsv /ci_disabled) + +# Process each test +echo "--------------------------------------------------" +echo " Starting tezt memory peak analysis:" +echo "--------------------------------------------------" + +# Display total number of tests found +total_tests=$(echo "${tezt_output}" | grep -v '^$' | wc -l) +echo "Found ${total_tests} tests to analyze" + +index=1 +echo "${tezt_output}" | while IFS=$'\t' read -r file title rest; do + if [ -n "${title}" ]; then # Skip empty lines + percentage=$((index * 100 / total_tests)) + echo -e "\nšŸ”„ Processing test #${index}/${total_tests} (${percentage}%): $title ..." + memory_peak=$(_build/default/memstat/main.exe run -r --bytes --only-peak _build/default/tezt/tests/main.exe \\-t \\${title}) + index=$((index + 1)) + echo "${title};${memory_peak}" >> ${REPORT_OUTPUT} + echo -e "Memory peak: ${memory_peak}\n" + fi +done + +echo "āœ… Tezt memory peak analysis completed." +echo "šŸ“‘ Below is the report of the memory peak for each test:" +cat ${REPORT_OUTPUT} +echo "--------------------------------------------------" +echo "Report saved to ${REPORT_OUTPUT}" +echo "--------------------------------------------------" diff --git a/tezt_analysis/v2/run_tezt_by_group_index.sh b/tezt_analysis/v2/run_tezt_by_group_index.sh new file mode 100755 index 0000000000000000000000000000000000000000..42019e4d5547838e68d536a25b3192176cfb6aeb --- /dev/null +++ b/tezt_analysis/v2/run_tezt_by_group_index.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +PARALLEL_INDEX=${1:-1} +PARALLEL_TOTAL=${2:-1} +REPORT_OUTPUT=$3 +INPUT_TESTS=$4 + +if ! [[ "$PARALLEL_INDEX" =~ ^[0-9]+$ ]] || ! [[ "$PARALLEL_TOTAL" =~ ^[0-9]+$ ]]; then + echo "Error: PARALLEL_INDEX and PARALLEL_TOTAL must be positive integers" + echo "Usage: $0 [PARALLEL_INDEX] [PARALLEL_TOTAL] [REPORT_OUTPUT] [INPUT_TESTS]" + exit 1 +fi + +if [ "$PARALLEL_INDEX" -lt 1 ] || [ "$PARALLEL_INDEX" -gt "$PARALLEL_TOTAL" ]; then + echo "Error: PARALLEL_INDEX must be between 1 and PARALLEL_TOTAL" + echo "Usage: $0 [PARALLEL_INDEX] [PARALLEL_TOTAL] [REPORT_OUTPUT] [INPUT_TESTS]" + exit 1 +fi + +if [ "$#" -ge 3 ] && [ -z "$REPORT_OUTPUT" ]; then + echo "Error: REPORT_OUTPUT parameter was specified but is empty" + echo "Usage: $0 [PARALLEL_INDEX] [PARALLEL_TOTAL] [REPORT_OUTPUT] [INPUT_TESTS]" + exit 1 +fi + +echo "Running in parallel mode: Processing group $PARALLEL_INDEX of $PARALLEL_TOTAL" + +echo "Report will be saved to: $REPORT_OUTPUT" + +# Create report directory if it doesn't exist +mkdir -p "$(dirname "$REPORT_OUTPUT")" + +# Create report directory if it doesn't exist +mkdir -p "$(dirname "$REPORT_OUTPUT")" + +# Check if INPUT_TESTS is provided and the file exists +full_mode="false" +tezt_output="" +if [ -n "$INPUT_TESTS" ]; then + if [ -f "$INPUT_TESTS" ]; then + echo "Using test list from file: $INPUT_TESTS" + tezt_output=$(cat "$INPUT_TESTS") + else + echo "Error: Input tests file '$INPUT_TESTS' does not exist" + exit 1 + fi +else + # List tezt tests and filter for the ones that are not disabled + echo "Getting test list from tezt command..." + full_mode="true" + # List tezt tests and filter for the ones that are not disabled + tezt_output=$(_build/default/tezt/tests/main.exe --list-tsv /ci_disabled) + #tezt_output=$(_build/default/tezt/tests/main.exe --list-tsv -f basic.ml /ci_disabled) +fi + +# Compute the total number of tests and the range for this group +total_tests=$(echo "${tezt_output}" | grep -v '^$' | wc -l) +tests_per_group=$((total_tests / PARALLEL_TOTAL + (total_tests % PARALLEL_TOTAL > 0))) +start_index=$(((PARALLEL_INDEX - 1) * tests_per_group + 1)) +end_index=$((PARALLEL_INDEX * tests_per_group)) + +# Ensure end_index doesn't exceed total_tests +if [ $end_index -gt $total_tests ]; then + end_index=$total_tests +fi + +my_test_count=$((end_index - start_index + 1)) + +echo "--------------------------------------------------" +echo " Starting tezt memory peak analysis:" +echo " Group: $PARALLEL_INDEX of $PARALLEL_TOTAL" +echo " Processing tests $start_index to $end_index of $total_tests" +echo " ($my_test_count tests in this group)" +echo "--------------------------------------------------" + +if [ "$full_mode" = "true" ]; then + echo "Running in full mode: All tests will be processed." + index=1 + group_index=1 + echo "${tezt_output}" | while IFS=$'\t' read -r file title rest; do + if [ -n "${title}" ]; then # Skip empty lines + # Process only tests that belong to our group + if [ $index -ge $start_index ] && [ $index -le $end_index ]; then + percentage=$((group_index * 100 / my_test_count)) + echo -e "\nšŸ”„ Processing test #${index}/${total_tests} (${percentage}% of this group): $title ..." + memory_peak=$(_build/default/memstat/main.exe run -r --bytes --only-peak _build/default/tezt/tests/main.exe \\-t \\"${title}") + echo "${title};${memory_peak}" >> ${REPORT_OUTPUT} + echo -e "Memory peak: ${memory_peak}\n" + group_index=$((group_index + 1)) + fi + index=$((index + 1)) + fi + done +else + echo "Running in partial mode: Only tests from the input file will be processed." + report_dir=$(dirname "$REPORT_OUTPUT") + index=1 + echo "${tezt_output}" | while IFS=$'\t' read -r title; do + valid_title_as_filename=$(echo "$title" | tr -c 'a-zA-Z0-9_' '_' | tr -s '_') + echo -e "\nšŸ”„ Processing test #${index}: $title ..." + memory_peak=$(_build/default/memstat/main.exe run -r --bytes --only-peak _build/default/tezt/tests/main.exe \\-t "${title}" \\--log-file "${report_dir}/${valid_title_as_filename}.log") + echo "${title};${memory_peak}" >> ${REPORT_OUTPUT} + echo -e "Memory peak: ${memory_peak}\n" + index=$((index + 1)) + done +fi + +echo "āœ… Tezt memory peak analysis completed for group $PARALLEL_INDEX of $PARALLEL_TOTAL." +echo "šŸ“‘ Below is the report of the memory peak for tests in this group:" +cat ${REPORT_OUTPUT} +echo "--------------------------------------------------" +echo "Report saved to ${REPORT_OUTPUT}" +echo "--------------------------------------------------"