diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d57ec464d533c1227261557e95f88d1172b89e94..640f674993ee5a888628a4fce4bb88ad033589d3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -263,7 +263,7 @@ oc.ocaml_fmt: key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: pull-push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -900,10 +900,10 @@ oc.build_amd64-extra-dev: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -996,10 +996,10 @@ oc.build_amd64-exp: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -2092,62 +2092,6 @@ oc.script:test-gen-genesis: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - scripts/snapshot_alpha.sh - - scripts/snapshot_alpha_and_link.sh - - scripts/user_activated_upgrade.sh - - src/proto_alpha/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - interruptible: true - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - ./scripts/ci/script:snapshot_alpha_and_link.sh - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - oc.script:b58_prefix: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -2390,11 +2334,11 @@ oc.unit:webassembly-x86_64: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.unit:non-proto-x86_64: +oc.unit:non-proto-arm64: image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - changes: - .gitlab-ci.yml @@ -2423,23 +2367,16 @@ oc.unit:non-proto-x86_64: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2449,14 +2386,13 @@ oc.unit:non-proto-x86_64: script: - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit + - make test-nonproto-unit test-webassembly - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + DISTRIBUTE_TESTS_TO_PARALLELS: "true" DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" @@ -2465,11 +2401,8 @@ oc.unit:non-proto-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 expire_in: 1 day paths: - test_results @@ -2477,44 +2410,22 @@ oc.unit:non-proto-x86_64: junit: test_results/*.xml when: always retry: 2 + parallel: 2 -oc.unit:non-proto-arm64: - image: ${ci_image_name}/test:${ci_image_tag} +etherlink.lint_wasm_runtime: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_arm64 + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - src/lib_wasm_runtime/**/*.rs when: on_success needs: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 - trigger dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2528,16 +2439,15 @@ oc.unit:non-proto-arm64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: + - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit test-webassembly + - etherlink/lib_wasm_runtime/lint.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - DISTRIBUTE_TESTS_TO_PARALLELS: "true" - DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2545,67 +2455,39 @@ oc.unit:non-proto-arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 - parallel: 2 -oc.unit:proto-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_kernel: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - contrib/mir/**/*.lalrpop + - contrib/mir/**/*.rs + - contrib/mir/**/*.toml + - contrib/mir/**/Cargo.lock + - etherlink.mk + - etherlink/**/*.rs + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh + - sdk/rust/**/* + - src/kernel_sdk/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2613,17 +2495,15 @@ oc.unit:proto-x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-proto-unit + - make -f etherlink.mk check + - make -f etherlink.mk test - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2631,69 +2511,33 @@ oc.unit:proto-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -oc.unit:other-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_firehose: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_high_cpu + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - etherlink/firehose/**/* + - etherlink/tezt/tests/evm_kernel_inputs/erc20tok.* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2701,17 +2545,14 @@ oc.unit:other-x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-other-unit + - make -C etherlink/firehose check - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2719,78 +2560,82 @@ oc.unit:other-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -ocaml-check: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_evm_compatibility: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: - - '**/*.ml' - - '**/*.mli' - - devtools/**/* - - src/**/* - - tezt/**/* + - etherlink.mk + - etherlink/kernel_latest/evm_evaluation/**/* + - etherlink/kernel_latest/revm/**/* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - key: dune_cache-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - - $CI_PROJECT_DIR/_dune_cache + - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - scripts/ci/dune.sh build @check --stop-on-first-error + - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs evm-evaluation-assessor + - git clone --depth 1 --branch v14.1@etherlink https://github.com/functori/tests + ethereum_tests + - ./evm-evaluation-assessor --eth-tests ./ethereum_tests/ --resources ./etherlink/kernel_latest/evm_evaluation/resources/ + -c - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - eval $(opam env) - - dune cache trim --size=5GB + - ./scripts/ci/sccache-stop.sh variables: + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" -etherlink.lint_wasm_runtime: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_revm_compatibility: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - gcp rules: - changes: - - src/lib_wasm_runtime/**/*.rs + - etherlink.mk + - etherlink/kernel_latest/revm/**/* + - etherlink/kernel_latest/revm_evaluation/**/* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2804,15 +2649,16 @@ etherlink.lint_wasm_runtime: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - etherlink/lib_wasm_runtime/lint.sh + - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs revm-evaluation-assessor + - git clone --depth 1 https://github.com/functori/evm-fixtures evm_fixtures + - ./revm-evaluation-assessor --test-cases ./evm_fixtures/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2821,16 +2667,17 @@ etherlink.lint_wasm_runtime: SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" -etherlink.unit_tests: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.mir_unit: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* + - contrib/mir/**/*.lalrpop + - contrib/mir/**/*.rs + - contrib/mir/**/*.toml + - contrib/mir/**/Cargo.lock when: on_success needs: - oc.docker:ci:amd64 @@ -2839,54 +2686,25 @@ etherlink.unit_tests: - oc.docker:ci:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune_cache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-etherlink-unit + - cargo test --manifest-path contrib/mir/Cargo.toml - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -etherlink.test_kernel: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +etherlink.mir_tzt: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp @@ -2896,21 +2714,13 @@ etherlink.test_kernel: - contrib/mir/**/*.rs - contrib/mir/**/*.toml - contrib/mir/**/Cargo.lock - - etherlink.mk - - etherlink/**/*.rs - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - - sdk/rust/**/* - - src/kernel_sdk/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 - trigger - - oc.build_kernels dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2922,45 +2732,68 @@ etherlink.test_kernel: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk check - - make -f etherlink.mk test + - cargo run --manifest-path contrib/mir/Cargo.toml --bin tzt_runner tzt_reference_test_suite/*.tzt - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" -etherlink.test_firehose: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test +documentation.rst-check: + image: ${ci_image_name_protected}/test:amd64--master + stage: sanity tags: - gcp rules: - changes: - - etherlink/firehose/**/* - - etherlink/tezt/tests/evm_kernel_inputs/erc20tok.* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh + - '**/*.rst' + when: on_success + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . $HOME/.venv/bin/activate + - make --silent -C docs sphinx-check + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: on_success needs: - - oc.docker:rust-toolchain:amd64 - - trigger - - oc.build_kernels + - oc.docker:ci:amd64 dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2974,14 +2807,13 @@ etherlink.test_firehose: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - make -C etherlink/firehose check + - eval $(opam env) + - make -C docs -j docexes-gen - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2989,28 +2821,32 @@ etherlink.test_firehose: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst -etherlink.test_evm_compatibility: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +client-libs.kaitai_checks: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink.mk - - etherlink/kernel_latest/evm_evaluation/**/* - - etherlink/kernel_latest/revm/**/* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - when: on_success + - client-libs/*kaitai*/**/* + - images/client-libs-dependencies/**/* + - src/**/* + when: manual + allow_failure: true needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 - trigger - - oc.build_kernels dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -3024,18 +2860,16 @@ etherlink.test_evm_compatibility: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs evm-evaluation-assessor - - git clone --depth 1 --branch v14.1@etherlink https://github.com/functori/tests - ethereum_tests - - ./evm-evaluation-assessor --eth-tests ./ethereum_tests/ --resources ./etherlink/kernel_latest/evm_evaluation/resources/ - -c + - . ./scripts/version.sh + - eval $(opam env) + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -3043,113 +2877,94 @@ etherlink.test_evm_compatibility: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + expire_in: 1 hour + paths: + - _build/default/client-libs/bin_codec_kaitai/codec.exe + when: on_success -etherlink.test_revm_compatibility: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +client-libs.kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink.mk - - etherlink/kernel_latest/revm/**/* - - etherlink/kernel_latest/revm_evaluation/**/* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - when: on_success + - client-libs/*kaitai*/**/* + - images/client-libs-dependencies/**/* + - src/**/* + when: manual + allow_failure: true needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:client-libs-dependencies - trigger - - oc.build_kernels + - client-libs.kaitai_checks dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:client-libs-dependencies + - client-libs.kaitai_checks timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs revm-evaluation-assessor - - git clone --depth 1 https://github.com/functori/evm-fixtures evm_fixtures - - ./revm-evaluation-assessor --test-cases ./evm_fixtures/ + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + 2>/dev/null - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" -etherlink.mir_unit: - image: ${ci_image_name}/test:${ci_image_tag} +grafazos.build: + image: ${jsonnet_image_name}:${jsonnet_image_tag} stage: test tags: - gcp rules: - changes: - - contrib/mir/**/*.lalrpop - - contrib/mir/**/*.rs - - contrib/mir/**/*.toml - - contrib/mir/**/Cargo.lock + - grafazos/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:jsonnet:amd64 - trigger dependencies: - - oc.docker:ci:amd64 + - oc.docker:jsonnet:amd64 timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - cargo test --manifest-path contrib/mir/Cargo.toml + - cd grafazos/ + - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec + - make - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - CARGO_NET_OFFLINE: "false" + artifacts: + name: grafazos-dashboards + expire_in: 1 day + paths: + - grafazos/output/**/*.json + when: on_success -etherlink.mir_tzt: - image: ${ci_image_name}/test:${ci_image_tag} +teztale.build-arm64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - changes: - - contrib/mir/**/*.lalrpop - - contrib/mir/**/*.rs - - contrib/mir/**/*.toml - - contrib/mir/**/Cargo.lock - - tzt_reference_test_suite/**/* + - teztale/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 - trigger dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -3161,60 +2976,54 @@ etherlink.mir_tzt: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - cargo run --manifest-path contrib/mir/Cargo.toml --bin tzt_runner tzt_reference_test_suite/*.tzt + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - make teztale + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + name: teztale-binaries + expire_in: 1 day + paths: + - teztale-binaries/arm64/octez-teztale-* + when: on_success -etherlink.build_tezt: +teztale.build-x86_64: image: ${ci_image_name}/build:${ci_image_tag} - stage: build + stage: test tags: - - gcp + - gcp_very_high_cpu_ramfs rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - teztale/**/* when: on_success needs: - oc.docker:ci:amd64 + - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -3225,14 +3034,15 @@ etherlink.build_tezt: - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - dune build etherlink/tezt/tests/main.exe + - make teztale + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + PROFILE: static CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -3240,27 +3050,51 @@ etherlink.build_tezt: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: etherlink_tezt_exe + name: teztale-binaries expire_in: 1 day paths: - - _build/default/etherlink/tezt/tests/main.exe + - teztale-binaries/x86_64/octez-teztale-* when: on_success -documentation.rst-check: - image: ${ci_image_name_protected}/test:amd64--master - stage: sanity +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: on_success - needs: [] - dependencies: [] + needs: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-exp + - oc.build_kernels + dependencies: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-exp + - oc.build_kernels timeout: 60 minutes interruptible: true before_script: @@ -3268,13 +3102,21 @@ documentation.rst-check: - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - . $HOME/.venv/bin/activate - - make --silent -C docs sphinx-check + - eval $(opam env) + - make -C docs -j octez-gen - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json -documentation.docgen: +documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build tags: @@ -3318,7 +3160,7 @@ documentation.docgen: - . ./scripts/ci/sccache-start.sh script: - eval $(opam env) - - make -C docs -j docexes-gen + - make -C docs odoc-lite - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' @@ -3332,86 +3174,109 @@ documentation.docgen: SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 1 week + expire_in: 4 hours paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - docs/_build/api/odoc/ + - docs/odoc.log + when: always -client-libs.kaitai_checks: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - - client-libs/*kaitai*/**/* - - images/client-libs-dependencies/**/* - - src/**/* - when: manual - allow_failure: true + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success needs: - oc.docker:ci:amd64 - trigger + - documentation.odoc + - documentation.manuals + - documentation.docgen dependencies: - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - eval $(opam env) - - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings - and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` - and commit the resulting diff.' ; false) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 1 hour + expire_in: 1 week paths: - - _build/default/client-libs/bin_codec_kaitai/codec.exe - when: on_success + - docs/_build/ + expose_as: Documentation - excluding old protocols -client-libs.kaitai_e2e_checks: - image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +documentation.linkcheck: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: manual + allow_failure: true - changes: - - client-libs/*kaitai*/**/* - - images/client-libs-dependencies/**/* - - src/**/* + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: manual allow_failure: true needs: - - oc.docker:client-libs-dependencies + - oc.docker:ci:amd64 - trigger - - client-libs.kaitai_checks + - documentation.manuals + - documentation.docgen + - documentation.build_all dependencies: - - oc.docker:client-libs-dependencies - - client-libs.kaitai_checks + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + allow_failure: true timeout: 60 minutes interruptible: true before_script: @@ -3420,27 +3285,30 @@ client-libs.kaitai_e2e_checks: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - . ./scripts/version.sh - - . ./scripts/install_build_deps.js.sh - - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files - 2>/dev/null + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs redirectcheck + - make -C docs linkcheck - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -grafazos.build: - image: ${jsonnet_image_name}:${jsonnet_image_tag} +documentation.install_python_bookworm: + image: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/debian:bookworm stage: test tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - - grafazos/**/* + - docs/developer/install-python-debian-ubuntu.sh + - poetry.lock + - pyproject.toml when: on_success needs: - - oc.docker:jsonnet:amd64 - trigger - dependencies: - - oc.docker:jsonnet:amd64 + dependencies: [] timeout: 60 minutes interruptible: true before_script: @@ -3448,39 +3316,58 @@ grafazos.build: - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - cd grafazos/ - - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec - - make + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - name: grafazos-dashboards - expire_in: 1 day - paths: - - grafazos/output/**/*.json - when: on_success -teztale.build-arm64: +etherlink.build_tezt: image: ${ci_image_name}/build:${ci_image_tag} - stage: test + stage: build tags: - - gcp_arm64 + - gcp rules: - changes: - - teztale/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:arm64 - - trigger + - oc.docker:ci:amd64 dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -3491,15 +3378,14 @@ teztale.build-arm64: - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - make teztale - - mkdir -p ./teztale-binaries/arm64 - - mv octez-teztale-* ./teztale-binaries/arm64/ + - dune build etherlink/tezt/tests/main.exe - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - PROFILE: static CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -3507,531 +3393,625 @@ teztale.build-arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: teztale-binaries + name: etherlink_tezt_exe expire_in: 1 day paths: - - teztale-binaries/arm64/octez-teztale-* + - _build/default/etherlink/tezt/tests/main.exe when: on_success -teztale.build-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.tezt-flaky: + image: ${ci_image_name}/e2etest:${ci_image_tag} stage: test tags: - - gcp_very_high_cpu_ramfs + - gcp_tezt rules: - changes: - - teztale/**/* - when: on_success + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + allow_failure: true + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - - eval $(opam env) - - make teztale - - mkdir -p ./teztale-binaries/x86_64 - - mv octez-teztale-* ./teztale-binaries/x86_64/ + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/flaky --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/flaky + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - PROFILE: static - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + TEZT_NO_NPX: "true" artifacts: - name: teztale-binaries - expire_in: 1 day + expire_in: 7 days paths: - - teztale-binaries/x86_64/octez-teztale-* - when: on_success + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 2 -documentation.manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt-extra: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 + - trigger - oc.build_x86_64-released - - oc.build_amd64-extra-dev - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt - oc.build_kernels dependencies: - oc.docker:ci:amd64 - oc.build_x86_64-released - - oc.build_amd64-extra-dev - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt - oc.build_kernels - timeout: 60 minutes + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - eval $(opam env) - - make -C docs -j octez-gen + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/extra --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/extra + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -extra + TESTS: extra && /ci_disabled && /time_sensitive && /slow && /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "6" + TEZT_NO_NPX: "true" artifacts: - expire_in: 1 week + expire_in: 7 days paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 12 -documentation.odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt-slow: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 + - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - eval $(opam env) - - make -C docs odoc-lite + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --junit-mem-peak + 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results.json --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} + --record-mem-peak --mem-warn 5_000_000_000 --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' + --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' --junit-tag 'dd_tags[tezt-tag.slow]=slow' + --junit-tag 'dd_tags[tezt-tag.extra]=extra' --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' + --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' + --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /time_sensitive && /extra && /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + TEZT_NO_NPX: "true" artifacts: - expire_in: 4 hours + expire_in: 7 days paths: - - docs/_build/api/odoc/ - - docs/odoc.log + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT when: always + retry: 2 + parallel: 3 -documentation.build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - oc.docker:ci:amd64 - trigger - - documentation.odoc - - documentation.manuals - - documentation.docgen + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - documentation.odoc - - documentation.manuals - - documentation.docgen - timeout: 60 minutes + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - eval $(opam env) - - . $HOME/.venv/bin/activate - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "6" + TEZT_NO_NPX: "true" artifacts: - expire_in: 1 week + expire_in: 7 days paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 18 -documentation.linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} +etherlink.unit_tests: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - gcp rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: manual - allow_failure: true - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: manual - allow_failure: true + - etherlink/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* + when: on_success needs: - oc.docker:ci:amd64 - trigger - - documentation.manuals - - documentation.docgen - - documentation.build_all dependencies: - oc.docker:ci:amd64 - - documentation.manuals - - documentation.docgen - - documentation.build_all - allow_failure: true timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - . ./scripts/version.sh - eval $(opam env) - - . $HOME/.venv/bin/activate - - make -C docs redirectcheck - - make -C docs linkcheck + - make test-etherlink-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + DUNE_ARGS: -j 12 + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + artifacts: + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day + paths: + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 -documentation.install_python_bookworm: - image: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/debian:bookworm +etherlink.build_evm_node_static_arm64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - docs/developer/install-python-debian-ubuntu.sh - - poetry.lock - - pyproject.toml - when: on_success + - etherlink/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* + when: manual + allow_failure: true needs: + - oc.docker:ci:arm64 - trigger - dependencies: [] + dependencies: + - oc.docker:ci:arm64 timeout: 60 minutes + cache: + key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - ./scripts/ci/build_static_binaries.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + ARCH: arm64 + VERSION_EXECUTABLE: octez-evm-node + EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + name: octez-binaries + paths: + - octez-binaries/$ARCH/* + when: on_success -etherlink.tezt-flaky: - image: ${ci_image_name}/e2etest:${ci_image_tag} +etherlink.build_evm_node_static_x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* when: manual allow_failure: true needs: - oc.docker:ci:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - allow_failure: true - timeout: 40 minutes + timeout: 60 minutes + cache: + key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/flaky --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/flaky - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - ./scripts/ci/build_static_binaries.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -flaky - TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && flaky - TEZT_RETRY: "3" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" + ARCH: x86_64 + VERSION_EXECUTABLE: octez-evm-node + EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 7 days + name: octez-binaries paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 2 + - octez-binaries/$ARCH/* + when: on_success -etherlink.tezt-extra: - image: ${ci_image_name}/e2etest:${ci_image_tag} +ocaml-check: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* + - '**/*.ml' + - '**/*.mli' + - devtools/**/* - src/**/* - tezt/**/* - - tzt_reference_test_suite/**/* - when: manual - allow_failure: true + when: on_success needs: - oc.docker:ci:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes + timeout: 60 minutes + cache: + key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: + - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/extra --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/extra - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - eval $(opam env) + - scripts/ci/dune.sh build @check --stop-on-first-error - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - eval $(opam env) + - dune cache trim --size=5GB variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -extra - TESTS: extra && /ci_disabled && /time_sensitive && /slow && /cloud && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "6" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 12 + CARGO_NET_OFFLINE: "false" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache -etherlink.tezt-slow: - image: ${ci_image_name}/e2etest:${ci_image_tag} +oc.unit:other-x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_high_cpu rules: - changes: - .gitlab-ci.yml @@ -4058,76 +4038,68 @@ etherlink.tezt-slow: - src/**/* - tezt/**/* - tzt_reference_test_suite/**/* - when: manual - allow_failure: true + when: on_success needs: - oc.docker:ci:amd64 - trigger - oc.build_x86_64-released - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --junit-mem-peak - 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results.json --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - --record-mem-peak --mem-warn 5_000_000_000 --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' - --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' --junit-tag 'dd_tags[tezt-tag.slow]=slow' - --junit-tag 'dd_tags[tezt-tag.extra]=extra' --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' - --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' - --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - eval $(opam env) + - make test-other-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -slow - TESTS: slow && /ci_disabled && /time_sensitive && /extra && /cloud - TEZT_RETRY: "1" - TEZT_PARALLEL: "3" - TEZT_NO_NPX: "true" + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + DUNE_ARGS: -j 12 + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - expire_in: 7 days + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT + - test_results reports: - junit: $JUNIT + junit: test_results/*.xml when: always retry: 2 - parallel: 3 -etherlink.tezt: - image: ${ci_image_name}/e2etest:${ci_image_tag} +oc.unit:proto-x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml @@ -4160,88 +4132,19 @@ etherlink.tezt: - trigger - oc.build_x86_64-released - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes - interruptible: true - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: "" - TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "6" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 18 - -etherlink.build_evm_node_static_arm64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - rules: - - changes: - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true - needs: - - oc.docker:ci:arm64 - - trigger - dependencies: - - oc.docker:ci:arm64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -4249,17 +4152,17 @@ etherlink.build_evm_node_static_arm64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - ./scripts/ci/build_static_binaries.sh + - make test-proto-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - ARCH: arm64 - VERSION_EXECUTABLE: octez-evm-node - EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -4267,35 +4170,69 @@ etherlink.build_evm_node_static_arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: octez-binaries + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - octez-binaries/$ARCH/* - when: on_success + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 -etherlink.build_evm_node_static_x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +oc.unit:non-proto-x86_64: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: on_success needs: - oc.docker:ci:amd64 - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -4303,17 +4240,17 @@ etherlink.build_evm_node_static_x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - ./scripts/ci/build_static_binaries.sh + - make test-nonproto-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - ARCH: x86_64 - VERSION_EXECUTABLE: octez-evm-node - EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -4321,11 +4258,18 @@ etherlink.build_evm_node_static_x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: octez-binaries + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - octez-binaries/$ARCH/* - when: on_success + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 de.unit:arm64: image: ${ci_image_name}/test:${ci_image_tag} @@ -4435,6 +4379,62 @@ resto.unit:x86_64: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +oc.script:snapshot_alpha_and_link: + image: ${ci_image_name}/build:${ci_image_tag} + stage: test + tags: + - gcp_very_high_cpu + rules: + - changes: + - scripts/snapshot_alpha.sh + - scripts/snapshot_alpha_and_link.sh + - scripts/user_activated_upgrade.sh + - src/proto_alpha/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull + interruptible: true + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - ./scripts/ci/script:snapshot_alpha_and_link.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + tezt-static-binaries: image: ${ci_image_name}/e2etest:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/etherlink.daily.yml b/.gitlab/ci/pipelines/etherlink.daily.yml index 6ca68dcd3996fc6082e56a83e4833b494785e0d9..ef1f315d6dcf7a6134d642c3dc90980f1817829e 100644 --- a/.gitlab/ci/pipelines/etherlink.daily.yml +++ b/.gitlab/ci/pipelines/etherlink.daily.yml @@ -178,10 +178,10 @@ oc.build_amd64-exp: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull-push before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -313,63 +313,6 @@ etherlink.lint_wasm_runtime: SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" -etherlink.unit_tests: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: dune_cache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-etherlink-unit - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - DUNE_ARGS: -j 12 - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 - etherlink.test_kernel: image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test @@ -589,25 +532,21 @@ etherlink.mir_tzt: variables: CARGO_NET_OFFLINE: "false" -etherlink.build_tezt: +etherlink.build_evm_node_static_arm64: image: ${ci_image_name}/build:${ci_image_tag} - stage: build + stage: test tags: - - gcp + - gcp_arm64 needs: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -616,16 +555,16 @@ etherlink.build_tezt: - . ./scripts/ci/sccache-start.sh script: - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - eval $(opam env) - - dune build etherlink/tezt/tests/main.exe + - ./scripts/ci/build_static_binaries.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + ARCH: arm64 + VERSION_EXECUTABLE: octez-evm-node + EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -633,25 +572,21 @@ etherlink.build_tezt: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: etherlink_tezt_exe - expire_in: 1 day + name: octez-binaries paths: - - _build/default/etherlink/tezt/tests/main.exe + - octez-binaries/$ARCH/* when: on_success -etherlink.build_evm_node_static_arm64: +etherlink.build_evm_node_static_x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_arm64 + - gcp_very_high_cpu needs: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -673,7 +608,7 @@ etherlink.build_evm_node_static_arm64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - ARCH: arm64 + ARCH: x86_64 VERSION_EXECUTABLE: octez-evm-node EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables CARGO_NET_OFFLINE: "false" @@ -689,21 +624,25 @@ etherlink.build_evm_node_static_arm64: - octez-binaries/$ARCH/* when: on_success -etherlink.build_evm_node_static_x86_64: +etherlink.build_tezt: image: ${ci_image_name}/build:${ci_image_tag} - stage: test + stage: build tags: - - gcp_very_high_cpu + - gcp needs: - oc.docker:ci:amd64 dependencies: - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -712,16 +651,16 @@ etherlink.build_evm_node_static_x86_64: - . ./scripts/ci/sccache-start.sh script: - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - ./scripts/ci/build_static_binaries.sh + - dune build etherlink/tezt/tests/main.exe - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - ARCH: x86_64 - VERSION_EXECUTABLE: octez-evm-node - EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -729,10 +668,14 @@ etherlink.build_evm_node_static_x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: octez-binaries + name: etherlink_tezt_exe + expire_in: 1 day paths: - - octez-binaries/$ARCH/* + - _build/default/etherlink/tezt/tests/main.exe when: on_success etherlink.tezt-flaky: @@ -999,3 +942,60 @@ etherlink.tezt: when: always retry: 2 parallel: 18 + +etherlink.unit_tests: + image: ${ci_image_name}/build:${ci_image_tag} + stage: test + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull-push + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - . ./scripts/version.sh + - eval $(opam env) + - make test-etherlink-unit + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + DUNE_ARGS: -j 12 + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + artifacts: + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day + paths: + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 5b3836a83ec866a6277d029aed3fd4f5f75ede8f..02b4684ad151cf40e3091dfe6197c6f52c8a5c97 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -263,7 +263,7 @@ oc.ocaml_fmt: key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: pull-push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -899,10 +899,10 @@ oc.build_amd64-extra-dev: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -995,10 +995,10 @@ oc.build_amd64-exp: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh @@ -2026,62 +2026,6 @@ oc.script:test-gen-genesis: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - rules: - - changes: - - scripts/snapshot_alpha.sh - - scripts/snapshot_alpha_and_link.sh - - scripts/user_activated_upgrade.sh - - src/proto_alpha/**/* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - interruptible: true - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - ./scripts/ci/script:snapshot_alpha_and_link.sh - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - oc.script:b58_prefix: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -2324,11 +2268,11 @@ oc.unit:webassembly-x86_64: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.unit:non-proto-x86_64: +oc.unit:non-proto-arm64: image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - changes: - .gitlab-ci.yml @@ -2357,23 +2301,16 @@ oc.unit:non-proto-x86_64: - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2383,14 +2320,13 @@ oc.unit:non-proto-x86_64: script: - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit + - make test-nonproto-unit test-webassembly - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + DISTRIBUTE_TESTS_TO_PARALLELS: "true" DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" @@ -2399,11 +2335,8 @@ oc.unit:non-proto-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 expire_in: 1 day paths: - test_results @@ -2411,44 +2344,22 @@ oc.unit:non-proto-x86_64: junit: test_results/*.xml when: always retry: 2 + parallel: 2 -oc.unit:non-proto-arm64: - image: ${ci_image_name}/test:${ci_image_tag} +etherlink.lint_wasm_runtime: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_arm64 + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - src/lib_wasm_runtime/**/*.rs when: on_success needs: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 - trigger dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2462,16 +2373,15 @@ oc.unit:non-proto-arm64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: + - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit test-webassembly + - etherlink/lib_wasm_runtime/lint.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - DISTRIBUTE_TESTS_TO_PARALLELS: "true" - DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2479,67 +2389,39 @@ oc.unit:non-proto-arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 - parallel: 2 -oc.unit:proto-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_kernel: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - contrib/mir/**/*.lalrpop + - contrib/mir/**/*.rs + - contrib/mir/**/*.toml + - contrib/mir/**/Cargo.lock + - etherlink.mk + - etherlink/**/*.rs + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh + - sdk/rust/**/* + - src/kernel_sdk/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2547,17 +2429,15 @@ oc.unit:proto-x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-proto-unit + - make -f etherlink.mk check + - make -f etherlink.mk test - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2565,69 +2445,33 @@ oc.unit:proto-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -oc.unit:other-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_firehose: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_high_cpu + - gcp rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - etherlink/firehose/**/* + - etherlink/tezt/tests/evm_kernel_inputs/erc20tok.* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -2635,17 +2479,14 @@ oc.unit:other-x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-other-unit + - make -C etherlink/firehose check - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2653,78 +2494,82 @@ oc.unit:other-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -ocaml-check: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_evm_compatibility: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: - - '**/*.ml' - - '**/*.mli' - - devtools/**/* - - src/**/* - - tezt/**/* + - etherlink.mk + - etherlink/kernel_latest/evm_evaluation/**/* + - etherlink/kernel_latest/revm/**/* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: - key: dune_cache-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - - $CI_PROJECT_DIR/_dune_cache + - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - scripts/ci/dune.sh build @check --stop-on-first-error + - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs evm-evaluation-assessor + - git clone --depth 1 --branch v14.1@etherlink https://github.com/functori/tests + ethereum_tests + - ./evm-evaluation-assessor --eth-tests ./ethereum_tests/ --resources ./etherlink/kernel_latest/evm_evaluation/resources/ + -c - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - eval $(opam env) - - dune cache trim --size=5GB + - ./scripts/ci/sccache-stop.sh variables: + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" -etherlink.lint_wasm_runtime: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.test_revm_compatibility: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: test tags: - gcp rules: - changes: - - src/lib_wasm_runtime/**/*.rs + - etherlink.mk + - etherlink/kernel_latest/revm/**/* + - etherlink/kernel_latest/revm_evaluation/**/* + - images/create_image.sh + - images/rust-toolchain/**/* + - images/scripts/install_datadog_static.sh + - scripts/version.sh when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 - trigger + - oc.build_kernels dependencies: - - oc.docker:ci:amd64 + - oc.docker:rust-toolchain:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2738,15 +2583,16 @@ etherlink.lint_wasm_runtime: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - etherlink/lib_wasm_runtime/lint.sh + - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs revm-evaluation-assessor + - git clone --depth 1 https://github.com/functori/evm-fixtures evm_fixtures + - ./revm-evaluation-assessor --test-cases ./evm_fixtures/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: + CC: clang + NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2755,16 +2601,17 @@ etherlink.lint_wasm_runtime: SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" -etherlink.unit_tests: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.mir_unit: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* + - contrib/mir/**/*.lalrpop + - contrib/mir/**/*.rs + - contrib/mir/**/*.toml + - contrib/mir/**/Cargo.lock when: on_success needs: - oc.docker:ci:amd64 @@ -2773,54 +2620,25 @@ etherlink.unit_tests: - oc.docker:ci:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune_cache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - eval $(opam env) - - make test-etherlink-unit + - cargo test --manifest-path contrib/mir/Cargo.toml - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: - DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 - expire_in: 1 day - paths: - - test_results - reports: - junit: test_results/*.xml - when: always - retry: 2 -etherlink.test_kernel: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +etherlink.mir_tzt: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp @@ -2830,21 +2648,13 @@ etherlink.test_kernel: - contrib/mir/**/*.rs - contrib/mir/**/*.toml - contrib/mir/**/Cargo.lock - - etherlink.mk - - etherlink/**/*.rs - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - - sdk/rust/**/* - - src/kernel_sdk/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 - trigger - - oc.build_kernels dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2856,45 +2666,68 @@ etherlink.test_kernel: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk check - - make -f etherlink.mk test + - cargo run --manifest-path contrib/mir/Cargo.toml --bin tzt_runner tzt_reference_test_suite/*.tzt - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" -etherlink.test_firehose: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: test +documentation.rst-check: + image: ${ci_image_name_protected}/test:amd64--master + stage: sanity tags: - gcp rules: - changes: - - etherlink/firehose/**/* - - etherlink/tezt/tests/evm_kernel_inputs/erc20tok.* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh + - '**/*.rst' + when: on_success + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . $HOME/.venv/bin/activate + - make --silent -C docs sphinx-check + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: on_success needs: - - oc.docker:rust-toolchain:amd64 - - trigger - - oc.build_kernels + - oc.docker:ci:amd64 dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2908,14 +2741,13 @@ etherlink.test_firehose: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - make -C etherlink/firehose check + - eval $(opam env) + - make -C docs -j docexes-gen - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2923,28 +2755,32 @@ etherlink.test_firehose: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst -etherlink.test_evm_compatibility: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +client-libs.kaitai_checks: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink.mk - - etherlink/kernel_latest/evm_evaluation/**/* - - etherlink/kernel_latest/revm/**/* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - when: on_success + - client-libs/*kaitai*/**/* + - images/client-libs-dependencies/**/* + - src/**/* + when: manual + allow_failure: true needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 - trigger - - oc.build_kernels dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -2958,18 +2794,16 @@ etherlink.test_evm_compatibility: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs evm-evaluation-assessor - - git clone --depth 1 --branch v14.1@etherlink https://github.com/functori/tests - ethereum_tests - - ./evm-evaluation-assessor --eth-tests ./ethereum_tests/ --resources ./etherlink/kernel_latest/evm_evaluation/resources/ - -c + - . ./scripts/version.sh + - eval $(opam env) + - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings + and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` + and commit the resulting diff.' ; false) - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -2977,113 +2811,94 @@ etherlink.test_evm_compatibility: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + expire_in: 1 hour + paths: + - _build/default/client-libs/bin_codec_kaitai/codec.exe + when: on_success -etherlink.test_revm_compatibility: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +client-libs.kaitai_e2e_checks: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} stage: test tags: - gcp rules: - changes: - - etherlink.mk - - etherlink/kernel_latest/revm/**/* - - etherlink/kernel_latest/revm_evaluation/**/* - - images/create_image.sh - - images/rust-toolchain/**/* - - images/scripts/install_datadog_static.sh - - scripts/version.sh - when: on_success + - client-libs/*kaitai*/**/* + - images/client-libs-dependencies/**/* + - src/**/* + when: manual + allow_failure: true needs: - - oc.docker:rust-toolchain:amd64 + - oc.docker:client-libs-dependencies - trigger - - oc.build_kernels + - client-libs.kaitai_checks dependencies: - - oc.docker:rust-toolchain:amd64 + - oc.docker:client-libs-dependencies + - client-libs.kaitai_checks timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - make -f etherlink.mk EVM_EVALUATION_FEATURES=disable-file-logs revm-evaluation-assessor - - git clone --depth 1 https://github.com/functori/evm-fixtures evm_fixtures - - ./revm-evaluation-assessor --test-cases ./evm_fixtures/ + - . ./scripts/version.sh + - . ./scripts/install_build_deps.js.sh + - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files + 2>/dev/null - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CC: clang - NATIVE_TARGET: x86_64-unknown-linux-musl - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" -etherlink.mir_unit: - image: ${ci_image_name}/test:${ci_image_tag} +grafazos.build: + image: ${jsonnet_image_name}:${jsonnet_image_tag} stage: test tags: - gcp rules: - changes: - - contrib/mir/**/*.lalrpop - - contrib/mir/**/*.rs - - contrib/mir/**/*.toml - - contrib/mir/**/Cargo.lock + - grafazos/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:jsonnet:amd64 - trigger dependencies: - - oc.docker:ci:amd64 + - oc.docker:jsonnet:amd64 timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - cargo test --manifest-path contrib/mir/Cargo.toml + - cd grafazos/ + - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec + - make - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - CARGO_NET_OFFLINE: "false" + artifacts: + name: grafazos-dashboards + expire_in: 1 day + paths: + - grafazos/output/**/*.json + when: on_success -etherlink.mir_tzt: - image: ${ci_image_name}/test:${ci_image_tag} +teztale.build-arm64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - changes: - - contrib/mir/**/*.lalrpop - - contrib/mir/**/*.rs - - contrib/mir/**/*.toml - - contrib/mir/**/Cargo.lock - - tzt_reference_test_suite/**/* + - teztale/**/* when: on_success needs: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 - trigger dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: key: cargo-$CI_JOB_NAME_SLUG @@ -3095,60 +2910,54 @@ etherlink.mir_tzt: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - cargo run --manifest-path contrib/mir/Cargo.toml --bin tzt_runner tzt_reference_test_suite/*.tzt + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - make teztale + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + name: teztale-binaries + expire_in: 1 day + paths: + - teztale-binaries/arm64/octez-teztale-* + when: on_success -etherlink.build_tezt: +teztale.build-x86_64: image: ${ci_image_name}/build:${ci_image_tag} - stage: build + stage: test tags: - - gcp + - gcp_very_high_cpu_ramfs rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - teztale/**/* when: on_success needs: - oc.docker:ci:amd64 + - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -3159,14 +2968,15 @@ etherlink.build_tezt: - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - dune build etherlink/tezt/tests/main.exe + - make teztale + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + PROFILE: static CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -3174,27 +2984,51 @@ etherlink.build_tezt: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: etherlink_tezt_exe + name: teztale-binaries expire_in: 1 day paths: - - _build/default/etherlink/tezt/tests/main.exe + - teztale-binaries/x86_64/octez-teztale-* when: on_success -documentation.rst-check: - image: ${ci_image_name_protected}/test:amd64--master - stage: sanity +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: on_success - needs: [] - dependencies: [] + needs: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-exp + - oc.build_kernels + dependencies: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-exp + - oc.build_kernels timeout: 60 minutes interruptible: true before_script: @@ -3202,13 +3036,21 @@ documentation.rst-check: - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - . $HOME/.venv/bin/activate - - make --silent -C docs sphinx-check + - eval $(opam env) + - make -C docs -j octez-gen - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json -documentation.docgen: +documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build tags: @@ -3252,7 +3094,7 @@ documentation.docgen: - . ./scripts/ci/sccache-start.sh script: - eval $(opam env) - - make -C docs -j docexes-gen + - make -C docs odoc-lite - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' @@ -3266,86 +3108,109 @@ documentation.docgen: SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 1 week + expire_in: 4 hours paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - docs/_build/api/odoc/ + - docs/odoc.log + when: always -client-libs.kaitai_checks: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - - client-libs/*kaitai*/**/* - - images/client-libs-dependencies/**/* - - src/**/* - when: manual - allow_failure: true + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success needs: - oc.docker:ci:amd64 - trigger + - documentation.odoc + - documentation.manuals + - documentation.docgen dependencies: - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - eval $(opam env) - - make -C ${CI_PROJECT_DIR} check-kaitai-struct-files || (echo 'Octez encodings - and Kaitai files seem to be out of sync. You might need to run `make check-kaitai-struct-files` - and commit the resulting diff.' ; false) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 1 hour + expire_in: 1 week paths: - - _build/default/client-libs/bin_codec_kaitai/codec.exe - when: on_success + - docs/_build/ + expose_as: Documentation - excluding old protocols -client-libs.kaitai_e2e_checks: - image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +documentation.linkcheck: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: manual + allow_failure: true - changes: - - client-libs/*kaitai*/**/* - - images/client-libs-dependencies/**/* - - src/**/* + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* when: manual allow_failure: true needs: - - oc.docker:client-libs-dependencies + - oc.docker:ci:amd64 - trigger - - client-libs.kaitai_checks + - documentation.manuals + - documentation.docgen + - documentation.build_all dependencies: - - oc.docker:client-libs-dependencies - - client-libs.kaitai_checks + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + allow_failure: true timeout: 60 minutes interruptible: true before_script: @@ -3354,27 +3219,30 @@ client-libs.kaitai_e2e_checks: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - . ./scripts/version.sh - - . ./scripts/install_build_deps.js.sh - - ./client-libs/kaitai-struct-files/scripts/kaitai_e2e.sh client-libs/kaitai-struct-files/files - 2>/dev/null + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs redirectcheck + - make -C docs linkcheck - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -grafazos.build: - image: ${jsonnet_image_name}:${jsonnet_image_tag} +documentation.install_python_bookworm: + image: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/debian:bookworm stage: test tags: - gcp rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success - changes: - - grafazos/**/* + - docs/developer/install-python-debian-ubuntu.sh + - poetry.lock + - pyproject.toml when: on_success needs: - - oc.docker:jsonnet:amd64 - trigger - dependencies: - - oc.docker:jsonnet:amd64 + dependencies: [] timeout: 60 minutes interruptible: true before_script: @@ -3382,39 +3250,58 @@ grafazos.build: - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - cd grafazos/ - - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec - - make + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - name: grafazos-dashboards - expire_in: 1 day - paths: - - grafazos/output/**/*.json - when: on_success -teztale.build-arm64: +etherlink.build_tezt: image: ${ci_image_name}/build:${ci_image_tag} - stage: test + stage: build tags: - - gcp_arm64 + - gcp rules: - changes: - - teztale/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - - oc.docker:ci:arm64 - - trigger + - oc.docker:ci:amd64 dependencies: - - oc.docker:ci:arm64 + - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -3425,15 +3312,14 @@ teztale.build-arm64: - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - make teztale - - mkdir -p ./teztale-binaries/arm64 - - mv octez-teztale-* ./teztale-binaries/arm64/ + - dune build etherlink/tezt/tests/main.exe - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - PROFILE: static CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -3441,531 +3327,625 @@ teztale.build-arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: teztale-binaries + name: etherlink_tezt_exe expire_in: 1 day paths: - - teztale-binaries/arm64/octez-teztale-* + - _build/default/etherlink/tezt/tests/main.exe when: on_success -teztale.build-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +etherlink.tezt-flaky: + image: ${ci_image_name}/e2etest:${ci_image_tag} stage: test tags: - - gcp_very_high_cpu_ramfs + - gcp_tezt rules: - changes: - - teztale/**/* - when: on_success + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + allow_failure: true + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - - eval $(opam env) - - make teztale - - mkdir -p ./teztale-binaries/x86_64 - - mv octez-teztale-* ./teztale-binaries/x86_64/ + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/flaky --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/flaky + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - PROFILE: static - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + TEZT_NO_NPX: "true" artifacts: - name: teztale-binaries - expire_in: 1 day + expire_in: 7 days paths: - - teztale-binaries/x86_64/octez-teztale-* - when: on_success + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 2 -documentation.manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt-extra: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 + - trigger - oc.build_x86_64-released - - oc.build_amd64-extra-dev - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt - oc.build_kernels dependencies: - oc.docker:ci:amd64 - oc.build_x86_64-released - - oc.build_amd64-extra-dev - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt - oc.build_kernels - timeout: 60 minutes + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - eval $(opam env) - - make -C docs -j octez-gen + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/extra --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/extra + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -extra + TESTS: extra && /ci_disabled && /time_sensitive && /slow && /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "6" + TEZT_NO_NPX: "true" artifacts: - expire_in: 1 week + expire_in: 7 days paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 12 -documentation.odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt-slow: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true needs: - oc.docker:ci:amd64 + - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh script: - - eval $(opam env) - - make -C docs odoc-lite + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --junit-mem-peak + 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results.json --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} + --record-mem-peak --mem-warn 5_000_000_000 --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' + --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' --junit-tag 'dd_tags[tezt-tag.slow]=slow' + --junit-tag 'dd_tags[tezt-tag.extra]=extra' --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' + --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' + --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /time_sensitive && /extra && /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + TEZT_NO_NPX: "true" artifacts: - expire_in: 4 hours + expire_in: 7 days paths: - - docs/_build/api/odoc/ - - docs/odoc.log + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT when: always + retry: 2 + parallel: 3 -documentation.build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build +etherlink.tezt: + image: ${ci_image_name}/e2etest:${ci_image_tag} + stage: test tags: - - gcp + - gcp_tezt rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* - dune - dune-project - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* + - etherlink/**/* + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* when: on_success needs: - oc.docker:ci:amd64 - trigger - - documentation.odoc - - documentation.manuals - - documentation.docgen + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - documentation.odoc - - documentation.manuals - - documentation.docgen - timeout: 60 minutes + - oc.build_x86_64-released + - oc.build_amd64-exp + - select_tezts + - etherlink.build_tezt + - oc.build_kernels + timeout: 40 minutes interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: - - eval $(opam env) - - . $HOME/.venv/bin/activate - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + - . ./scripts/version.sh + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts + -- "${TESTS}" --from-record tezt/records/etherlink --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit + ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' + --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log + --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit + ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink + --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count + ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 + --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' + --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' + --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' + --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' + --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "6" + TEZT_NO_NPX: "true" artifacts: - expire_in: 1 week + expire_in: 7 days paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results.json + - $JUNIT + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 18 -documentation.linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} +etherlink.unit_tests: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - gcp rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: manual - allow_failure: true - changes: - - '**/*.rst' - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: manual - allow_failure: true + - etherlink/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* + when: on_success needs: - oc.docker:ci:amd64 - trigger - - documentation.manuals - - documentation.docgen - - documentation.build_all dependencies: - oc.docker:ci:amd64 - - documentation.manuals - - documentation.docgen - - documentation.build_all - allow_failure: true timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - . ./scripts/version.sh - eval $(opam env) - - . $HOME/.venv/bin/activate - - make -C docs redirectcheck - - make -C docs linkcheck + - make test-etherlink-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + DUNE_ARGS: -j 12 + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + artifacts: + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day + paths: + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 -documentation.install_python_bookworm: - image: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/debian:bookworm +etherlink.build_evm_node_static_arm64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - changes: - - docs/developer/install-python-debian-ubuntu.sh - - poetry.lock - - pyproject.toml - when: on_success + - etherlink/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* + when: manual + allow_failure: true needs: + - oc.docker:ci:arm64 - trigger - dependencies: [] + dependencies: + - oc.docker:ci:arm64 timeout: 60 minutes + cache: + key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - ./scripts/ci/build_static_binaries.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + ARCH: arm64 + VERSION_EXECUTABLE: octez-evm-node + EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + name: octez-binaries + paths: + - octez-binaries/$ARCH/* + when: on_success -etherlink.tezt-flaky: - image: ${ci_image_name}/e2etest:${ci_image_tag} +etherlink.build_evm_node_static_x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* + - sdk/rust/**/* + - src/kernel_sdk/**/* when: manual allow_failure: true needs: - oc.docker:ci:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - allow_failure: true - timeout: 40 minutes + timeout: 60 minutes + cache: + key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/flaky --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/flaky - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - ./scripts/ci/build_static_binaries.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -flaky - TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && flaky - TEZT_RETRY: "3" - TEZT_PARALLEL: "1" - TEZT_NO_NPX: "true" + ARCH: x86_64 + VERSION_EXECUTABLE: octez-evm-node + EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" artifacts: - expire_in: 7 days + name: octez-binaries paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 2 + - octez-binaries/$ARCH/* + when: on_success -etherlink.tezt-extra: - image: ${ci_image_name}/e2etest:${ci_image_tag} +ocaml-check: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - Makefile - - brassaia-eio/**/* - - brassaia/**/* - - data-encoding/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/* - - irmin/**/* - - lwt_domain/**/* - - michelson_test_scripts/**/* - - opam/**/* - - prometheus/**/* - - resto/**/* - - rust-toolchain - - script-inputs/**/* - - scripts/**/* - - sdk/**/* + - '**/*.ml' + - '**/*.mli' + - devtools/**/* - src/**/* - tezt/**/* - - tzt_reference_test_suite/**/* - when: manual - allow_failure: true + when: on_success needs: - oc.docker:ci:amd64 - trigger - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes + timeout: 60 minutes + cache: + key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' script: + - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/extra --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/extra - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - eval $(opam env) + - scripts/ci/dune.sh build @check --stop-on-first-error - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - eval $(opam env) + - dune cache trim --size=5GB variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -extra - TESTS: extra && /ci_disabled && /time_sensitive && /slow && /cloud && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "6" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 12 + CARGO_NET_OFFLINE: "false" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache -etherlink.tezt-slow: - image: ${ci_image_name}/e2etest:${ci_image_tag} +oc.unit:other-x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_high_cpu rules: - changes: - .gitlab-ci.yml @@ -3992,76 +3972,68 @@ etherlink.tezt-slow: - src/**/* - tezt/**/* - tzt_reference_test_suite/**/* - when: manual - allow_failure: true + when: on_success needs: - oc.docker:ci:amd64 - trigger - oc.build_x86_64-released - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --junit-mem-peak - 'dd_tags[memory.peak]' --from-record tezt/records/etherlink/slow --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results.json --job-count ${TEZT_PARALLEL} --retry ${TEZT_RETRY} - --record-mem-peak --mem-warn 5_000_000_000 --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' - --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' --junit-tag 'dd_tags[tezt-tag.slow]=slow' - --junit-tag 'dd_tags[tezt-tag.extra]=extra' --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' - --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' - --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' + - eval $(opam env) + - make test-other-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: -slow - TESTS: slow && /ci_disabled && /time_sensitive && /extra && /cloud - TEZT_RETRY: "1" - TEZT_PARALLEL: "3" - TEZT_NO_NPX: "true" + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + DUNE_ARGS: -j 12 + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - expire_in: 7 days + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT + - test_results reports: - junit: $JUNIT + junit: test_results/*.xml when: always retry: 2 - parallel: 3 -etherlink.tezt: - image: ${ci_image_name}/e2etest:${ci_image_tag} +oc.unit:proto-x86_64: + image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_tezt + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml @@ -4094,88 +4066,19 @@ etherlink.tezt: - trigger - oc.build_x86_64-released - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - select_tezts - - etherlink.build_tezt - - oc.build_kernels - timeout: 40 minutes - interruptible: true - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - . ./scripts/version.sh - - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" - CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" - - ./scripts/ci/tezt.sh --tezt-exe 'etherlink/tezt/tests/main.exe' --with-select-tezts - -- "${TESTS}" --from-record tezt/records/etherlink --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh --send-junit - ${JUNIT} --datadog-service etherlink --tezt-exe 'etherlink/tezt/tests/main.exe' - --with-select-tezts -- "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log - --global-timeout 1800 --test-timeout 540 --on-unknown-regression-files fail --junit - ${JUNIT} --junit-mem-peak 'dd_tags[memory.peak]' --from-record tezt/records/etherlink - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record tezt-results.json --job-count - ${TEZT_PARALLEL} --retry ${TEZT_RETRY} --record-mem-peak --mem-warn 5_000_000_000 - --junit-tag 'dd_tags[tezt-tag.flaky]=flaky' --junit-tag 'dd_tags[tezt-tag.time_sensitive]=time_sensitive' - --junit-tag 'dd_tags[tezt-tag.slow]=slow' --junit-tag 'dd_tags[tezt-tag.extra]=extra' - --junit-tag 'dd_tags[tezt-tag.infrastructure]=infrastructure' --junit-tag 'dd_tags[tezt-tag.layer1]=layer1' - --junit-tag 'dd_tags[tezt-tag.tezos2]=tezos2' --junit-tag 'dd_tags[tezt-tag.etherlink]=etherlink' - --junit-tag 'dd_tags[tezt-tag.memory_hungry]=memory_hungry' - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - variables: - JUNIT: tezt-junit.xml - TEZT_VARIANT: "" - TESTS: /ci_disabled && /time_sensitive && /slow && /extra && /cloud && /flaky - TEZT_RETRY: "1" - TEZT_PARALLEL: "6" - TEZT_NO_NPX: "true" - artifacts: - expire_in: 7 days - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results.json - - $JUNIT - reports: - junit: $JUNIT - when: always - retry: 2 - parallel: 18 - -etherlink.build_evm_node_static_arm64: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - rules: - - changes: - - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true - needs: - - oc.docker:ci:arm64 - - trigger - dependencies: - - oc.docker:ci:arm64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -4183,17 +4086,17 @@ etherlink.build_evm_node_static_arm64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - ./scripts/ci/build_static_binaries.sh + - make test-proto-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - ARCH: arm64 - VERSION_EXECUTABLE: octez-evm-node - EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -4201,35 +4104,69 @@ etherlink.build_evm_node_static_arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: octez-binaries + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - octez-binaries/$ARCH/* - when: on_success + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 -etherlink.build_evm_node_static_x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +oc.unit:non-proto-x86_64: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp_very_high_cpu + - gcp rules: - changes: + - .gitlab-ci.yml + - .gitlab/ci/pipelines/before_merging.yml + - .gitlab/ci/pipelines/merge_train.yml + - Makefile + - brassaia-eio/**/* + - brassaia/**/* + - data-encoding/**/* + - dune + - dune-project + - dune-workspace - etherlink/**/* - - sdk/rust/**/* - - src/kernel_sdk/**/* - when: manual - allow_failure: true + - irmin/**/* + - lwt_domain/**/* + - michelson_test_scripts/**/* + - opam/**/* + - prometheus/**/* + - resto/**/* + - rust-toolchain + - script-inputs/**/* + - scripts/**/* + - sdk/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: on_success needs: - oc.docker:ci:amd64 - trigger + - oc.build_x86_64-released + - oc.build_amd64-exp + - oc.build_amd64-extra-dev dependencies: - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull interruptible: true before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -4237,17 +4174,17 @@ etherlink.build_evm_node_static_x86_64: - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - . ./scripts/ci/sccache-start.sh script: - - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh - eval $(opam env) - - ./scripts/ci/build_static_binaries.sh + - make test-nonproto-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - ARCH: x86_64 - VERSION_EXECUTABLE: octez-evm-node - EXECUTABLE_FILES: script-inputs/etherlink-experimental-executables + DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -4255,11 +4192,18 @@ etherlink.build_evm_node_static_x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: octez-binaries + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + expire_in: 1 day paths: - - octez-binaries/$ARCH/* - when: on_success + - test_results + reports: + junit: test_results/*.xml + when: always + retry: 2 de.unit:arm64: image: ${ci_image_name}/test:${ci_image_tag} @@ -4369,6 +4313,62 @@ resto.unit:x86_64: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +oc.script:snapshot_alpha_and_link: + image: ${ci_image_name}/build:${ci_image_tag} + stage: test + tags: + - gcp_very_high_cpu + rules: + - changes: + - scripts/snapshot_alpha.sh + - scripts/snapshot_alpha_and_link.sh + - scripts/user_activated_upgrade.sh + - src/proto_alpha/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull + interruptible: true + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - ./scripts/ci/script:snapshot_alpha_and_link.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB + variables: + CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + tezt-static-binaries: image: ${ci_image_name}/e2etest:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index b976a75d9af653ddbecddf50f03f47489a9ba92e..23c1ee4c411675a98affe9110a1b8a9facb3d0e5 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -639,10 +639,10 @@ oc.build_amd64-extra-dev: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -705,10 +705,10 @@ oc.build_amd64-exp: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: push + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -1317,54 +1317,6 @@ oc.script:test-gen-genesis: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp_very_high_cpu - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - ./scripts/ci/script:snapshot_alpha_and_link.sh - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB - variables: - CARGO_NET_OFFLINE: "false" - CARGO_INCREMENTAL: "0" - SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET - SCCACHE_GCS_RW_MODE: READ_ONLY - SCCACHE_GCS_KEY_PREFIX: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: "1" - SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - oc.script:b58_prefix: image: ${ci_image_name}/test:${ci_image_tag} stage: test @@ -1525,28 +1477,21 @@ oc.unit:webassembly-x86_64: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -oc.unit:non-proto-x86_64: +oc.unit:non-proto-arm64: image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp + - gcp_arm64 needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-exp - - oc.build_amd64-extra-dev + - oc.docker:ci:arm64 dependencies: - - oc.docker:ci:amd64 + - oc.docker:ci:arm64 timeout: 60 minutes cache: - - key: cargo-$CI_JOB_NAME_SLUG + key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -1556,14 +1501,13 @@ oc.unit:non-proto-x86_64: script: - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit + - make test-nonproto-unit test-webassembly - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh - - eval $(opam env) - - dune cache trim --size=5GB variables: + DISTRIBUTE_TESTS_TO_PARALLELS: "true" DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" @@ -1572,11 +1516,8 @@ oc.unit:non-proto-x86_64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 + name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 expire_in: 1 day paths: - test_results @@ -1584,8 +1525,9 @@ oc.unit:non-proto-x86_64: junit: test_results/*.xml when: always retry: 2 + parallel: 2 -oc.unit:non-proto-arm64: +de.unit:arm64: image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: @@ -1595,11 +1537,142 @@ oc.unit:non-proto-arm64: dependencies: - oc.docker:ci:arm64 timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - dune runtest data-encoding + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +de.unit:x86_64: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - dune runtest data-encoding + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +resto.unit:arm64: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp_arm64 + needs: + - oc.docker:ci:arm64 + dependencies: + - oc.docker:ci:arm64 + timeout: 10 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - dune runtest resto + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +resto.unit:x86_64: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 10 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - dune runtest resto + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +ocaml-check: + image: ${ci_image_name}/build:${ci_image_tag} + stage: test + tags: + - gcp_very_high_cpu + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG + key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull-push + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - scripts/ci/dune.sh build @check --stop-on-first-error + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - eval $(opam env) + - dune cache trim --size=5GB + variables: + CARGO_NET_OFFLINE: "false" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache + +oc.unit:other-x86_64: + image: ${ci_image_name}/build:${ci_image_tag} + stage: test + tags: + - gcp_high_cpu + needs: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-exp + - oc.build_amd64-extra-dev + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_dune_cache + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -1609,13 +1682,14 @@ oc.unit:non-proto-arm64: script: - . ./scripts/version.sh - eval $(opam env) - - make test-nonproto-unit test-webassembly + - make test-other-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - ./scripts/ci/sccache-stop.sh + - eval $(opam env) + - dune cache trim --size=5GB variables: - DISTRIBUTE_TESTS_TO_PARALLELS: "true" DUNE_ARGS: -j 12 CARGO_NET_OFFLINE: "false" CARGO_INCREMENTAL: "0" @@ -1624,8 +1698,11 @@ oc.unit:non-proto-arm64: SCCACHE_GCS_KEY_PREFIX: sccache SCCACHE_IGNORE_SERVER_IO_ERROR: "1" SCCACHE_IDLE_TIMEOUT: "0" + DUNE_CACHE: enabled + DUNE_CACHE_STORAGE_MODE: hardlink + DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache artifacts: - name: $CI_JOB_NAME-$CI_COMMIT_SHA-arm64 + name: $CI_JOB_NAME-$CI_COMMIT_SHA-x86_64 expire_in: 1 day paths: - test_results @@ -1633,7 +1710,6 @@ oc.unit:non-proto-arm64: junit: test_results/*.xml when: always retry: 2 - parallel: 2 oc.unit:proto-x86_64: image: ${ci_image_name}/build:${ci_image_tag} @@ -1653,10 +1729,10 @@ oc.unit:proto-x86_64: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: pull + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -1695,11 +1771,11 @@ oc.unit:proto-x86_64: when: always retry: 2 -oc.unit:other-x86_64: - image: ${ci_image_name}/build:${ci_image_tag} +oc.unit:non-proto-x86_64: + image: ${ci_image_name}/test:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp needs: - oc.docker:ci:amd64 - oc.build_x86_64-released @@ -1713,10 +1789,10 @@ oc.unit:other-x86_64: paths: - $CI_PROJECT_DIR/.cargo/registry/cache policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache - policy: pull + policy: pull-push interruptible: false before_script: - SCRIPT_STEP_BEGIN=$(date +%s) @@ -1726,7 +1802,7 @@ oc.unit:other-x86_64: script: - . ./scripts/version.sh - eval $(opam env) - - make test-other-unit + - make test-nonproto-unit - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' @@ -1755,7 +1831,7 @@ oc.unit:other-x86_64: when: always retry: 2 -ocaml-check: +oc.script:snapshot_alpha_and_link: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: @@ -1766,7 +1842,11 @@ ocaml-check: - oc.docker:ci:amd64 timeout: 60 minutes cache: - key: dune_cache-$CI_JOB_NAME_SLUG + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: dune_cache-$CI_JOB_NAME_SLUG paths: - $CI_PROJECT_DIR/_dune_cache policy: pull-push @@ -1775,110 +1855,30 @@ ocaml-check: - SCRIPT_STEP_BEGIN=$(date +%s) - . ./scripts/ci/datadog_send_job_info.sh - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh script: - ./scripts/ci/take_ownership.sh - . ./scripts/version.sh - eval $(opam env) - - scripts/ci/dune.sh build @check --stop-on-first-error + - ./scripts/ci/script:snapshot_alpha_and_link.sh - . ./scripts/ci/datadog_send_job_script_step_time.sh || true after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh - eval $(opam env) - dune cache trim --size=5GB variables: CARGO_NET_OFFLINE: "false" + CARGO_INCREMENTAL: "0" + SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET + SCCACHE_GCS_RW_MODE: READ_ONLY + SCCACHE_GCS_KEY_PREFIX: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_IDLE_TIMEOUT: "0" DUNE_CACHE: enabled DUNE_CACHE_STORAGE_MODE: hardlink DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache -de.unit:arm64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - needs: - - oc.docker:ci:arm64 - dependencies: - - oc.docker:ci:arm64 - timeout: 60 minutes - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - eval $(opam env) - - dune runtest data-encoding - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -de.unit:x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - eval $(opam env) - - dune runtest data-encoding - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -resto.unit:arm64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp_arm64 - needs: - - oc.docker:ci:arm64 - dependencies: - - oc.docker:ci:arm64 - timeout: 10 minutes - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - eval $(opam env) - - dune runtest resto - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -resto.unit:x86_64: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 - timeout: 10 minutes - interruptible: false - before_script: - - SCRIPT_STEP_BEGIN=$(date +%s) - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - eval $(opam env) - - dune runtest resto - - . ./scripts/ci/datadog_send_job_script_step_time.sh || true - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - tezt-static-binaries: image: ${ci_image_name}/e2etest:${ci_image_tag} stage: test diff --git a/ci/lib_cacio/cacio.ml b/ci/lib_cacio/cacio.ml index 600209e35db6db594ef6c579e17b36e145f7b7eb..9b830a73c849b8dfb35d05881087de2bdf14ae29 100644 --- a/ci/lib_cacio/cacio.ml +++ b/ci/lib_cacio/cacio.ml @@ -41,12 +41,9 @@ type sccache_config = { let sccache ?error_log ?log ?policy () = {error_log; log; policy} -type dune_cache_config = { - key : Tezos_ci.Cache.cache_key option; - policy : Gitlab_ci.Types.cache_policy option; -} +type dune_cache_config = {pipeline_type : Tezos_ci.Cache.pipeline_type} -let dune_cache ?key ?policy () = {key; policy} +let dune_cache ~pipeline_type () = {pipeline_type} (* Conditions are disjunctions: the job is included in the pipeline if ANY file in [changed] changed, or if the merge request has ANY of the [label]s. *) @@ -496,8 +493,8 @@ let convert_graph ?(interruptible_pipeline = true) let maybe_enable_dune_cache job = match dune_cache with | None -> job - | Some {key; policy} -> - Tezos_ci.Cache.enable_dune_cache ?key ?policy job + | Some {pipeline_type} -> + Tezos_ci.Cache.enable_dune_cache ~pipeline_type job in Tezos_ci.job ~__POS__:source_location diff --git a/ci/lib_cacio/cacio.mli b/ci/lib_cacio/cacio.mli index 136ddbf98fa1dcd30934ae49c7fe04f609eef794..fd637b104cece223e6148c4edc1e82283c3999ae 100644 --- a/ci/lib_cacio/cacio.mli +++ b/ci/lib_cacio/cacio.mli @@ -66,10 +66,7 @@ type dune_cache_config See {!Tezos_ci.Cache.enable_dune_cache}. *) val dune_cache : - ?key:Tezos_ci.Cache.cache_key -> - ?policy:Gitlab_ci.Types.cache_policy -> - unit -> - dune_cache_config + pipeline_type:Tezos_ci.Cache.pipeline_type -> unit -> dune_cache_config (** Pipeline jobs. diff --git a/ci/lib_tezos_ci/tezos_ci.ml b/ci/lib_tezos_ci/tezos_ci.ml index bb86cc0a88bc9f74dcd8e2cda777baeb0ae971f1..d06eab85c34baaec267941cb6278f71d311a4dda 100644 --- a/ci/lib_tezos_ci/tezos_ci.ml +++ b/ci/lib_tezos_ci/tezos_ci.ml @@ -1472,15 +1472,17 @@ let job_docker_authenticated ?(skip_docker_initialization = false) (** {2 Caches} *) module Cache = struct - type cache_key = Job | Pipeline + type pipeline_type = Cache_pipeline | Regular_pipeline - let enable_dune_cache ?(key = Job) ?(policy = Gitlab_ci.Types.Pull_push) job = + let enable_dune_cache ~pipeline_type job = let path = "$CI_PROJECT_DIR/_dune_cache" in + let policy = + match pipeline_type with + | Regular_pipeline -> Gitlab_ci.Types.Pull + | Cache_pipeline -> Pull_push + in let key = - match key with - | Job -> "dune_cache-" ^ Gitlab_ci.Predefined_vars.(show ci_job_name_slug) - | Pipeline -> - "dune-build-cache-" ^ Gitlab_ci.Predefined_vars.(show ci_pipeline_id) + "dune_cache-" ^ Gitlab_ci.Predefined_vars.(show ci_job_name_slug) in job |> append_variables diff --git a/ci/lib_tezos_ci/tezos_ci.mli b/ci/lib_tezos_ci/tezos_ci.mli index 50e7859ff253f8b6387f5ce9591e5376f7fe1c74..d54a7d321c60844b0ae56d8ea814b5f284f4ca2c 100644 --- a/ci/lib_tezos_ci/tezos_ci.mli +++ b/ci/lib_tezos_ci/tezos_ci.mli @@ -184,29 +184,21 @@ module Pipeline : sig end module Cache : sig - (* Possible values for [~key] arguments. - - - [Job]: share the cache between all instances of the job, across all pipelines. - - [Pipeline]: share the cache between all jobs of each pipeline, but not across pipelines. *) - type cache_key = Job | Pipeline - - (** Add variable enabling dune cache. + type pipeline_type = + | Cache_pipeline + | Regular_pipeline + (** Add variable enabling dune cache. This function can be applied to jobs that run dune. - - [key] specifies which jobs the cache should be shared with. - See {!cache_key}. - - - [policy] specifies whether to pull the cache when the job starts, - and whether to push the cache when the job finishes. + - [pipeline_type] specifies the pipeline type where the job is + running, if the pipeline if of type [Regular_pipeline] then the + policy for the cache is set to pull, if of [Cache_pipeline] it is + set to pull_push *) - val enable_dune_cache : - ?key:cache_key -> - ?policy:Gitlab_ci.Types.cache_policy -> - tezos_job -> - tezos_job + val enable_dune_cache : pipeline_type:pipeline_type -> tezos_job -> tezos_job (** Add variable enabling sccache. diff --git a/ci/lib_tezos_ci_jobs/code_verification.ml b/ci/lib_tezos_ci_jobs/code_verification.ml index c47409c59263cca7a7a022952a14687cb9bae1ed..56485b4ea2e9b61223bf00d3eb00ccf3b001da20 100644 --- a/ci/lib_tezos_ci_jobs/code_verification.ml +++ b/ci/lib_tezos_ci_jobs/code_verification.ml @@ -50,6 +50,11 @@ type code_verification_pipeline = | Schedule_extended_test | Merge_train +let cache_pipeline_type pipeline_type = + match pipeline_type with + | Schedule_extended_test -> Tezos_ci.Cache.Cache_pipeline + | Before_merging | Merge_train -> Tezos_ci.Cache.Regular_pipeline + let code_verification_pipeline_name = function | Before_merging -> "before_merging" | Schedule_extended_test -> "schedule_extended_test" @@ -184,7 +189,7 @@ let job_build_x86_64_extra_dev = ~rules:(make_rules ~pipeline_type ~changes:changeset_octez_or_doc ()) ~extra:true "script-inputs/dev-executables" - |> enable_dune_cache ~key:Pipeline ~policy:Push + |> enable_dune_cache ~pipeline_type:(cache_pipeline_type pipeline_type) let job_build_x86_64_exp = depending_on_pipeline_type @@ fun pipeline_type -> @@ -196,7 +201,7 @@ let job_build_x86_64_exp = ~dependencies:(dependencies_needs_start pipeline_type) ~rules:(make_rules ~pipeline_type ~changes:changeset_octez_or_doc ()) "script-inputs/experimental-executables" - |> enable_dune_cache ~key:Pipeline ~policy:Push + |> enable_dune_cache ~pipeline_type:(cache_pipeline_type pipeline_type) let build_arm_rules ~pipeline_type = make_rules ~pipeline_type ~label:"ci--arm64" ~manual:Yes () @@ -312,7 +317,7 @@ let jobs pipeline_type = (* Check actual formatting. *) "scripts/ci/dune.sh build --profile=dev @fmt"; ] - |> enable_dune_cache + |> enable_dune_cache ~pipeline_type:(cache_pipeline_type pipeline_type) in let job_semgrep : tezos_job = job diff --git a/ci/lib_tezos_ci_jobs/misc.ml b/ci/lib_tezos_ci_jobs/misc.ml index a4846fdb6cc1f373945a81785ba47a6f99e56905..0747dafb887a4439eb69686fd8659706c01a0e9e 100644 --- a/ci/lib_tezos_ci_jobs/misc.ml +++ b/ci/lib_tezos_ci_jobs/misc.ml @@ -81,9 +81,6 @@ let job_script_test_gen_genesis = ~only_if_changed:(Changesets.changeset_octez |> Tezos_ci.Changeset.encode) ["eval $(opam env)"; "dune build scripts/gen-genesis/gen_genesis.exe"] -let dune_cache_pull_from_pipeline = - Cacio.dune_cache ~key:Pipeline ~policy:Pull () - (* Add these jobs to the [needs_legacy] of jobs that use [dune_cache_pull_from_pipeline] so that the cache is actually available. *) let needs_cache_from_x86_64_build_jobs = @@ -95,6 +92,7 @@ let needs_cache_from_x86_64_build_jobs = ] let job_script_snapshot_alpha_and_link = + Cacio.parameterize @@ fun pipeline_type -> CI.job "oc.script:snapshot_alpha_and_link" ~__POS__ @@ -112,7 +110,7 @@ let job_script_snapshot_alpha_and_link = ] ~cargo_cache:true ~sccache:(Cacio.sccache ()) - ~dune_cache:dune_cache_pull_from_pipeline + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) [ "./scripts/ci/take_ownership.sh"; ". ./scripts/version.sh"; @@ -277,6 +275,7 @@ let artifacts_test_results_xml arch = ~when_:Always let job_oc_unit_non_proto_x86_64 = + Cacio.parameterize @@ fun pipeline_type -> CI.job "oc.unit:non-proto-x86_64" ~__POS__ @@ -291,7 +290,7 @@ let job_oc_unit_non_proto_x86_64 = ~needs_legacy:needs_cache_from_x86_64_build_jobs ~variables:[("DUNE_ARGS", "-j 12")] ~artifacts:(artifacts_test_results_xml Amd64) - ~dune_cache:dune_cache_pull_from_pipeline + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) ~cargo_cache:true ~sccache:(Cacio.sccache ()) [ @@ -336,6 +335,7 @@ let job_oc_unit_non_proto_arm64 = ] let job_oc_unit_proto_x86_64 = + Cacio.parameterize @@ fun pipeline_type -> CI.job "oc.unit:proto-x86_64" ~__POS__ @@ -349,13 +349,14 @@ let job_oc_unit_proto_x86_64 = ~needs_legacy:needs_cache_from_x86_64_build_jobs ~variables:[("DUNE_ARGS", "-j 12")] ~artifacts:(artifacts_test_results_xml Amd64) - ~dune_cache:dune_cache_pull_from_pipeline + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) [". ./scripts/version.sh"; "eval $(opam env)"; "make test-proto-unit"] ~cargo_cache:true ~sccache:(Cacio.sccache ()) (* TODO: the changeset is silly since this tests non-Octez parts of the repository. *) let job_oc_unit_other_x86_64 = + Cacio.parameterize @@ fun pipeline_type -> CI.job "oc.unit:other-x86_64" ~__POS__ @@ -369,12 +370,13 @@ let job_oc_unit_other_x86_64 = ~needs_legacy:needs_cache_from_x86_64_build_jobs ~variables:[("DUNE_ARGS", "-j 12")] ~artifacts:(artifacts_test_results_xml Amd64) - ~dune_cache:dune_cache_pull_from_pipeline + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) [". ./scripts/version.sh"; "eval $(opam env)"; "make test-other-unit"] ~cargo_cache:true ~sccache:(Cacio.sccache ()) let job_ocaml_check = + Cacio.parameterize @@ fun pipeline_type -> CI.job "ocaml-check" ~__POS__ @@ -385,7 +387,7 @@ let job_ocaml_check = ~only_if_changed: ["src/**/*"; "tezt/**/*"; "devtools/**/*"; "**/*.ml"; "**/*.mli"] ~variables:[("CARGO_NET_OFFLINE", "false")] - ~dune_cache:(Cacio.dune_cache ()) + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) [ "./scripts/ci/take_ownership.sh"; ". ./scripts/version.sh"; @@ -397,13 +399,14 @@ let job_ocaml_check = ] let register () = + let pipeline_type = Tezos_ci.Cache.Regular_pipeline in CI.register_before_merging_jobs [ (Auto, job_check_lift_limits_patch); (Auto, job_python_check); (Auto, job_integration_compiler_rejections); (Auto, job_script_test_gen_genesis); - (Auto, job_script_snapshot_alpha_and_link); + (Auto, job_script_snapshot_alpha_and_link pipeline_type); (Auto, job_script_b58_prefix); (Auto, job_test_liquidity_baking_scripts); (Auto, job_oc_script_test_release_versions); @@ -414,19 +417,20 @@ let register () = (Auto, job_de_unit Arm64); (Auto, job_oc_unit_protocol_compiles); (Auto, job_oc_unit_webassembly_x86_64); - (Auto, job_oc_unit_non_proto_x86_64); + (Auto, job_oc_unit_non_proto_x86_64 pipeline_type); (Auto, job_oc_unit_non_proto_arm64); - (Auto, job_oc_unit_proto_x86_64); - (Auto, job_oc_unit_other_x86_64); - (Auto, job_ocaml_check); + (Auto, job_oc_unit_proto_x86_64 pipeline_type); + (Auto, job_oc_unit_other_x86_64 pipeline_type); + (Auto, job_ocaml_check pipeline_type); ] ; + let pipeline_type = Tezos_ci.Cache.Cache_pipeline in CI.register_schedule_extended_test_jobs [ (Auto, job_check_lift_limits_patch); (Auto, job_python_check); (Auto, job_integration_compiler_rejections); (Auto, job_script_test_gen_genesis); - (Auto, job_script_snapshot_alpha_and_link); + (Auto, job_script_snapshot_alpha_and_link pipeline_type); (Auto, job_script_b58_prefix); (Auto, job_test_liquidity_baking_scripts); (Auto, job_oc_script_test_release_versions); @@ -437,10 +441,10 @@ let register () = (Auto, job_de_unit Arm64); (Auto, job_oc_unit_protocol_compiles); (Auto, job_oc_unit_webassembly_x86_64); - (Auto, job_oc_unit_non_proto_x86_64); + (Auto, job_oc_unit_non_proto_x86_64 pipeline_type); (Auto, job_oc_unit_non_proto_arm64); - (Auto, job_oc_unit_proto_x86_64); - (Auto, job_oc_unit_other_x86_64); - (Auto, job_ocaml_check); + (Auto, job_oc_unit_proto_x86_64 pipeline_type); + (Auto, job_oc_unit_other_x86_64 pipeline_type); + (Auto, job_ocaml_check pipeline_type); ] ; () diff --git a/etherlink/ci/etherlink_ci.ml b/etherlink/ci/etherlink_ci.ml index 8e6c86bed694da8f396273f4c314f9ba632058cf..e5cc00cdf2498d4ab1e2f63c8a41269d3b38b9fa 100644 --- a/etherlink/ci/etherlink_ci.ml +++ b/etherlink/ci/etherlink_ci.ml @@ -165,6 +165,7 @@ let job_lint_wasm_runtime = ] let job_unit_tests = + Cacio.parameterize @@ fun pipeline_type -> CI.job "unit_tests" ~__POS__ @@ -181,7 +182,7 @@ let job_unit_tests = ~when_:Always) ~cargo_cache:true ~sccache:(Cacio.sccache ()) - ~dune_cache:(Cacio.dune_cache ()) + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) ~variables:[("DUNE_ARGS", "-j 12")] ~retry:{max = 2; when_ = []} [". ./scripts/version.sh"; "eval $(opam env)"; "make test-etherlink-unit"] @@ -281,6 +282,7 @@ let job_mir_tzt = ] let job_build_tezt = + Cacio.parameterize @@ fun pipeline_type -> CI.job "build_tezt" ~__POS__ @@ -293,7 +295,7 @@ let job_build_tezt = ~when_:On_success ~expire_in:(Duration (Days 1)) ["_build/default/etherlink/tezt/tests/main.exe"]) - ~dune_cache:(Cacio.dune_cache ~key:Pipeline ()) + ~dune_cache:(Cacio.dune_cache ~pipeline_type ()) ~cargo_cache:true ~sccache:(Cacio.sccache ()) [ @@ -311,11 +313,6 @@ let tezt_job ?(retry_tests = 1) = CI.tezt_job ~tezt_exe:"etherlink/tezt/tests/main.exe" ~only_if_changed:(Changeset.encode Tezos_ci_jobs.Changesets.changeset_octez) - ~needs: - [ - (Artifacts, job_build_tezt); - (Artifacts, Tezos_ci_jobs.Kernels.job_build_kernels); - ] ~needs_legacy: [ ( Artifacts, @@ -328,10 +325,20 @@ let tezt_job ?(retry_tests = 1) = let job_tezt = Cacio.parameterize @@ fun pipeline -> + let pipeline_type = + match pipeline with + | `merge_request -> Tezos_ci.Cache.Regular_pipeline + | `scheduled -> Tezos_ci.Cache.Cache_pipeline + in tezt_job "" ~__POS__ ~pipeline + ~needs: + [ + (Artifacts, job_build_tezt pipeline_type); + (Artifacts, Tezos_ci_jobs.Kernels.job_build_kernels); + ] ~description:"Run normal Etherlink Tezt tests." ~test_selection: (Tezos_ci_jobs.Tezt.tests_tag_selector [Not (Has_tag "flaky")]) @@ -341,10 +348,20 @@ let job_tezt = let job_tezt_slow = Cacio.parameterize @@ fun pipeline -> + let pipeline_type = + match pipeline with + | `merge_request -> Tezos_ci.Cache.Regular_pipeline + | `scheduled -> Tezos_ci.Cache.Cache_pipeline + in tezt_job "slow" ~__POS__ ~pipeline + ~needs: + [ + (Artifacts, job_build_tezt pipeline_type); + (Artifacts, Tezos_ci_jobs.Kernels.job_build_kernels); + ] ~description:"Run Etherlink Tezt tests tagged as slow." ~test_selection:(Tezos_ci_jobs.Tezt.tests_tag_selector ~slow:true []) ~test_timeout:No_timeout @@ -354,10 +371,20 @@ let job_tezt_slow = let job_tezt_extra = Cacio.parameterize @@ fun pipeline -> + let pipeline_type = + match pipeline with + | `merge_request -> Tezos_ci.Cache.Regular_pipeline + | `scheduled -> Tezos_ci.Cache.Cache_pipeline + in tezt_job "extra" ~__POS__ ~pipeline + ~needs: + [ + (Artifacts, job_build_tezt pipeline_type); + (Artifacts, Tezos_ci_jobs.Kernels.job_build_kernels); + ] ~description:"Run Etherlink Tezt tests tagged as extra and not flaky." ~test_selection: (Tezos_ci_jobs.Tezt.tests_tag_selector @@ -369,10 +396,20 @@ let job_tezt_extra = let job_tezt_flaky = Cacio.parameterize @@ fun pipeline -> + let pipeline_type = + match pipeline with + | `merge_request -> Tezos_ci.Cache.Regular_pipeline + | `scheduled -> Tezos_ci.Cache.Cache_pipeline + in tezt_job "flaky" ~__POS__ ~pipeline + ~needs: + [ + (Artifacts, job_build_tezt pipeline_type); + (Artifacts, Tezos_ci_jobs.Kernels.job_build_kernels); + ] ~description:"Run Etherlink Tezt tests tagged as flaky." ~allow_failure:Yes ~test_selection:(Tezos_ci_jobs.Tezt.tests_tag_selector [Has_tag "flaky"]) @@ -447,7 +484,7 @@ let register () = (Manual, job_build_evm_node_static Amd64 Test); (Manual, job_build_evm_node_static Arm64 Test); (Auto, job_lint_wasm_runtime); - (Auto, job_unit_tests); + (Auto, job_unit_tests Tezos_ci.Cache.Regular_pipeline); (* We rely on the fact that [Tezos_ci_pipelines.Code_verification.job_build_kernels] returns an equivalent job for [Before_merging] and [Merge_train]. *) (Auto, job_test_kernel); @@ -481,7 +518,7 @@ let register () = (Auto, job_build_evm_node_static Amd64 Test); (Auto, job_build_evm_node_static Arm64 Test); (Auto, job_lint_wasm_runtime); - (Auto, job_unit_tests); + (Auto, job_unit_tests Tezos_ci.Cache.Cache_pipeline); (Auto, job_test_kernel); (Auto, job_test_firehose); (Auto, job_test_evm_compatibility);