From 4ee06a856e19ddee54deee6b31d0c485052a20f4 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 12:30:04 +0100 Subject: [PATCH 01/12] CIAO: document changesets in [common.ml] --- ci/bin/common.ml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 23a55bc27e56..1591fb4c6cb5 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -265,6 +265,7 @@ let enable_kernels = (** {2 Changesets} *) +(* Only if octez source code has changed *) let changeset_octez = [ "src/**/*"; @@ -276,10 +277,13 @@ let changeset_octez = "tzt_reference_test_suite/**/*"; ] +(* Only if octez source code has changed, if the images has changed or + if kernels.mk changed. *) let changeset_octez_or_kernels = ["images/**/*"; "scripts/ci/**/*"; "kernels.mk"; "etherlink.mk"] @ changeset_octez +(* Only if documentation has changed *) let changeset_octez_docs = [ "scripts/**/*/"; -- GitLab From c90bda8af46d2f36577c31f538b85ab73271f030 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 14:39:23 +0100 Subject: [PATCH 02/12] CIAO: document [dependent] in [Code_verification.make_rules] --- ci/bin/code_verification.ml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 1e885b045d6b..70c69bb71eed 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -43,6 +43,10 @@ type code_verification_pipeline = Before_merging | Schedule_extended_test - automatic in scheduled pipelines; - conditional in [before_merging] pipelines. + If a job has non-optional dependencies, then [dependent] must be + set to [true] to ensure that we only run the job in case previous + jobs succeeded (setting [when: on_success]). + If [label], [changes] and [manual] are omitted, then rules will enable the job [On_success] in the [before_merging] pipeline. This is safe, but prefer specifying a [changes] clause -- GitLab From 4fac8401b94714326af3aecc76b5ff028cadc97b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 12:23:10 +0100 Subject: [PATCH 03/12] CIAO: include omitted [job_opam_prepare] in stage [packaging] --- ci/bin/code_verification.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 70c69bb71eed..2760d4c26dbd 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -717,7 +717,7 @@ let jobs pipeline_type = ~dependencies:(Dependent [Artifacts job_opam_prepare])) |> jobs_external_once ~path:"packaging/opam_package.yml" in - jobs_opam_packages + job_opam_prepare :: jobs_opam_packages in let test = (* check that ksy files are still up-to-date with octez *) -- GitLab From b37fd74243ab471d951eacdbf6f136a8ce52f4c8 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 16:37:49 +0100 Subject: [PATCH 04/12] CIAO: delete exclude of [oc.integration:compiler-rejections.yml] This file is no longer exist so it makes no sense to list it. --- ci/bin/main.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 03f85a4de398..d121f8f1602a 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -194,7 +194,6 @@ let () = | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" -- GitLab From 113f8522d1f68f473b4fa207be55d8a59425830f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 25 Mar 2024 13:04:47 +0100 Subject: [PATCH 05/12] CIAO: refactor, no need to for optional trigger in [Before_merging] --- ...c-x86_64-linux-binaries-before_merging.yml | 3 +- ...ld_x86_64-exp-dev-extra-before_merging.yml | 3 +- ...c.build_x86_64-released-before_merging.yml | 3 +- ...c.docker:rust-toolchain-before_merging.yml | 3 +- .../jobs/build/ocaml-check-before_merging.yml | 3 +- ...re.yml => opam:prepare-before_merging.yml} | 3 +- .../opam:prepare-scheduled_extended_test.yml | 43 +++++++++++++++++++ .gitlab/ci/jobs/test/commit_titles.yml | 3 +- .../test/kaitai_checks-before_merging.yml | 3 +- .../test/misc_opam_checks-before_merging.yml | 3 +- ...check_lift_limits_patch-before_merging.yml | 3 +- .../test/oc.misc_checks-before_merging.yml | 3 +- .../jobs/test/oc.semgrep-before_merging.yml | 3 +- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 5 +-- 16 files changed, 59 insertions(+), 29 deletions(-) rename .gitlab/ci/jobs/packaging/{opam:prepare.yml => opam:prepare-before_merging.yml} (97%) create mode 100644 .gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml index 6b9a9dcca0aa..b9df6eef26d4 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml @@ -17,8 +17,7 @@ oc.build:static-x86_64-linux-binaries: - tzt_reference_test_suite/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml index 6d416deebae4..2da0627a4e7c 100644 --- a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml @@ -17,8 +17,7 @@ oc.build_x86_64-exp-dev-extra: - tzt_reference_test_suite/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml index c4cbfb5ec6a8..58c4beeaf7e2 100644 --- a/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml @@ -17,8 +17,7 @@ oc.build_x86_64-released: - tzt_reference_test_suite/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml index 3576b95d3756..46b87cb64f7a 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml @@ -23,8 +23,7 @@ oc.docker:rust-toolchain: - when: manual allow_failure: true needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: [] script: diff --git a/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml index 59994e8289f3..fcd32e692684 100644 --- a/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml +++ b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml @@ -15,8 +15,7 @@ ocaml-check: - devtools/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/jobs/packaging/opam:prepare.yml b/.gitlab/ci/jobs/packaging/opam:prepare-before_merging.yml similarity index 97% rename from .gitlab/ci/jobs/packaging/opam:prepare.yml rename to .gitlab/ci/jobs/packaging/opam:prepare-before_merging.yml index 2711564ee7ac..0d431c5b3f86 100644 --- a/.gitlab/ci/jobs/packaging/opam:prepare.yml +++ b/.gitlab/ci/jobs/packaging/opam:prepare-before_merging.yml @@ -31,8 +31,7 @@ opam:prepare: start_in: 1 minute - when: never needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - eval $(opam env) diff --git a/.gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml b/.gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml new file mode 100644 index 000000000000..d90aeef15718 --- /dev/null +++ b/.gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml @@ -0,0 +1,43 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +opam:prepare: + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} + stage: packaging + tags: + - gcp + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: delayed + start_in: 1 minute + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--opam(?:$|,)/ + when: delayed + start_in: 1 minute + - changes: + - '**/dune' + - '**/dune.inc' + - '**/*.dune.inc' + - '**/dune-project' + - '**/dune-workspace' + - '**/*.opam' + - .gitlab/ci/jobs/packaging/opam:prepare.yml + - .gitlab/ci/jobs/packaging/opam_package.yml + - manifest/manifest.ml + - manifest/main.ml + - scripts/opam-prepare-repo.sh + - scripts/version.sh + if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: delayed + start_in: 1 minute + - when: never + dependencies: [] + before_script: + - eval $(opam env) + script: + - git init _opam-repo-for-release + - ./scripts/opam-prepare-repo.sh dev ./ ./_opam-repo-for-release + - git -C _opam-repo-for-release add packages + - git -C _opam-repo-for-release commit -m "tezos packages" + artifacts: + paths: + - _opam-repo-for-release/ diff --git a/.gitlab/ci/jobs/test/commit_titles.yml b/.gitlab/ci/jobs/test/commit_titles.yml index e2d913711e63..1034fc1273de 100644 --- a/.gitlab/ci/jobs/test/commit_titles.yml +++ b/.gitlab/ci/jobs/test/commit_titles.yml @@ -7,8 +7,7 @@ commit_titles: tags: - gcp needs: - - job: trigger - optional: true + - trigger dependencies: [] allow_failure: exit_codes: 65 diff --git a/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml index 34566285dcf0..75da9d25c2de 100644 --- a/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/kaitai_checks-before_merging.yml @@ -16,8 +16,7 @@ kaitai_checks: - .gitlab-ci.yml when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - . ./scripts/version.sh diff --git a/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml index 0ae8eaff2337..bec3bfb3da7d 100644 --- a/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/misc_opam_checks-before_merging.yml @@ -17,8 +17,7 @@ misc_opam_checks: - tzt_reference_test_suite/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - . ./scripts/version.sh diff --git a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml index 98eb7dcfad50..610cf568666b 100644 --- a/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.check_lift_limits_patch-before_merging.yml @@ -14,8 +14,7 @@ oc.check_lift_limits_patch: - .gitlab-ci.yml when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - . ./scripts/version.sh diff --git a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml index 1e3bef25efbe..51444abe06e2 100644 --- a/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml @@ -20,8 +20,7 @@ oc.misc_checks: - .gitlab/**/* when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml index 10121659decf..7a61020a0fb8 100644 --- a/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml +++ b/.gitlab/ci/jobs/test/oc.semgrep-before_merging.yml @@ -16,8 +16,7 @@ oc.semgrep: - .gitlab-ci.yml when: on_success needs: - - job: trigger - optional: true + - trigger dependencies: [] script: - echo "OCaml code linting. For information on how to reproduce locally, check out diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 7153c0f902c7..8cefd5593587 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -7,7 +7,7 @@ include: - .gitlab/ci/jobs/sanity/docker:hadolint-before_merging.yml # Stage: package - - .gitlab/ci/jobs/packaging/opam:prepare.yml + - .gitlab/ci/jobs/packaging/opam:prepare-before_merging.yml - .gitlab/ci/jobs/packaging/opam_package.yml - .gitlab/ci/jobs/packaging/debian_repository.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index f0ce579f8c03..204b8b435554 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -26,7 +26,7 @@ include: - .gitlab/ci/jobs/doc/documentation.yml # Stage: packaging - - .gitlab/ci/jobs/packaging/opam:prepare.yml + - .gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml - .gitlab/ci/jobs/packaging/opam_package.yml # Stage: test diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 2760d4c26dbd..e6253a97b199 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -433,8 +433,7 @@ let jobs pipeline_type = ] |> job_external in - (* TODO: the dependency on job_trigger does not have to be optional *) - ([job_trigger], Dependent [Optional job_trigger]) + ([job_trigger], Dependent [Job job_trigger]) in let sanity = let job_sanity_ci : tezos_job = @@ -708,7 +707,7 @@ let jobs pipeline_type = "git -C _opam-repo-for-release add packages"; "git -C _opam-repo-for-release commit -m \"tezos packages\""; ] - |> job_external_once + |> job_external_split in let (jobs_opam_packages : tezos_job list) = read_opam_packages -- GitLab From 540193829df7df344bf5588c9424b7e9c8c107ec Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 21 Mar 2024 16:58:41 +0100 Subject: [PATCH 06/12] CI: generate [oc.script:test-gen-genesis] --- ...script:test-gen-genesis-before_merging.yml | 26 +++++++++++++++++++ ...st-gen-genesis-scheduled_extended_test.yml | 16 ++++++++++++ .../jobs/test/oc.script:test-gen-genesis.yml | 12 --------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 14 ++++++++++ ci/bin/main.ml | 1 - 7 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml diff --git a/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml b/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml new file mode 100644 index 000000000000..494766065f7b --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml @@ -0,0 +1,26 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:test-gen-genesis: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - trigger + dependencies: [] + before_script: + - eval $(opam env) + - cd scripts/gen-genesis + script: + - dune build gen_genesis.exe diff --git a/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml new file mode 100644 index 000000000000..f420fc2678c5 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml @@ -0,0 +1,16 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:test-gen-genesis: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - eval $(opam env) + - cd scripts/gen-genesis + script: + - dune build gen_genesis.exe diff --git a/.gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml b/.gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml deleted file mode 100644 index d0f126ac8cc1..000000000000 --- a/.gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml +++ /dev/null @@ -1,12 +0,0 @@ -oc.script:test-gen-genesis: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - - .needs__trigger - stage: test - before_script: - - eval $(opam env) - - cd scripts/gen-genesis - script: - - dune build gen_genesis.exe diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 8cefd5593587..764cfa0fd6e8 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -40,7 +40,7 @@ include: - .gitlab/ci/jobs/test/oc.unit-before_merging.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-before_merging.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml - - .gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml + - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 204b8b435554..d79f424232c1 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -46,7 +46,7 @@ include: - .gitlab/ci/jobs/test/oc.unit-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml - - .gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml + - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e6253a97b199..ada4027ac53d 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -877,6 +877,19 @@ let jobs pipeline_type = ["dune build @runtest_rejections"] |> job_external_split in + let job_oc_script_test_gen_genesis : tezos_job = + job + ~__POS__ + ~name:"oc.script:test-gen-genesis" + ~stage:Stages.test + ~image:Images.runtime_build_dependencies + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + ~before_script: + (before_script ~eval_opam:true ["cd scripts/gen-genesis"]) + ["dune build gen_genesis.exe"] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -885,6 +898,7 @@ let jobs pipeline_type = job_misc_opam_checks; job_semgrep; job_oc_integration_compiler_rejections; + job_oc_script_test_gen_genesis; ] @ jobs_unit @ diff --git a/ci/bin/main.ml b/ci/bin/main.ml index d121f8f1602a..944f16db3fd3 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -196,7 +196,6 @@ let () = | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" - | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" -- GitLab From 062171c6ba34cd424780f9eb592e9be46cfe154a Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 25 Mar 2024 13:06:00 +0100 Subject: [PATCH 07/12] CI: generate [oc.script:snapshot_alpha_and_link] --- ...snapshot_alpha_and_link-before_merging.yml | 30 ++++++++++ ...alpha_and_link-scheduled_extended_test.yml | 17 ++++++ .../oc.script:snapshot_alpha_and_link.yml | 32 ----------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 57 +++++++++++++++++-- ci/bin/common.ml | 12 ++++ ci/bin/main.ml | 1 - 8 files changed, 114 insertions(+), 39 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml diff --git a/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml b/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml new file mode 100644 index 000000000000..13dfb9ad0cab --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml @@ -0,0 +1,30 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:snapshot_alpha_and_link: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/proto_alpha/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - scripts/snapshot_alpha_and_link.sh + - scripts/snapshot_alpha.sh + - scripts/user_activated_upgrade.sh + when: on_success + needs: + - job: trigger + - job: oc.build_x86_64-released + optional: true + - job: oc.build_x86_64-exp-dev-extra + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./.gitlab/ci/jobs/test/script:snapshot_alpha_and_link.sh diff --git a/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml new file mode 100644 index 000000000000..7ab5157597d6 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml @@ -0,0 +1,17 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:snapshot_alpha_and_link: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./.gitlab/ci/jobs/test/script:snapshot_alpha_and_link.sh diff --git a/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml b/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml deleted file mode 100644 index c2f9907a1a5f..000000000000 --- a/.gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Note: this job actually probably doesn't need the oc.build_x86_64 job -# to have finished, but we don't want to start before oc.build_x86_64 has finished either. -# However, when oc.build_x86_64-* don't exist, we don't need to wait for them. -oc.script:snapshot_alpha_and_link: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - stage: test - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - needs: - - job: trigger - optional: true - - job: "oc.build_x86_64-released" - optional: true - - job: "oc.build_x86_64-exp-dev-extra" - optional: true - script: - - ./.gitlab/ci/jobs/test/script:snapshot_alpha_and_link.sh - rules: - # We only need to run this if protocol Alpha or if the scripts changed. - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - src/proto_alpha/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - scripts/snapshot_alpha_and_link.sh - - scripts/snapshot_alpha.sh - - scripts/user_activated_upgrade.sh - when: on_success diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 764cfa0fd6e8..9e10c209f3fa 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -39,7 +39,7 @@ include: - .gitlab/ci/jobs/test/oc.semgrep-before_merging.yml - .gitlab/ci/jobs/test/oc.unit-before_merging.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-before_merging.yml - - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml + - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d79f424232c1..a2cfdaa041a0 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -45,7 +45,7 @@ include: - .gitlab/ci/jobs/test/oc.semgrep-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.unit-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml + - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index ada4027ac53d..3b1282284407 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -394,9 +394,13 @@ let jobs pipeline_type = in (* Stages *) (* All stages should be empty, as explained below, until the full pipeline is generated. *) - let trigger, dependencies_needs_trigger = + let trigger_stage, make_dependencies = match pipeline_type with - | Schedule_extended_test -> ([], Staged []) + | Schedule_extended_test -> + let make_dependencies ~before_merging:_ ~schedule_extended_test = + schedule_extended_test () + in + ([], make_dependencies) | Before_merging -> (* Define the [trigger] job @@ -433,7 +437,17 @@ let jobs pipeline_type = ] |> job_external in - ([job_trigger], Dependent [Job job_trigger]) + let make_dependencies ~before_merging ~schedule_extended_test:_ = + before_merging job_trigger + in + ([job_trigger], make_dependencies) + in + (* Short-cut for jobs that has no dependencies except [job_trigger] + on [Before_merging] pipelines. *) + let dependencies_needs_trigger = + make_dependencies + ~before_merging:(fun job_trigger -> Dependent [Job job_trigger]) + ~schedule_extended_test:(fun () -> Staged []) in let sanity = let job_sanity_ci : tezos_job = @@ -718,6 +732,22 @@ let jobs pipeline_type = in job_opam_prepare :: jobs_opam_packages in + (* Dependencies for jobs that should run immediately after jobs + [job_build_x86_64] in [Before_merging] if they are present + (otherwise, they run immediately after [job_trigger]). In + [Scheduled_extended_test] we are not in a hurry and we let them + be [Staged []]. *) + let order_after_build = + make_dependencies + ~before_merging:(fun job_trigger -> + Dependent + (Job job_trigger + :: [ + Optional job_build_x86_64_release; + Optional job_build_x86_64_exp_dev_extra; + ])) + ~schedule_extended_test:(fun () -> Staged []) + in let test = (* check that ksy files are still up-to-date with octez *) let job_kaitai_checks : tezos_job = @@ -890,6 +920,24 @@ let jobs pipeline_type = ["dune build gen_genesis.exe"] |> job_external_split in + let job_oc_script_snapshot_alpha_and_link : tezos_job = + job + ~__POS__ + ~name:"oc.script:snapshot_alpha_and_link" + ~stage:Stages.test + ~image:Images.runtime_build_dependencies + ~dependencies:order_after_build + (* Since the above dependencies are only for ordering, we do not set [dependent] *) + ~rules:(make_rules ~changes:changeset_script_snapshot_alpha_and_link ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ["./.gitlab/ci/jobs/test/script:snapshot_alpha_and_link.sh"] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -899,6 +947,7 @@ let jobs pipeline_type = job_semgrep; job_oc_integration_compiler_rejections; job_oc_script_test_gen_genesis; + job_oc_script_snapshot_alpha_and_link; ] @ jobs_unit @ @@ -989,5 +1038,5 @@ let jobs pipeline_type = (using {!job_external} or {!jobs_external}) and included by hand in the files [.gitlab/ci/pipelines/before_merging.yml] and [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) - ignore (trigger @ sanity @ build @ packaging @ test @ doc @ manual) ; + ignore (trigger_stage @ sanity @ build @ packaging @ test @ doc @ manual) ; [] diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 1591fb4c6cb5..2e1722df0a6b 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -386,6 +386,18 @@ let changeset_semgrep_files = ".gitlab-ci.yml"; ] +(* We only need to run the [oc.script:snapshot_alpha_and_link] job if + protocol Alpha or if the scripts changed. *) +let changeset_script_snapshot_alpha_and_link = + [ + "src/proto_alpha/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + "scripts/snapshot_alpha_and_link.sh"; + "scripts/snapshot_alpha.sh"; + "scripts/user_activated_upgrade.sh"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 944f16db3fd3..8a8ad777f39b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -195,7 +195,6 @@ let () = | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" - | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" -- GitLab From a054ab854adee3db20bb611b0decbec9269878b7 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 25 Mar 2024 11:45:16 +0100 Subject: [PATCH 08/12] CI: refactor, align file and job name of [test_octez_release_versions] --- .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml index 2314239e5c7a..604f390e8647 100644 --- a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml +++ b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml @@ -1,4 +1,4 @@ -oc.script:test_release_versions: +oc.script:test_octez_release_versions: extends: .test_template before_script: - ./scripts/ci/take_ownership.sh -- GitLab From 9a47a9f546c6da5d4203db2a94b3ce0b2fda3764 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 14:40:49 +0100 Subject: [PATCH 09/12] CI: generate [oc.script:test_octez_release_versions] --- ..._octez_release_versions-before_merging.yml | 28 +++++++++++++++++++ ...lease_versions-scheduled_extended_test.yml | 20 +++++++++++++ .../oc.script:test_octez_release_versions.yml | 8 ------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 21 ++++++++++++++ ci/bin/main.ml | 1 - 7 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml diff --git a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml new file mode 100644 index 000000000000..d76c03d4d61b --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml @@ -0,0 +1,28 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:test_octez_release_versions: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/test_octez_release_version.sh diff --git a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml new file mode 100644 index 000000000000..4778be5f1245 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml @@ -0,0 +1,20 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:test_octez_release_versions: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + needs: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/test_octez_release_version.sh diff --git a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml b/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml deleted file mode 100644 index 604f390e8647..000000000000 --- a/.gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml +++ /dev/null @@ -1,8 +0,0 @@ -oc.script:test_octez_release_versions: - extends: .test_template - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - ./scripts/test_octez_release_version.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 9e10c209f3fa..55f59e0fcc0a 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -41,7 +41,7 @@ include: - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-before_merging.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml - - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml + - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml - .gitlab/ci/jobs/test/install_octez.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index a2cfdaa041a0..f1b4c69d1e66 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -47,7 +47,7 @@ include: - .gitlab/ci/jobs/test/oc.integration:compiler-rejections-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml + - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml - .gitlab/ci/jobs/test/tezt.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 3b1282284407..e85efae5494a 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -938,6 +938,26 @@ let jobs pipeline_type = ["./.gitlab/ci/jobs/test/script:snapshot_alpha_and_link.sh"] |> job_external_split in + let job_oc_script_test_release_versions : tezos_job = + job + ~__POS__ + ~name:"oc.script:test_octez_release_versions" + ~stage:Stages.test + ~image:Images.runtime_build_dependencies + ~dependencies: + (Dependent + [Job job_build_x86_64_release; Job job_build_x86_64_exp_dev_extra]) + (* Since the above dependencies are only for ordering, we do not set [dependent] *) + ~rules:(make_rules ~changes:changeset_octez ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ["./scripts/test_octez_release_version.sh"] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -948,6 +968,7 @@ let jobs pipeline_type = job_oc_integration_compiler_rejections; job_oc_script_test_gen_genesis; job_oc_script_snapshot_alpha_and_link; + job_oc_script_test_release_versions; ] @ jobs_unit @ diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 8a8ad777f39b..d1a8dac57174 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -195,7 +195,6 @@ let () = | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" - | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml" -- GitLab From 81e56feaa1ba9ee3d7bb42d9275643d92d848cec Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 14:52:27 +0100 Subject: [PATCH 10/12] CI: generate [oc.script:b58_prefix] --- .../oc.script:b58_prefix-before_merging.yml | 25 +++++++++++++++++++ ...ipt:b58_prefix-scheduled_extended_test.yml | 18 +++++++++++++ .gitlab/ci/jobs/test/oc.script:b58_prefix.yml | 24 ------------------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 21 ++++++++++++++++ ci/bin/common.ml | 8 ++++++ ci/bin/main.ml | 1 - 8 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.script:b58_prefix.yml diff --git a/.gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml b/.gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml new file mode 100644 index 000000000000..79adc98add53 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml @@ -0,0 +1,25 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:b58_prefix: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - scripts/b58_prefix/b58_prefix.py + - scripts/b58_prefix/test_b58_prefix.py + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - trigger + dependencies: [] + before_script: + - . ./scripts/version.sh + - . $HOME/.venv/bin/activate + script: + - poetry run pylint scripts/b58_prefix/b58_prefix.py --disable=missing-docstring + --disable=invalid-name + - poetry run pytest scripts/b58_prefix/test_b58_prefix.py diff --git a/.gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml new file mode 100644 index 000000000000..643f15e8276d --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml @@ -0,0 +1,18 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.script:b58_prefix: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - . ./scripts/version.sh + - . $HOME/.venv/bin/activate + script: + - poetry run pylint scripts/b58_prefix/b58_prefix.py --disable=missing-docstring + --disable=invalid-name + - poetry run pytest scripts/b58_prefix/test_b58_prefix.py diff --git a/.gitlab/ci/jobs/test/oc.script:b58_prefix.yml b/.gitlab/ci/jobs/test/oc.script:b58_prefix.yml deleted file mode 100644 index 1d3523f1f77b..000000000000 --- a/.gitlab/ci/jobs/test/oc.script:b58_prefix.yml +++ /dev/null @@ -1,24 +0,0 @@ -oc.script:b58_prefix: - # Can be changed to a python image, but using the build docker image to keep - # in sync with the python version used for the tests - extends: - - .default_settings_template - # Requires Python - - .image_template__runtime_build_test_dependencies - - .needs__trigger - rules: - - changes: - - scripts/b58_prefix/b58_prefix.py - - scripts/b58_prefix/test_b58_prefix.py - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - stage: test - before_script: - - . ./scripts/version.sh - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - script: - - poetry run pylint scripts/b58_prefix/b58_prefix.py --disable=missing-docstring --disable=invalid-name - - poetry run pytest scripts/b58_prefix/test_b58_prefix.py diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 55f59e0fcc0a..6657d2ca888d 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -42,7 +42,7 @@ include: - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml - - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml + - .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml - .gitlab/ci/jobs/test/install_octez.yml - .gitlab/ci/jobs/test/tezt.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index f1b4c69d1e66..3da1a502a9d1 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -48,7 +48,7 @@ include: - .gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.script:b58_prefix.yml + - .gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml - .gitlab/ci/jobs/test/tezt.yml - .gitlab/ci/jobs/test/test_kernels.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e85efae5494a..002cea100de6 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -958,6 +958,26 @@ let jobs pipeline_type = ["./scripts/test_octez_release_version.sh"] |> job_external_split in + let job_oc_script_b58_prefix = + job + ~__POS__ + ~name:"oc.script:b58_prefix" + ~stage:Stages.test + (* Requires Python. Can be changed to a python image, but using + the build docker image to keep in sync with the python + version used for the tests *) + ~image:Images.runtime_build_test_dependencies + ~rules:(make_rules ~changes:changeset_script_b58_prefix ()) + ~dependencies:dependencies_needs_trigger + ~before_script: + (before_script ~source_version:true ~init_python_venv:true []) + [ + "poetry run pylint scripts/b58_prefix/b58_prefix.py \ + --disable=missing-docstring --disable=invalid-name"; + "poetry run pytest scripts/b58_prefix/test_b58_prefix.py"; + ] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -969,6 +989,7 @@ let jobs pipeline_type = job_oc_script_test_gen_genesis; job_oc_script_snapshot_alpha_and_link; job_oc_script_test_release_versions; + job_oc_script_b58_prefix; ] @ jobs_unit @ diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 2e1722df0a6b..e0506d996167 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -398,6 +398,14 @@ let changeset_script_snapshot_alpha_and_link = "scripts/user_activated_upgrade.sh"; ] +let changeset_script_b58_prefix = + [ + "scripts/b58_prefix/b58_prefix.py"; + "scripts/b58_prefix/test_b58_prefix.py"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index d1a8dac57174..acd2ce1fcf1c 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -194,7 +194,6 @@ let () = | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml" -- GitLab From 6c7100c4fe25546f1800523e12fde0cb8c7b3288 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 14:54:45 +0100 Subject: [PATCH 11/12] CI: generate [oc.test-liquidity-baking-scripts] --- ...iquidity-baking-scripts-before_merging.yml | 27 +++++++++++++++++++ ...baking-scripts-scheduled_extended_test.yml | 21 +++++++++++++++ .../test/oc.test-liquidity-baking-scripts.yml | 17 ------------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 22 +++++++++++++++ ci/bin/common.ml | 9 +++++++ ci/bin/main.ml | 1 - 8 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml diff --git a/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml b/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml new file mode 100644 index 000000000000..c3ac1da85667 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.test-liquidity-baking-scripts: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - scripts/ci/test_liquidity_baking_scripts.sh + - scripts/check-liquidity-baking-scripts.sh + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + dependencies: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/test_liquidity_baking_scripts.sh diff --git a/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml new file mode 100644 index 000000000000..b90e396e8ff1 --- /dev/null +++ b/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml @@ -0,0 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.test-liquidity-baking-scripts: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: on_success + needs: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + dependencies: + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/test_liquidity_baking_scripts.sh diff --git a/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml b/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml deleted file mode 100644 index 8f14b6cff389..000000000000 --- a/.gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -oc.test-liquidity-baking-scripts: - extends: - - .test_template - rules: - - changes: - - src/**/* - - scripts/ci/test_liquidity_baking_scripts.sh - - scripts/check-liquidity-baking-scripts.sh - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - dependencies: - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - script: ./scripts/ci/test_liquidity_baking_scripts.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 6657d2ca888d..61be8727e210 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -43,7 +43,7 @@ include: - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-before_merging.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml - - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml + - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml - .gitlab/ci/jobs/test/install_octez.yml - .gitlab/ci/jobs/test/tezt.yml - .gitlab/ci/jobs/test/tezt-slow-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 3da1a502a9d1..c95662e5e55d 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -49,7 +49,7 @@ include: - .gitlab/ci/jobs/test/oc.script:test-gen-genesis-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml + - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml - .gitlab/ci/jobs/test/tezt.yml - .gitlab/ci/jobs/test/test_kernels.yml - .gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 002cea100de6..5c2f3799bd23 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -978,6 +978,27 @@ let jobs pipeline_type = ] |> job_external_split in + let job_oc_test_liquidity_baking_scripts : tezos_job = + job + ~__POS__ + ~name:"oc.test-liquidity-baking-scripts" + ~stage:Stages.test + ~image:Images.runtime_build_dependencies + ~dependencies: + (Dependent + [ + Artifacts job_build_x86_64_release; + Artifacts job_build_x86_64_exp_dev_extra; + ]) + ~rules: + (make_rules + ~dependent:true + ~changes:changeset_test_liquidity_baking_scripts + ()) + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + ["./scripts/ci/test_liquidity_baking_scripts.sh"] + |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -990,6 +1011,7 @@ let jobs pipeline_type = job_oc_script_snapshot_alpha_and_link; job_oc_script_test_release_versions; job_oc_script_b58_prefix; + job_oc_test_liquidity_baking_scripts; ] @ jobs_unit @ diff --git a/ci/bin/common.ml b/ci/bin/common.ml index e0506d996167..53f4be3ade13 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -406,6 +406,15 @@ let changeset_script_b58_prefix = ".gitlab-ci.yml"; ] +let changeset_test_liquidity_baking_scripts = + [ + "src/**/*"; + "scripts/ci/test_liquidity_baking_scripts.sh"; + "scripts/check-liquidity-baking-scripts.sh"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index acd2ce1fcf1c..4e9e33ed90b3 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -194,7 +194,6 @@ let () = | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/jobs/test/install_octez.yml" - | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml" | ".gitlab/ci/jobs/test/test_evm_compatibility.yml" -- GitLab From fac80261d5864ad878dbb89a1a149823e44c512d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 15:04:59 +0100 Subject: [PATCH 12/12] CI: generate [install_octez.yml] --- .gitlab/ci/jobs/shared/images.yml | 12 ++ .../test/install_octez-before_merging.yml | 187 ++++++++++++++++++ .../install_octez-scheduled_extended_test.yml | 121 ++++++++++++ .gitlab/ci/jobs/test/install_octez.yml | 123 ------------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 122 +++++++++++- ci/bin/common.ml | 27 +++ ci/bin/main.ml | 1 - 9 files changed, 470 insertions(+), 127 deletions(-) create mode 100644 .gitlab/ci/jobs/test/install_octez-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/install_octez.yml diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 3c9fde318195..43cdde0eb3b7 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -11,10 +11,18 @@ image: debian:bookworm .image_template__docker: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 +.image_template__fedora_37: + image: fedora:37 .image_template__fedora_39: image: fedora:39 .image_template__hadolint: image: hadolint/hadolint:2.9.3-debian +.image_template__opam_debian_bullseye: + image: ocaml/opam:debian-11 +.image_template__opam_ubuntu_focal: + image: ocaml/opam:ubuntu-20.04 +.image_template__opam_ubuntu_mantic: + image: ocaml/opam:ubuntu-23.10 .image_template__runtime_build_dependencies: image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} .image_template__runtime_build_test_dependencies: @@ -27,3 +35,7 @@ image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} .image_template__semgrep_agent: image: returntocorp/semgrep-agent:sha-c6cd7cf +.image_template__ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable +.image_template__ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable diff --git a/.gitlab/ci/jobs/test/install_octez-before_merging.yml b/.gitlab/ci/jobs/test/install_octez-before_merging.yml new file mode 100644 index 000000000000..0ae2d2999fc9 --- /dev/null +++ b/.gitlab/ci/jobs/test/install_octez-before_merging.yml @@ -0,0 +1,187 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.install_bin_fedora_37: + image: fedora:37 + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/introduction/install-bin-fedora.sh + +oc.install_bin_rc_fedora_37: + image: fedora:37 + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/introduction/install-bin-fedora.sh rc + +oc.install_bin_ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh + +oc.install_bin_ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh + +oc.install_bin_rc_ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh rc + +oc.install_bin_rc_ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh rc + +oc.install_opam_focal: + image: ocaml/opam:ubuntu-20.04 + stage: test + tags: + - gcp + rules: + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-opam.sh + variables: + OPAMJOBS: "4" + +oc.compile_release_sources_bullseye: + image: ocaml/opam:debian-11 + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh tezos/tezos latest-release + +oc.compile_sources_bullseye: + image: ocaml/opam:debian-11 + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.compile_sources_mantic: + image: ocaml/opam:ubuntu-23.10 + stage: test + tags: + - gcp + rules: + - changes: + - docs/introduction/install*.sh + - docs/introduction/compile*.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml new file mode 100644 index 000000000000..c2ee6c4a06c8 --- /dev/null +++ b/.gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml @@ -0,0 +1,121 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.install_bin_fedora_37: + image: fedora:37 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/introduction/install-bin-fedora.sh + +oc.install_bin_rc_fedora_37: + image: fedora:37 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/introduction/install-bin-fedora.sh rc + +oc.install_bin_ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh + +oc.install_bin_ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh + +oc.install_bin_rc_ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh rc + +oc.install_bin_rc_ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-bin-ubuntu.sh rc + +oc.install_opam_focal: + image: ocaml/opam:ubuntu-20.04 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + script: + - ./docs/introduction/install-opam.sh + variables: + OPAMJOBS: "4" + +oc.compile_release_sources_bullseye: + image: ocaml/opam:debian-11 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh tezos/tezos latest-release + +oc.compile_sources_bullseye: + image: ocaml/opam:debian-11 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.compile_sources_mantic: + image: ocaml/opam:ubuntu-23.10 + stage: test + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/jobs/test/install_octez.yml b/.gitlab/ci/jobs/test/install_octez.yml deleted file mode 100644 index b9564a242e27..000000000000 --- a/.gitlab/ci/jobs/test/install_octez.yml +++ /dev/null @@ -1,123 +0,0 @@ -.oc.base-test-install-scripts-template: - stage: test - extends: - - .needs__trigger - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - docs/introduction/install*.sh - - docs/introduction/compile*.sh - when: on_success - # Run when there is label on the merge request - - when: manual - allow_failure: true - -.oc.install_bin_ubuntu_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - script: - - ./docs/introduction/install-bin-ubuntu.sh - # These jobs currently fail because the last rc packages can't be installed anymore. - # See https://gitlab.com/tezos/tezos/-/issues/6902. - # TODO: https://gitlab.com/tezos/tezos/-/issues/6915 - # This should be removed after the next release candidate. - allow_failure: true - -oc.install_bin_ubuntu_focal: - image: public.ecr.aws/lts/ubuntu:20.04_stable - extends: .oc.install_bin_ubuntu_template - -oc.install_bin_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable - extends: .oc.install_bin_ubuntu_template - -.oc.install_bin_rc_ubuntu_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - script: - - ./docs/introduction/install-bin-ubuntu.sh rc - # These jobs currently fail because the last rc packages can't be installed anymore. - # See https://gitlab.com/tezos/tezos/-/issues/6902. - # TODO: https://gitlab.com/tezos/tezos/-/issues/6915 - # This should be removed after the next release candidate. - allow_failure: true - -oc.install_bin_rc_ubuntu_focal: - image: public.ecr.aws/lts/ubuntu:20.04_stable - extends: .oc.install_bin_rc_ubuntu_template - -oc.install_bin_rc_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable - extends: .oc.install_bin_rc_ubuntu_template - -.oc.install_bin_fedora_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - script: - - ./docs/introduction/install-bin-fedora.sh - -oc.install_bin_fedora_37: - image: fedora:37 - extends: .oc.install_bin_fedora_template - -.oc.install_bin_rc_fedora_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - script: - - ./docs/introduction/install-bin-fedora.sh rc - -oc.install_bin_rc_fedora_37: - image: fedora:37 - extends: .oc.install_bin_rc_fedora_template - -.oc.install_opam_ubuntu_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - variables: - # The default behavior of opam is to use `nproc` to determine its level of - # parallelism. This returns the number of CPU of the "host" CI runner - # instead of the number of cores a single CI job can reasonably use. - OPAMJOBS: "4" - script: - - ./docs/introduction/install-opam.sh - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - allow_failure: true - # temporarily disable until these jobs are optimized - - when: manual - allow_failure: true - -oc.install_opam_focal: - image: ocaml/opam:ubuntu-20.04 - extends: .oc.install_opam_ubuntu_template - -.oc.compile_sources_template: - extends: - - .default_settings_template - - .oc.base-test-install-scripts-template - -oc.compile_release_sources_bullseye: - image: ocaml/opam:debian-11 - extends: .oc.compile_sources_template - script: - - ./docs/introduction/compile-sources.sh tezos/tezos latest-release - -oc.compile_sources_bullseye: - image: ocaml/opam:debian-11 - extends: .oc.compile_sources_template - script: - - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - -oc.compile_sources_mantic: - image: ocaml/opam:ubuntu-23.10 - extends: .oc.compile_sources_template - script: - - ./docs/introduction/compile-sources.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 61be8727e210..651c5a601548 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -44,7 +44,7 @@ include: - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-before_merging.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml - - .gitlab/ci/jobs/test/install_octez.yml + - .gitlab/ci/jobs/test/install_octez-before_merging.yml - .gitlab/ci/jobs/test/tezt.yml - .gitlab/ci/jobs/test/tezt-slow-before_merging.yml - .gitlab/ci/jobs/test/tezt-flaky-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index c95662e5e55d..d646d102053c 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -30,7 +30,7 @@ include: - .gitlab/ci/jobs/packaging/opam_package.yml # Stage: test - - .gitlab/ci/jobs/test/install_octez.yml + - .gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml # Flaky tezts - .gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml # Slow tezts diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 5c2f3799bd23..6463d2aad3e6 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -351,6 +351,13 @@ let jobs_unit_tests ~job_build_x86_64_release ~job_build_x86_64_exp_dev_extra oc_unit_protocol_compiles; ] +type install_octez_distribution = Ubuntu_focal | Ubuntu_jammy | Fedora_37 + +let image_of_distribution = function + | Ubuntu_focal -> Images.ubuntu_focal + | Ubuntu_jammy -> Images.ubuntu_jammy + | Fedora_37 -> Images.fedora_37 + (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) let jobs pipeline_type = @@ -999,6 +1006,119 @@ let jobs pipeline_type = ["./scripts/ci/test_liquidity_baking_scripts.sh"] |> job_external_split in + (* The set of installation test jobs *) + let jobs_install_octez : tezos_job list = + let changeset_install_jobs = + ["docs/introduction/install*.sh"; "docs/introduction/compile*.sh"] + in + let install_octez_rules = + make_rules ~changes:changeset_install_jobs ~manual:true () + in + let job_install_bin ~__POS__ ~name ?allow_failure ?(rc = false) + distribution = + let distribution_string = + match distribution with + | Ubuntu_focal | Ubuntu_jammy -> "ubuntu" + | Fedora_37 -> "fedora" + in + let script = + sf "./docs/introduction/install-bin-%s.sh" distribution_string + ^ if rc then " rc" else "" + in + job + ?allow_failure + ~__POS__ + ~name + ~image:(image_of_distribution distribution) + ~dependencies:dependencies_needs_trigger + ~rules:install_octez_rules + ~stage:Stages.test + [script] + in + let job_install_opam_focal : tezos_job = + job + ~__POS__ + ~name:"oc.install_opam_focal" + ~image:Images.opam_ubuntu_focal + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~manual:true ()) + ~allow_failure:Yes + ~stage:Stages.test + (* The default behavior of opam is to use `nproc` to determine its level of + parallelism. This returns the number of CPU of the "host" CI runner + instead of the number of cores a single CI job can reasonably use. *) + ~variables:[("OPAMJOBS", "4")] + ["./docs/introduction/install-opam.sh"] + in + let job_compile_sources ~__POS__ ~name ~image ~project ~branch = + job + ~__POS__ + ~name + ~image + ~dependencies:dependencies_needs_trigger + ~rules:install_octez_rules + ~stage:Stages.test + [sf "./docs/introduction/compile-sources.sh %s %s" project branch] + in + [ + (* Test installing binary / binary RC distributions in all distributions *) + job_install_bin ~__POS__ ~name:"oc.install_bin_fedora_37" Fedora_37; + job_install_bin + ~__POS__ + ~name:"oc.install_bin_rc_fedora_37" + ~rc:true + Fedora_37; + (* The Ubuntu jobs currently fail because the last rc packages can't be installed anymore. + See https://gitlab.com/tezos/tezos/-/issues/6902. + TODO: https://gitlab.com/tezos/tezos/-/issues/6915 + This should be removed after the next release candidate. *) + job_install_bin + ~__POS__ + ~name:"oc.install_bin_ubuntu_focal" + ~allow_failure:Yes + Ubuntu_focal; + job_install_bin + ~__POS__ + ~name:"oc.install_bin_ubuntu_jammy" + ~allow_failure:Yes + Ubuntu_jammy; + job_install_bin + ~__POS__ + ~name:"oc.install_bin_rc_ubuntu_focal" + ~allow_failure:Yes + ~rc:true + Ubuntu_focal; + job_install_bin + ~__POS__ + ~name:"oc.install_bin_rc_ubuntu_jammy" + ~allow_failure:Yes + ~rc:true + Ubuntu_jammy; + (* Test installing through opam *) + job_install_opam_focal; + (* Test compiling the [latest-release] branch on Bullseye *) + job_compile_sources + ~__POS__ + ~name:"oc.compile_release_sources_bullseye" + ~image:Images.opam_debian_bullseye + ~project:"tezos/tezos" + ~branch:"latest-release"; + (* Test compiling the [master] branch on Bullseye *) + job_compile_sources + ~__POS__ + ~name:"oc.compile_sources_bullseye" + ~image:Images.opam_debian_bullseye + ~project:"${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos}" + ~branch:"${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master}"; + job_compile_sources + ~__POS__ + ~name:"oc.compile_sources_mantic" + ~image:Images.opam_ubuntu_mantic + ~project:"${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos}" + ~branch:"${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master}"; + ] + |> jobs_external_split ~path:"test/install_octez" + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -1013,7 +1133,7 @@ let jobs pipeline_type = job_oc_script_b58_prefix; job_oc_test_liquidity_baking_scripts; ] - @ jobs_unit + @ jobs_unit @ jobs_install_octez @ match pipeline_type with | Before_merging -> diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 53f4be3ade13..5abfd2be1f7d 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -145,8 +145,35 @@ module Images = struct let debian_bookworm = Image.register ~name:"debian_bookworm" ~image_path:"debian:bookworm" + let ubuntu_focal = + Image.register + ~name:"ubuntu_focal" + ~image_path:"public.ecr.aws/lts/ubuntu:20.04_stable" + + let ubuntu_jammy = + Image.register + ~name:"ubuntu_jammy" + ~image_path:"public.ecr.aws/lts/ubuntu:22.04_stable" + + let fedora_37 = Image.register ~name:"fedora_37" ~image_path:"fedora:37" + let fedora_39 = Image.register ~name:"fedora_39" ~image_path:"fedora:39" + let opam_ubuntu_focal = + Image.register + ~name:"opam_ubuntu_focal" + ~image_path:"ocaml/opam:ubuntu-20.04" + + let opam_ubuntu_mantic = + Image.register + ~name:"opam_ubuntu_mantic" + ~image_path:"ocaml/opam:ubuntu-23.10" + + let opam_debian_bullseye = + Image.register + ~name:"opam_debian_bullseye" + ~image_path:"ocaml/opam:debian-11" + let ci_release = Image.register ~name:"ci_release" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 4e9e33ed90b3..c1d817a1505e 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -193,7 +193,6 @@ let () = | ".gitlab/ci/jobs/packaging/debian_repository.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" - | ".gitlab/ci/jobs/test/install_octez.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" | ".gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml" | ".gitlab/ci/jobs/test/test_evm_compatibility.yml" -- GitLab