diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml similarity index 100% rename from .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml rename to .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml similarity index 66% rename from .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml rename to .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml index f1a171f820f6f73be98af6da376e44897decec8e..7605ead2f3de47049042664b195ec1586915d8d5 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml @@ -1,21 +1,13 @@ # This file was automatically generated, do not edit. # Edit file ci/bin/main.ml instead. -oc.build:static-arm64-linux-binaries: +oc.build:static-x86_64-linux-binaries: image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} stage: build tags: - - gcp_arm64 + - gcp rules: - - changes: - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success + - when: always dependencies: [] before_script: - ./scripts/ci/take_ownership.sh @@ -23,7 +15,7 @@ oc.build:static-arm64-linux-binaries: script: - ./scripts/ci/build_static_binaries.sh variables: - ARCH: arm64 + ARCH: x86_64 EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables artifacts: paths: diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml similarity index 85% rename from .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml rename to .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml index c6f4f4db5b28f2544a1ced9b10b6c6bb1bf23642..cfd8cd2cdc26f7bb8c01fa51373807bd84507f7e 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml @@ -7,15 +7,9 @@ oc.build_arm64-exp-dev-extra: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..9679753d8f4c19776c5103104ba20b622191f5b1 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml @@ -0,0 +1,32 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_arm64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml similarity index 83% rename from .gitlab/ci/jobs/build/oc.build_arm64-released.yml rename to .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml index 287950d42cd9d18645553e1c7f882d2876b77dea..90bce029d120728fbfd333ac5655925d5f182d1a 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml @@ -7,15 +7,9 @@ oc.build_arm64-released: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..a5b1e85df907cf151ae2e81c753cb958aba4bf8d --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml @@ -0,0 +1,30 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_arm64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml similarity index 94% rename from .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml rename to .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml index ec72da648d206c4eca450e572575b5a5a096723c..5385d46a70a8087c9909f779c0c9afb7f5d3e87f 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml @@ -6,6 +6,8 @@ oc.docker:rust-toolchain: stage: build tags: - gcp + rules: + - when: always dependencies: [] before_script: [] script: diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 9f618faa7825eec5c21af160b322d98c6a5c39cd..56c722216e852543b68a13f725035080e40f371c 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -15,9 +15,9 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml - .gitlab/ci/jobs/build/ocaml-check.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml + - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml + - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml + - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml - .gitlab/ci/jobs/build/oc.build_kernels.yml diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index ba0a816899d214c76bab0ff7160d553b4b728e27..b6551a0503bfd366d95e0752844da20e5aded2bc 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -28,9 +28,6 @@ oc.build:static-x86_64-linux-binaries: - gcp rules: - when: always - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh @@ -70,15 +67,7 @@ oc.build_arm64-released: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: @@ -107,15 +96,7 @@ oc.build_arm64-exp-dev-extra: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index b5d49fabd4e1a7729a16f7530988a0fc7b476594..455d509283d81e727466645254704fca7c8ccebd 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 4a618b16e510805a77da75026cce194e25fc927c..d2f5712f20cc7dccdfb7d9a9d7ecf7cf098a335c 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 0f2740ec529a9a7dae51b6919a95775223fdfe90..693ed5a322a2bfe48ff2253c6496b8dc8cb1bee1 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index af5307de2fe79c1432fd4e9160000777e0ad9c81..7aacdea83d7ebaeec2b699d5b7ffa26285b5a0af 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 127ee6d9614987ca22c075504838c3d50bb6fbba..fd100658ac88bc85743ac7683f17ed00bf2e7e75 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 068ad27bc242a83d3d7774f5f0a91cb395778480..e81624b5c027cd9bf88f63727317e12916cf1142 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -4,15 +4,15 @@ include: - .gitlab/ci/jobs/sanity/docker:hadolint-schedule_extended_test.yml # Stage: build - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml + - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml + - .gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml # Check that binary packages build - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml - .gitlab/ci/jobs/build/select_tezts.yml # These jobs are necessary to run flaky tezts - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml + - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml @@ -22,7 +22,7 @@ include: # These jobs may not have been run in before_merging pipeline # because of absence of certain changes - .gitlab/ci/jobs/build/ocaml-check.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml + - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml - .gitlab/ci/jobs/doc/documentation.yml # Stage: packaging diff --git a/ci/bin/main.ml b/ci/bin/main.ml index b1b2bb135b8225067598d8473bfa740bedb279ae..e4b9c0b4194cc80fd6e1c4ab9cc5fea1e96fb1cb 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -273,41 +273,6 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () -(* Define the [trigger] job - - §1: The purpose of this job is to launch the CI manually in certain cases. - The objective is not to run computing when it is not - necessary and the decision to do so belongs to the developer - - §2: We also perform some fast sanity checks. *) -let job_trigger = - job - ~__POS__ - ~image:Images.alpine - ~stage:Stages.trigger - ~allow_failure:No - ~rules: - [ - job_rule - ~if_:(If.not Rules.assigned_to_marge_bot) - ~allow_failure:No - ~when_:Manual - (); - job_rule ~when_:Always (); - ] - ~timeout:(Minutes 10) - ~name:"trigger" - [ - "echo 'Trigger pipeline!'"; - (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and - [scripts/version.sh]'s [opam_repository_tag] are the same. *) - "./scripts/ci/check_opam_repository_tag.sh"; - (* Check that the Alpine version of the trigger job's image - corresponds to the value in scripts/version.sh. *) - "./scripts/ci/check_alpine_version.sh"; - ] - |> job_external - (** Helper to create jobs that uses the docker deamon. It: @@ -356,8 +321,8 @@ let job_docker_promote_to_latest ~ci_docker_hub : tezos_job = (no need to test that we pass the -static flag twice) - released variants exist, that are used in release tag pipelines (they do not build experimental executables) *) -let job_build_static_binaries ~__POS__ ~arch ?(release = false) - ?(needs_trigger = false) ?rules () : Tezos_ci.tezos_job = +let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules + ?dependencies () : Tezos_ci.tezos_job = let arch_string = match arch with Tezos_ci.Amd64 -> "x86_64" | Arm64 -> "arm64" in @@ -371,16 +336,9 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) "script-inputs/released-executables" ^ if not release then " script-inputs/experimental-executables" else "" in - let dependencies = - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration of - pipelines. So when requested through [needs_trigger] we start - this job as early as possible, without waiting for - sanity_ci. *) - if needs_trigger then Dependent [Optional job_trigger] else Staged [] - in job ?rules + ?dependencies ~__POS__ ~stage:Stages.build ~arch @@ -388,29 +346,9 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ~image:Images.runtime_build_dependencies ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) ~variables:[("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] - ~dependencies ~artifacts ["./scripts/ci/build_static_binaries.sh"] -let rules_static_build_other = [job_rule ~changes:changeset_octez ()] - -let _job_static_arm64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_other - () - |> job_external ~filename_suffix:"experimental" - -let _job_static_x86_64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - ~needs_trigger:true - ~rules:rules_static_build_other - () - |> job_external ~filename_suffix:"experimental" - let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = job_docker_authenticated ?rules @@ -426,21 +364,6 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = []) ["./scripts/ci/docker_rust_toolchain_build.sh"] -let _job_docker_rust_toolchain_before_merging = - job_docker_rust_toolchain - ~__POS__ - ~dependencies:(Dependent [Optional job_trigger]) - ~rules: - [ - job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); - job_rule ~when_:Manual (); - ] - () - |> job_external ~filename_suffix:"before_merging" - -let _job_docker_rust_toolchain_other = - job_docker_rust_toolchain ~__POS__ () |> job_external ~filename_suffix:"other" - (** Type of Docker build jobs. The semantics of the type is summed up in this table: @@ -546,24 +469,6 @@ let rules_octez_docker_changes_or_master = job_rule ~changes:changeset_octez_docker_changes_or_master (); ] -let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies: - (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) - ~arch:Amd64 - Test_manual - -let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies: - (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) - ~arch:Arm64 - Test_manual - (* Note: here we rely on [$IMAGE_ARCH_PREFIX] to be empty. Otherwise, [$DOCKER_IMAGE_TAG] would contain [$IMAGE_ARCH_PREFIX] too. [$IMAGE_ARCH_PREFIX] is only used when building Docker images, @@ -655,45 +560,19 @@ let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch "make $TARGET"; ] -let job_build_dpkg_amd64 = +let job_build_dpkg_amd64 : unit -> tezos_job = job_build_bin_package ~__POS__ ~name:"oc.build:dpkg:amd64" ~target:Dpkg ~arch:Tezos_ci.Amd64 - () - |> job_external -let job_build_rpm_amd64 = +let job_build_rpm_amd64 : unit -> tezos_job = job_build_bin_package ~__POS__ ~name:"oc.build:rpm:amd64" ~target:Rpm ~arch:Tezos_ci.Amd64 - () - |> job_external - -let _job_build_dpkg_amd64_manual = - job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - ~rules:[job_rule ~when_:Manual ()] - ~stage:Stages.manual - () - |> job_external ~directory:"build" ~filename_suffix:"manual" - -let _job_build_rpm_amd64_manual = - job_build_bin_package - ~__POS__ - ~rules:[job_rule ~when_:Manual ()] - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - ~stage:Stages.manual - () - |> job_external ~directory:"build" ~filename_suffix:"manual" (** Type of release tag pipelines. @@ -749,12 +628,7 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = job_build_static_binaries ~__POS__ ~arch:Arm64 ~release:true () in let job_static_x86_64_release = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - ~release:true - ~needs_trigger:true - () + job_build_static_binaries ~__POS__ ~arch:Amd64 ~release:true () in let job_gitlab_release ~dependencies : Tezos_ci.tezos_job = job @@ -779,6 +653,8 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = ~name:"gitlab:publish" ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] in + let job_build_dpkg_amd64 = job_build_dpkg_amd64 () in + let job_build_rpm_amd64 = job_build_rpm_amd64 () in let job_gitlab_release_or_publish = let dependencies = Dependent @@ -833,7 +709,7 @@ let amd64_build_extra = ] let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) - ?(needs_trigger = false) () = + ?dependencies () = let arch_string = match arch with Amd64 -> "x86_64" | Arm64 -> "arm64" in let name = sf @@ -872,17 +748,10 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) "_build/default/contrib/octez_injector_server/octez_injector_server.exe"; ] in - let dependencies = - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration of - pipelines. So when requested through [needs_trigger] we start - this job as early as possible, without waiting for - sanity_ci. *) - if needs_trigger then Dependent [Optional job_trigger] else Staged [] - in let job = job ?rules + ?dependencies ~__POS__ ~stage:Stages.build ~arch @@ -895,46 +764,20 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) ~eval_opam:true []) ~variables - ~dependencies ~artifacts ["./scripts/ci/build_full_unreleased.sh"] in (* Disable coverage for arm64 *) if arch = Amd64 then enable_coverage_instrumentation job else job -let build_arm_rules = - [ - job_rule ~if_:Rules.schedule_extended_tests ~when_:Always (); - job_rule ~if_:Rules.(has_mr_label "ci--arm64") ~when_:On_success (); - job_rule - ~changes:["src/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"] - ~when_:Manual - ~allow_failure:Yes - (); - ] - (* Write external files for build_arm64_jobs. Used in external pipelines [before_merging] and [schedule_extended_test]. *) -let job_build_arm64_release : tezos_job = - job_build_dynamic_binaries - ~__POS__ - ~arch:Arm64 - ~needs_trigger:false - ~release:true - ~rules:build_arm_rules - () - |> job_external +let job_build_arm64_release ?rules () : tezos_job = + job_build_dynamic_binaries ?rules ~__POS__ ~arch:Arm64 ~release:true () -let job_build_arm64_exp_dev_extra : tezos_job = - job_build_dynamic_binaries - ~__POS__ - ~arch:Arm64 - ~needs_trigger:false - ~release:false - ~rules:build_arm_rules - () - |> job_external +let job_build_arm64_exp_dev_extra ?rules () : tezos_job = + job_build_dynamic_binaries ?rules ~__POS__ ~arch:Arm64 ~release:false () let enable_coverage_report job : tezos_job = job @@ -953,6 +796,195 @@ let enable_coverage_report job : tezos_job = ["_coverage_report/"; "$BISECT_FILE"] |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] +type code_verification_pipeline = Before_merging | Schedule_extended_test + +(* Encodes the conditional [before_merging] pipeline and its unconditional variant + [schedule_extended_test]. *) +let code_verification_pipeline pipeline_type = + (* Externalization *) + let job_external_split ?(before_merging_suffix = "before_merging") + ?(scheduled_suffix = "scheduled_extended_test") job = + job_external + ~filename_suffix: + (match pipeline_type with + | Before_merging -> before_merging_suffix + | Schedule_extended_test -> scheduled_suffix) + job + in + (* [make_rules] makes rules for jobs that are: + - automatic in scheduled pipelines; + - conditional in [before_merging] pipelines. + + If [label], [changes] and [manual] are omitted, then rules will + enable the job [On_success] in the [before_merging] + pipeline. This is safe, but prefer specifying a [changes] clause + if possible. *) + let make_rules ?label ?changes ?(manual = false) () = + match pipeline_type with + | Schedule_extended_test -> + (* The scheduled pipeline always runs all tests unconditionally. *) + [job_rule ~when_:Always ()] + | Before_merging -> + (* MR labels can be used to force tests to run. *) + (match label with + | Some label -> + [job_rule ~if_:Rules.(has_mr_label label) ~when_:On_success ()] + | None -> []) + (* Modifying some files can force tests to run. *) + @ (match changes with + | None -> [] + | Some changes -> [job_rule ~changes ~when_:On_success ()]) + (* For some tests, it can be relevant to have a manual trigger. *) + @ if manual then [job_rule ~when_:Manual ()] else [] + in + (* Stages *) + (* All stages should be empty, as explained below, until the full pipeline is generated. *) + let trigger, dependencies_needs_trigger = + match pipeline_type with + | Schedule_extended_test -> ([], Staged []) + | Before_merging -> + (* Define the [trigger] job + + §1: The purpose of this job is to launch the CI manually in certain cases. + The objective is not to run computing when it is not + necessary and the decision to do so belongs to the developer + + §2: We also perform some fast sanity checks. *) + let job_trigger = + job + ~__POS__ + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + |> job_external + in + (* TODO: put job_trigger here when full pipeline is generated *) + ([], Dependent [Optional job_trigger]) + in + let sanity = [] in + let job_docker_rust_toolchain = + job_docker_rust_toolchain + ~__POS__ + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:true ()) + ~dependencies:dependencies_needs_trigger + () + |> job_external_split + in + let build = + let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in + let _job_build_arm64_release : Tezos_ci.tezos_job = + job_build_arm64_release ~rules:build_arm_rules () |> job_external_split + in + let _job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = + job_build_arm64_exp_dev_extra ~rules:build_arm_rules () + |> job_external_split + in + let _job_static_x86_64_experimental = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration + of pipelines. So we start this job as early as possible, + without waiting for sanity_ci. *) + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + () + |> job_external_split + in + (* TODO: The code is a bit convulted here because these jobs are + either in the build or in the manual stage depeneding on the + pipeline type. However, we can put them in the build stage on + [before_merging] pipelines as long as we're careful to put + [allow_failure: true]. *) + (match pipeline_type with + | Schedule_extended_test -> + let _job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in + let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in + () + | Before_merging -> ()) ; + (* TODO: include the jobs defined above when full pipeline is generated *) + [] + in + let packaging = [] in + let test = [] in + let doc = [] in + let manual = + match pipeline_type with + | Before_merging -> + let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = + job_docker_build + ~__POS__ + ~external_:true + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Amd64 + Test_manual + in + let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = + job_docker_build + ~__POS__ + ~external_:true + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Arm64 + Test_manual + in + let _job_build_dpkg_amd64_manual = + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" + in + let _job_build_rpm_amd64_manual = + job_build_bin_package + ~__POS__ + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" + in + (* TODO: include the jobs defined above when full pipeline is generated *) + [] + (* No manual jobs on the scheduled pipeline *) + | Schedule_extended_test -> [] + in + (* Empty place-holder: this has the effect of not overwriting the pipeline file in question. + Once all the jobs in these pipelines are defined, we will return them here which + will cause the pipeline files to contain the definition of all those jobs. + + Until that time, all the jobs are written ot external files + (using {!job_external} or {!jobs_external}) and included by hand + in the files [.gitlab/ci/pipelines/before_merging.yml] and + [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) + trigger @ sanity @ build @ packaging @ test @ doc @ manual + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -974,7 +1006,10 @@ let () = let has_non_release_tag = If.(Predefined_vars.ci_commit_tag != null && not has_any_octez_release_tag) in - register "before_merging" If.(on_tezos_namespace && merge_request) ; + register + "before_merging" + If.(on_tezos_namespace && merge_request) + ~jobs:(code_verification_pipeline Before_merging) ; register "octez_latest_release" ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:true] @@ -1034,9 +1069,6 @@ let () = job_build_static_binaries ~__POS__ ~arch:Amd64 - (* TODO: this job doesn't actually need trigger and there is no - need to set it optional since we know this job is only on the master branch. *) - ~needs_trigger:true ~rules:[job_rule ~when_:Always ()] () in @@ -1120,6 +1152,14 @@ let () = publish-sdk"; ] in + (* arm builds are manual on the master branch pipeline *) + let build_arm_rules = [job_rule ~when_:Manual ~allow_failure:Yes ()] in + let job_build_arm64_release = + job_build_arm64_release ~rules:build_arm_rules () + in + let job_build_arm64_exp_dev_extra = + job_build_arm64_exp_dev_extra ~rules:build_arm_rules () + in [ (* Stage: build *) job_docker_rust_toolchain; @@ -1158,7 +1198,10 @@ let () = "non_release_tag_test" If.(not_on_tezos_namespace && push && has_non_release_tag) ~jobs:(release_tag_pipeline ~test:true Non_release_tag) ; - register "schedule_extended_test" schedule_extended_tests + register + "schedule_extended_test" + schedule_extended_tests + ~jobs:(code_verification_pipeline Schedule_extended_test) (* Split pipelines and writes image templates *) let config () =