From 22e00ec2d25f08491a1b9e0ee632b39fada2b4b7 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:00:58 +0100 Subject: [PATCH 01/10] CI: Delete unused [oc.build:static-arm64-linux-binaries] We still do static arm builds in master_branch and release_tag pipelines, but those pipelines are fully generated. This external version of the jobs is not included by any other file so can be removed. --- ...atic-arm64-linux-binaries-experimental.yml | 30 ------------------- ci/bin/main.ml | 8 ----- 2 files changed, 38 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml deleted file mode 100644 index f1a171f820f6..000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml +++ /dev/null @@ -1,30 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-arm64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp_arm64 - rules: - - changes: - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - artifacts: - paths: - - octez-binaries/$ARCH/* diff --git a/ci/bin/main.ml b/ci/bin/main.ml index b1b2bb135b82..41aae7c07c59 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -394,14 +394,6 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) let rules_static_build_other = [job_rule ~changes:changeset_octez ()] -let _job_static_arm64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_other - () - |> job_external ~filename_suffix:"experimental" - let _job_static_x86_64_experimental = job_build_static_binaries ~__POS__ -- GitLab From 44fe13df1a2979230f411d8607b6211a000b4d3d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 12:15:54 +0100 Subject: [PATCH 02/10] CI: Make scheduled job [job_docker_rust_toolchain] run [Always] --- .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml | 2 ++ ci/bin/main.ml | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml index ec72da648d20..5385d46a70a8 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml @@ -6,6 +6,8 @@ oc.docker:rust-toolchain: stage: build tags: - gcp + rules: + - when: always dependencies: [] before_script: [] script: diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 41aae7c07c59..9efbece149e4 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -430,8 +430,13 @@ let _job_docker_rust_toolchain_before_merging = () |> job_external ~filename_suffix:"before_merging" +(* Used in external [scheduled_extended_test] pipeline *) let _job_docker_rust_toolchain_other = - job_docker_rust_toolchain ~__POS__ () |> job_external ~filename_suffix:"other" + job_docker_rust_toolchain + ~__POS__ + ~rules:[(* Scheduled jobs should run [Always] *) job_rule ~when_:Always ()] + () + |> job_external ~filename_suffix:"other" (** Type of Docker build jobs. -- GitLab From 7f9c526f37556df979c0c4b30bf7b81c1a774393 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 11:43:38 +0100 Subject: [PATCH 03/10] CI: Inline [job_docker_rust_toolchain] and rev dependencies --- ci/bin/main.ml | 143 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 103 insertions(+), 40 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 9efbece149e4..625e505904e4 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -418,26 +418,6 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = []) ["./scripts/ci/docker_rust_toolchain_build.sh"] -let _job_docker_rust_toolchain_before_merging = - job_docker_rust_toolchain - ~__POS__ - ~dependencies:(Dependent [Optional job_trigger]) - ~rules: - [ - job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); - job_rule ~when_:Manual (); - ] - () - |> job_external ~filename_suffix:"before_merging" - -(* Used in external [scheduled_extended_test] pipeline *) -let _job_docker_rust_toolchain_other = - job_docker_rust_toolchain - ~__POS__ - ~rules:[(* Scheduled jobs should run [Always] *) job_rule ~when_:Always ()] - () - |> job_external ~filename_suffix:"other" - (** Type of Docker build jobs. The semantics of the type is summed up in this table: @@ -543,24 +523,6 @@ let rules_octez_docker_changes_or_master = job_rule ~changes:changeset_octez_docker_changes_or_master (); ] -let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies: - (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) - ~arch:Amd64 - Test_manual - -let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies: - (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) - ~arch:Arm64 - Test_manual - (* Note: here we rely on [$IMAGE_ARCH_PREFIX] to be empty. Otherwise, [$DOCKER_IMAGE_TAG] would contain [$IMAGE_ARCH_PREFIX] too. [$IMAGE_ARCH_PREFIX] is only used when building Docker images, @@ -950,6 +912,101 @@ let enable_coverage_report job : tezos_job = ["_coverage_report/"; "$BISECT_FILE"] |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] +type code_verification_pipeline = Before_merging | Schedule_extended_test + +(* Encodes the conditional [before_merging] pipeline and its unconditional variant + [schedule_extended_test]. *) +let code_verification_pipeline pipeline_type = + (* Externalization *) + let job_external_split ?(before_merging_suffix = "before_merging") + ?(scheduled_suffix = "scheduled_extended_test") job = + job_external + ~filename_suffix: + (match pipeline_type with + | Before_merging -> before_merging_suffix + | Schedule_extended_test -> scheduled_suffix) + job + in + (* [make_rules] makes rules for jobs that are: + - automatic in scheduled pipelines; + - conditional in [before_merging] pipelines. + + If [label], [changes] and [manual] are omitted, then rules will + enable the job [On_success] in the [before_merging] + pipeline. This is safe, but prefer specifying a [changes] clause + if possible. *) + let make_rules ?label ?changes ?(manual = false) () = + match pipeline_type with + | Schedule_extended_test -> + (* The scheduled pipeline always runs all tests unconditionally. *) + [job_rule ~when_:Always ()] + | Before_merging -> + (* MR labels can be used to force tests to run. *) + (match label with + | Some label -> + [job_rule ~if_:Rules.(has_mr_label label) ~when_:On_success ()] + | None -> []) + (* Modifying some files can force tests to run. *) + @ (match changes with + | None -> [] + | Some changes -> [job_rule ~changes ~when_:On_success ()]) + (* For some tests, it can be relevant to have a manual trigger. *) + @ if manual then [job_rule ~when_:Manual ()] else [] + in + (* Stages *) + (* All stages should be empty, as explained below, until the full pipeline is generated. *) + let trigger = [] in + let sanity = [] in + let job_docker_rust_toolchain = + job_docker_rust_toolchain + ~__POS__ + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:true ()) + ~dependencies: + (match pipeline_type with + | Schedule_extended_test -> Staged [] + | Before_merging -> Dependent [Optional job_trigger]) + () + |> job_external_split + ~before_merging_suffix:"before_merging" + ~scheduled_suffix:"other" + in + let build = [] in + let packaging = [] in + let test = [] in + let doc = [] in + let manual = + match pipeline_type with + | Before_merging -> + let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = + job_docker_build + ~__POS__ + ~external_:true + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Amd64 + Test_manual + in + let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = + job_docker_build + ~__POS__ + ~external_:true + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Arm64 + Test_manual + in + [] + (* No manual jobs on the scheduled pipeline *) + | Schedule_extended_test -> [] + in + (* Empty place-holder: this has the effect of not overwriting the pipeline file in question. + Once all the jobs in these pipelines are defined, we will return them here which + will cause the pipeline files to contain the definition of all those jobs. + + Until that time, all the jobs are written ot external files + (using {!job_external} or {!jobs_external}) and included by hand + in the files [.gitlab/ci/pipelines/before_merging.yml] and + [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) + trigger @ sanity @ build @ packaging @ test @ doc @ manual + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -971,7 +1028,10 @@ let () = let has_non_release_tag = If.(Predefined_vars.ci_commit_tag != null && not has_any_octez_release_tag) in - register "before_merging" If.(on_tezos_namespace && merge_request) ; + register + "before_merging" + If.(on_tezos_namespace && merge_request) + ~jobs:(code_verification_pipeline Before_merging) ; register "octez_latest_release" ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:true] @@ -1155,7 +1215,10 @@ let () = "non_release_tag_test" If.(not_on_tezos_namespace && push && has_non_release_tag) ~jobs:(release_tag_pipeline ~test:true Non_release_tag) ; - register "schedule_extended_test" schedule_extended_tests + register + "schedule_extended_test" + schedule_extended_tests + ~jobs:(code_verification_pipeline Schedule_extended_test) (* Split pipelines and writes image templates *) let config () = -- GitLab From f071817924d0fbc1f4886f0b0655c98a9f21ddc8 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 12:20:23 +0100 Subject: [PATCH 04/10] CI: split the [build_arm64] jobs Before, the generated external job had a rule set that combined rules intended for the scheduled pipeline and rules intended for the before_merging pipeline. This commits splits the job into one external file set per pipeline. Furthermore, it inlines the definition of these jobs to the pipeline in which they belong. --- ...ld_arm64-exp-dev-extra-before_merging.yml} | 8 +--- ...-exp-dev-extra-scheduled_extended_test.yml | 32 +++++++++++++++ ...c.build_arm64-released-before_merging.yml} | 8 +--- ...arm64-released-scheduled_extended_test.yml | 30 ++++++++++++++ .gitlab/ci/pipelines/before_merging.yml | 4 +- .gitlab/ci/pipelines/master_branch.yml | 20 +-------- .../ci/pipelines/schedule_extended_test.yml | 4 +- ci/bin/main.ml | 41 +++++++++++-------- 8 files changed, 93 insertions(+), 54 deletions(-) rename .gitlab/ci/jobs/build/{oc.build_arm64-exp-dev-extra.yml => oc.build_arm64-exp-dev-extra-before_merging.yml} (85%) create mode 100644 .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml rename .gitlab/ci/jobs/build/{oc.build_arm64-released.yml => oc.build_arm64-released-before_merging.yml} (83%) create mode 100644 .gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml similarity index 85% rename from .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml rename to .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml index c6f4f4db5b28..cfd8cd2cdc26 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml @@ -7,15 +7,9 @@ oc.build_arm64-exp-dev-extra: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml new file mode 100644 index 000000000000..9679753d8f4c --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml @@ -0,0 +1,32 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_arm64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml similarity index 83% rename from .gitlab/ci/jobs/build/oc.build_arm64-released.yml rename to .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml index 287950d42cd9..90bce029d120 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml @@ -7,15 +7,9 @@ oc.build_arm64-released: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml new file mode 100644 index 000000000000..a5b1e85df907 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml @@ -0,0 +1,30 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_arm64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 9f618faa7825..018b3d9b90dd 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -16,8 +16,8 @@ include: - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml - .gitlab/ci/jobs/build/ocaml-check.yml - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml + - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml + - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml - .gitlab/ci/jobs/build/oc.build_kernels.yml diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index ba0a816899d2..8e26cc0c01d0 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -70,15 +70,7 @@ oc.build_arm64-released: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: @@ -107,15 +99,7 @@ oc.build_arm64-exp-dev-extra: tags: - gcp_arm64 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" - when: always - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual + - when: manual allow_failure: true dependencies: [] before_script: diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 068ad27bc242..7db7c3dc1276 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -4,8 +4,8 @@ include: - .gitlab/ci/jobs/sanity/docker:hadolint-schedule_extended_test.yml # Stage: build - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml + - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-scheduled_extended_test.yml + - .gitlab/ci/jobs/build/oc.build_arm64-released-scheduled_extended_test.yml # Check that binary packages build - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 625e505904e4..0c5eeb075fde 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -861,39 +861,26 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) (* Disable coverage for arm64 *) if arch = Amd64 then enable_coverage_instrumentation job else job -let build_arm_rules = - [ - job_rule ~if_:Rules.schedule_extended_tests ~when_:Always (); - job_rule ~if_:Rules.(has_mr_label "ci--arm64") ~when_:On_success (); - job_rule - ~changes:["src/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"] - ~when_:Manual - ~allow_failure:Yes - (); - ] - (* Write external files for build_arm64_jobs. Used in external pipelines [before_merging] and [schedule_extended_test]. *) -let job_build_arm64_release : tezos_job = +let job_build_arm64_release ?rules () : tezos_job = job_build_dynamic_binaries + ?rules ~__POS__ ~arch:Arm64 ~needs_trigger:false ~release:true - ~rules:build_arm_rules () - |> job_external -let job_build_arm64_exp_dev_extra : tezos_job = +let job_build_arm64_exp_dev_extra ?rules () : tezos_job = job_build_dynamic_binaries + ?rules ~__POS__ ~arch:Arm64 ~needs_trigger:false ~release:false - ~rules:build_arm_rules () - |> job_external let enable_coverage_report job : tezos_job = job @@ -970,7 +957,17 @@ let code_verification_pipeline pipeline_type = ~before_merging_suffix:"before_merging" ~scheduled_suffix:"other" in - let build = [] in + let build = + let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in + let _job_build_arm64_release : Tezos_ci.tezos_job = + job_build_arm64_release ~rules:build_arm_rules () |> job_external_split + in + let _job_build_arm64_exp_dev_extra : Tezos_ci.tezos_job = + job_build_arm64_exp_dev_extra ~rules:build_arm_rules () + |> job_external_split + in + [] + in let packaging = [] in let test = [] in let doc = [] in @@ -1177,6 +1174,14 @@ let () = publish-sdk"; ] in + (* arm builds are manual on the master branch pipeline *) + let build_arm_rules = [job_rule ~when_:Manual ~allow_failure:Yes ()] in + let job_build_arm64_release = + job_build_arm64_release ~rules:build_arm_rules () + in + let job_build_arm64_exp_dev_extra = + job_build_arm64_exp_dev_extra ~rules:build_arm_rules () + in [ (* Stage: build *) job_docker_rust_toolchain; -- GitLab From 9db69c4f73f8943dc9437c58da95e583176020ed Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:04:42 +0100 Subject: [PATCH 05/10] CI: remove redundant optional dependency on [job_trigger] This removes the optional dependency on [job_trigger] for jobs that are not in the [before_merging] pipeline. The trigger job is not defined in those pipeline. --- .gitlab/ci/pipelines/master_branch.yml | 3 - .gitlab/ci/pipelines/non_release_tag.yml | 3 - .gitlab/ci/pipelines/non_release_tag_test.yml | 3 - .../ci/pipelines/octez_beta_release_tag.yml | 3 - .gitlab/ci/pipelines/octez_release_tag.yml | 3 - .../ci/pipelines/octez_release_tag_test.yml | 3 - ci/bin/main.ml | 58 +++++-------------- 7 files changed, 13 insertions(+), 63 deletions(-) diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 8e26cc0c01d0..b6551a0503bf 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -28,9 +28,6 @@ oc.build:static-x86_64-linux-binaries: - gcp rules: - when: always - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index b5d49fabd4e1..455d509283d8 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 4a618b16e510..d2f5712f20cc 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 0f2740ec529a..693ed5a322a2 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index af5307de2fe7..7aacdea83d7e 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 127ee6d96149..fd100658ac88 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -24,9 +24,6 @@ oc.build:static-x86_64-linux-binaries: stage: build tags: - gcp - needs: - - job: trigger - optional: true dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 0c5eeb075fde..b80f88a5932d 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -356,8 +356,8 @@ let job_docker_promote_to_latest ~ci_docker_hub : tezos_job = (no need to test that we pass the -static flag twice) - released variants exist, that are used in release tag pipelines (they do not build experimental executables) *) -let job_build_static_binaries ~__POS__ ~arch ?(release = false) - ?(needs_trigger = false) ?rules () : Tezos_ci.tezos_job = +let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules + ?dependencies () : Tezos_ci.tezos_job = let arch_string = match arch with Tezos_ci.Amd64 -> "x86_64" | Arm64 -> "arm64" in @@ -371,16 +371,9 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) "script-inputs/released-executables" ^ if not release then " script-inputs/experimental-executables" else "" in - let dependencies = - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration of - pipelines. So when requested through [needs_trigger] we start - this job as early as possible, without waiting for - sanity_ci. *) - if needs_trigger then Dependent [Optional job_trigger] else Staged [] - in job ?rules + ?dependencies ~__POS__ ~stage:Stages.build ~arch @@ -388,7 +381,6 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ~image:Images.runtime_build_dependencies ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) ~variables:[("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] - ~dependencies ~artifacts ["./scripts/ci/build_static_binaries.sh"] @@ -398,7 +390,11 @@ let _job_static_x86_64_experimental = job_build_static_binaries ~__POS__ ~arch:Amd64 - ~needs_trigger:true + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration + of pipelines. So we start this job as early as possible, + without waiting for sanity_ci. *) + ~dependencies:(Dependent [Optional job_trigger]) ~rules:rules_static_build_other () |> job_external ~filename_suffix:"experimental" @@ -708,12 +704,7 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = job_build_static_binaries ~__POS__ ~arch:Arm64 ~release:true () in let job_static_x86_64_release = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - ~release:true - ~needs_trigger:true - () + job_build_static_binaries ~__POS__ ~arch:Amd64 ~release:true () in let job_gitlab_release ~dependencies : Tezos_ci.tezos_job = job @@ -792,7 +783,7 @@ let amd64_build_extra = ] let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) - ?(needs_trigger = false) () = + ?dependencies () = let arch_string = match arch with Amd64 -> "x86_64" | Arm64 -> "arm64" in let name = sf @@ -831,17 +822,10 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) "_build/default/contrib/octez_injector_server/octez_injector_server.exe"; ] in - let dependencies = - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration of - pipelines. So when requested through [needs_trigger] we start - this job as early as possible, without waiting for - sanity_ci. *) - if needs_trigger then Dependent [Optional job_trigger] else Staged [] - in let job = job ?rules + ?dependencies ~__POS__ ~stage:Stages.build ~arch @@ -854,7 +838,6 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) ~eval_opam:true []) ~variables - ~dependencies ~artifacts ["./scripts/ci/build_full_unreleased.sh"] in @@ -865,22 +848,10 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) Used in external pipelines [before_merging] and [schedule_extended_test]. *) let job_build_arm64_release ?rules () : tezos_job = - job_build_dynamic_binaries - ?rules - ~__POS__ - ~arch:Arm64 - ~needs_trigger:false - ~release:true - () + job_build_dynamic_binaries ?rules ~__POS__ ~arch:Arm64 ~release:true () let job_build_arm64_exp_dev_extra ?rules () : tezos_job = - job_build_dynamic_binaries - ?rules - ~__POS__ - ~arch:Arm64 - ~needs_trigger:false - ~release:false - () + job_build_dynamic_binaries ?rules ~__POS__ ~arch:Arm64 ~release:false () let enable_coverage_report job : tezos_job = job @@ -1088,9 +1059,6 @@ let () = job_build_static_binaries ~__POS__ ~arch:Amd64 - (* TODO: this job doesn't actually need trigger and there is no - need to set it optional since we know this job is only on the master branch. *) - ~needs_trigger:true ~rules:[job_rule ~when_:Always ()] () in -- GitLab From be5eeb4eccf8fc42838741bfeb673f379adb2e64 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:10:08 +0100 Subject: [PATCH 06/10] CI: split and inline the [job_static_x86_64_experimental] job This splits the job in one version that has a dependency on trigger, and one that does not. Furthermore, it inlines the definition of these jobs to the pipeline in which they belong. --- ...-x86_64-linux-binaries-before_merging.yml} | 0 ...linux-binaries-scheduled_extended_test.yml | 30 +++++++++++++++ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/main.ml | 38 +++++++++---------- 5 files changed, 51 insertions(+), 21 deletions(-) rename .gitlab/ci/jobs/build/{oc.build:static-x86_64-linux-binaries-experimental.yml => oc.build:static-x86_64-linux-binaries-before_merging.yml} (100%) create mode 100644 .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml similarity index 100% rename from .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml rename to .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml new file mode 100644 index 000000000000..0b77b5b2fdb5 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml @@ -0,0 +1,30 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 018b3d9b90dd..56c722216e85 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -15,7 +15,7 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml - .gitlab/ci/jobs/build/ocaml-check.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml + - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 7db7c3dc1276..08dba5372f0e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -22,7 +22,7 @@ include: # These jobs may not have been run in before_merging pipeline # because of absence of certain changes - .gitlab/ci/jobs/build/ocaml-check.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml + - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml - .gitlab/ci/jobs/doc/documentation.yml # Stage: packaging diff --git a/ci/bin/main.ml b/ci/bin/main.ml index b80f88a5932d..b1468e44f3a4 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -384,21 +384,6 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules ~artifacts ["./scripts/ci/build_static_binaries.sh"] -let rules_static_build_other = [job_rule ~changes:changeset_octez ()] - -let _job_static_x86_64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration - of pipelines. So we start this job as early as possible, - without waiting for sanity_ci. *) - ~dependencies:(Dependent [Optional job_trigger]) - ~rules:rules_static_build_other - () - |> job_external ~filename_suffix:"experimental" - let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = job_docker_authenticated ?rules @@ -915,14 +900,16 @@ let code_verification_pipeline pipeline_type = (* All stages should be empty, as explained below, until the full pipeline is generated. *) let trigger = [] in let sanity = [] in + let dependencies_needs_trigger = + match pipeline_type with + | Schedule_extended_test -> Staged [] + | Before_merging -> Dependent [Optional job_trigger] + in let job_docker_rust_toolchain = job_docker_rust_toolchain ~__POS__ ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:true ()) - ~dependencies: - (match pipeline_type with - | Schedule_extended_test -> Staged [] - | Before_merging -> Dependent [Optional job_trigger]) + ~dependencies:dependencies_needs_trigger () |> job_external_split ~before_merging_suffix:"before_merging" @@ -937,6 +924,19 @@ let code_verification_pipeline pipeline_type = job_build_arm64_exp_dev_extra ~rules:build_arm_rules () |> job_external_split in + let _job_static_x86_64_experimental = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration + of pipelines. So we start this job as early as possible, + without waiting for sanity_ci. *) + ~dependencies:dependencies_needs_trigger + ~rules:[job_rule ~changes:changeset_octez ()] + () + |> job_external_split + in [] in let packaging = [] in -- GitLab From ecb60bb7f0746c38afc2d130a9449adf20fcd106 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:15:06 +0100 Subject: [PATCH 07/10] CI: refine rules for [oc.build:static-x86_64-linux-binaries] The rules for this job on the scheduled pipelines should not be conditioned on [changes:]. --- ...c-x86_64-linux-binaries-scheduled_extended_test.yml | 10 +--------- ci/bin/main.ml | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml index 0b77b5b2fdb5..7605ead2f3de 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml @@ -7,15 +7,7 @@ oc.build:static-x86_64-linux-binaries: tags: - gcp rules: - - changes: - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success + - when: always dependencies: [] before_script: - ./scripts/ci/take_ownership.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index b1468e44f3a4..0ff7b6068848 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -933,7 +933,7 @@ let code_verification_pipeline pipeline_type = of pipelines. So we start this job as early as possible, without waiting for sanity_ci. *) ~dependencies:dependencies_needs_trigger - ~rules:[job_rule ~changes:changeset_octez ()] + ~rules:(make_rules ~changes:changeset_octez ()) () |> job_external_split in -- GitLab From e5af1c227b9f21997da348decb2632c3781eb4e4 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:19:39 +0100 Subject: [PATCH 08/10] CI: inline [job_trigger] into the [code_verification] pipeline --- ci/bin/main.ml | 81 +++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 0ff7b6068848..fbff14190a5e 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -273,41 +273,6 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () -(* Define the [trigger] job - - §1: The purpose of this job is to launch the CI manually in certain cases. - The objective is not to run computing when it is not - necessary and the decision to do so belongs to the developer - - §2: We also perform some fast sanity checks. *) -let job_trigger = - job - ~__POS__ - ~image:Images.alpine - ~stage:Stages.trigger - ~allow_failure:No - ~rules: - [ - job_rule - ~if_:(If.not Rules.assigned_to_marge_bot) - ~allow_failure:No - ~when_:Manual - (); - job_rule ~when_:Always (); - ] - ~timeout:(Minutes 10) - ~name:"trigger" - [ - "echo 'Trigger pipeline!'"; - (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and - [scripts/version.sh]'s [opam_repository_tag] are the same. *) - "./scripts/ci/check_opam_repository_tag.sh"; - (* Check that the Alpine version of the trigger job's image - corresponds to the value in scripts/version.sh. *) - "./scripts/ci/check_alpine_version.sh"; - ] - |> job_external - (** Helper to create jobs that uses the docker deamon. It: @@ -898,13 +863,49 @@ let code_verification_pipeline pipeline_type = in (* Stages *) (* All stages should be empty, as explained below, until the full pipeline is generated. *) - let trigger = [] in - let sanity = [] in - let dependencies_needs_trigger = + let trigger, dependencies_needs_trigger = match pipeline_type with - | Schedule_extended_test -> Staged [] - | Before_merging -> Dependent [Optional job_trigger] + | Schedule_extended_test -> ([], Staged []) + | Before_merging -> + (* Define the [trigger] job + + §1: The purpose of this job is to launch the CI manually in certain cases. + The objective is not to run computing when it is not + necessary and the decision to do so belongs to the developer + + §2: We also perform some fast sanity checks. *) + let job_trigger = + job + ~__POS__ + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + |> job_external + in + (* TODO: put job_trigger here when full pipeline is generated *) + ([], Dependent [Optional job_trigger]) in + let sanity = [] in let job_docker_rust_toolchain = job_docker_rust_toolchain ~__POS__ -- GitLab From d7981d624d4ee6efdb33109daa42779847598251 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 13:28:59 +0100 Subject: [PATCH 09/10] CI: inline [job_build_dpkg_amd64] job definitions --- ci/bin/main.ml | 67 +++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index fbff14190a5e..aaac3569bfd4 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -560,45 +560,19 @@ let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch "make $TARGET"; ] -let job_build_dpkg_amd64 = +let job_build_dpkg_amd64 : unit -> tezos_job = job_build_bin_package ~__POS__ ~name:"oc.build:dpkg:amd64" ~target:Dpkg ~arch:Tezos_ci.Amd64 - () - |> job_external - -let job_build_rpm_amd64 = - job_build_bin_package - ~__POS__ - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - () - |> job_external - -let _job_build_dpkg_amd64_manual = - job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - ~rules:[job_rule ~when_:Manual ()] - ~stage:Stages.manual - () - |> job_external ~directory:"build" ~filename_suffix:"manual" -let _job_build_rpm_amd64_manual = +let job_build_rpm_amd64 : unit -> tezos_job = job_build_bin_package ~__POS__ - ~rules:[job_rule ~when_:Manual ()] ~name:"oc.build:rpm:amd64" ~target:Rpm ~arch:Tezos_ci.Amd64 - ~stage:Stages.manual - () - |> job_external ~directory:"build" ~filename_suffix:"manual" (** Type of release tag pipelines. @@ -679,6 +653,8 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = ~name:"gitlab:publish" ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] in + let job_build_dpkg_amd64 = job_build_dpkg_amd64 () in + let job_build_rpm_amd64 = job_build_rpm_amd64 () in let job_gitlab_release_or_publish = let dependencies = Dependent @@ -938,6 +914,18 @@ let code_verification_pipeline pipeline_type = () |> job_external_split in + (* TODO: The code is a bit convulted here because these jobs are + either in the build or in the manual stage depeneding on the + pipeline type. However, we can put them in the build stage on + [before_merging] pipelines as long as we're careful to put + [allow_failure: true]. *) + (match pipeline_type with + | Schedule_extended_test -> + let _job_build_dpkg_amd64 = job_build_dpkg_amd64 () |> job_external in + let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in + () + | Before_merging -> ()) ; + (* TODO: include the jobs defined above when full pipeline is generated *) [] in let packaging = [] in @@ -962,6 +950,29 @@ let code_verification_pipeline pipeline_type = ~arch:Arm64 Test_manual in + let _job_build_dpkg_amd64_manual = + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" + in + let _job_build_rpm_amd64_manual = + job_build_bin_package + ~__POS__ + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" + in + (* TODO: include the jobs defined above when full pipeline is generated *) [] (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] -- GitLab From 521902c37cd82f963309a9f0c47ec69e2bcc87c5 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 12 Mar 2024 14:00:47 +0100 Subject: [PATCH 10/10] CI: rename [oc.docker:rust-toolchain-{other=>scheduled_...}.yml] --- ...yml => oc.docker:rust-toolchain-scheduled_extended_test.yml} | 0 .gitlab/ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/main.ml | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) rename .gitlab/ci/jobs/build/{oc.docker:rust-toolchain-other.yml => oc.docker:rust-toolchain-scheduled_extended_test.yml} (100%) diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml similarity index 100% rename from .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml rename to .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 08dba5372f0e..e81624b5c027 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -12,7 +12,7 @@ include: - .gitlab/ci/jobs/build/select_tezts.yml # These jobs are necessary to run flaky tezts - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml + - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml diff --git a/ci/bin/main.ml b/ci/bin/main.ml index aaac3569bfd4..e4b9c0b4194c 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -889,8 +889,6 @@ let code_verification_pipeline pipeline_type = ~dependencies:dependencies_needs_trigger () |> job_external_split - ~before_merging_suffix:"before_merging" - ~scheduled_suffix:"other" in let build = let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in -- GitLab