From dea2e0c1cad39563fd0b91e4eba8a363db57c881 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 5 Mar 2024 14:06:59 +0100 Subject: [PATCH 01/13] CI-in-OCaml: pipe everything --- ci/bin/main.ml | 224 ++++++++++++++++++++++++------------------------- 1 file changed, 112 insertions(+), 112 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 524b8b0a973f..ed0aad356f7b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -244,32 +244,32 @@ let job_dummy : job = §2: We also perform some fast sanity checks. *) let trigger = - job_external - @@ job - ~__POS__ - ~image:Images.alpine - ~stage:Stages.trigger - ~allow_failure:No - ~rules: - [ - job_rule - ~if_:(If.not Rules.assigned_to_marge_bot) - ~allow_failure:No - ~when_:Manual - (); - job_rule ~when_:Always (); - ] - ~timeout:(Minutes 10) - ~name:"trigger" - [ - "echo 'Trigger pipeline!'"; - (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and - [scripts/version.sh]'s [opam_repository_tag] are the same. *) - "./scripts/ci/check_opam_repository_tag.sh"; - (* Check that the Alpine version of the trigger job's image - corresponds to the value in scripts/version.sh. *) - "./scripts/ci/check_alpine_version.sh"; - ] + job + ~__POS__ + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + |> job_external (** Helper to create jobs that uses the docker deamon. @@ -360,40 +360,40 @@ let rules_static_build_master = [job_rule ~when_:Always ()] let rules_static_build_other = [job_rule ~changes:changeset_octez ()] let _job_static_arm64_experimental = - job_external ~filename_suffix:"experimental" - @@ job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_other - () + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~rules:rules_static_build_other + () + |> job_external ~filename_suffix:"experimental" let _job_static_arm64_master = - job_external ~filename_suffix:"master" - @@ job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_master - () + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~rules:rules_static_build_master + () + |> job_external ~filename_suffix:"master" let _job_static_x86_64_experimental = - job_external ~filename_suffix:"experimental" - @@ job_build_static_binaries - ~__POS__ - ~arch:Amd64 - ~needs_trigger:true - ~rules:rules_static_build_other - () + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + ~needs_trigger:true + ~rules:rules_static_build_other + () + |> job_external ~filename_suffix:"experimental" let _job_static_x86_64_master = - job_external ~filename_suffix:"master" - @@ job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* TODO: this job doesn't actually need trigger and there is no - need to set it optional since we know this job is only on the master branch. *) - ~needs_trigger:true - ~rules:rules_static_build_master - () + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* TODO: this job doesn't actually need trigger and there is no + need to set it optional since we know this job is only on the master branch. *) + ~needs_trigger:true + ~rules:rules_static_build_master + () + |> job_external ~filename_suffix:"master" let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = job_docker_authenticated @@ -411,23 +411,23 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = ["./scripts/ci/docker_rust_toolchain_build.sh"] let _job_docker_rust_toolchain_before_merging = - job_external ~filename_suffix:"before_merging" - @@ job_docker_rust_toolchain - ~__POS__ - ~dependencies:(Dependent [Optional trigger]) - ~rules: - [ - job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); - job_rule ~when_:Manual (); - ] - () + job_docker_rust_toolchain + ~__POS__ + ~dependencies:(Dependent [Optional trigger]) + ~rules: + [ + job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); + job_rule ~when_:Manual (); + ] + () + |> job_external ~filename_suffix:"before_merging" let _job_docker_rust_toolchain_master = - job_external ~filename_suffix:"master" - @@ job_docker_rust_toolchain ~__POS__ ~rules:[job_rule ~when_:Always ()] () + job_docker_rust_toolchain ~__POS__ ~rules:[job_rule ~when_:Always ()] () + |> job_external ~filename_suffix:"master" let _job_docker_rust_toolchain_other = - job_external ~filename_suffix:"other" @@ job_docker_rust_toolchain ~__POS__ () + job_docker_rust_toolchain ~__POS__ () |> job_external ~filename_suffix:"other" (** Type of Docker build jobs. @@ -590,19 +590,19 @@ let job_docker_merge_manifests ~__POS__ ~ci_docker_hub ~job_docker_amd64 ["./scripts/ci/docker_merge_manifests.sh"] let _job_docker_merge_manifests_release = - job_external ~filename_suffix:"release" - @@ job_docker_merge_manifests - ~__POS__ - ~ci_docker_hub:true - (* TODO: In theory, actually uses either release or - experimental variant of docker jobs depending on - pipeline. In practice, this does not matter as these jobs - have the same name in the generated files - ([oc.build:ARCH]). However, when the merge_manifest jobs - are created directly in the appropriate pipeline, the - correcty variant must be used. *) - ~job_docker_amd64:job_docker_amd64_experimental - ~job_docker_arm64:job_docker_arm64_experimental + job_docker_merge_manifests + ~__POS__ + ~ci_docker_hub:true + (* TODO: In theory, actually uses either release or + experimental variant of docker jobs depending on + pipeline. In practice, this does not matter as these jobs + have the same name in the generated files + ([oc.build:ARCH]). However, when the merge_manifest jobs + are created directly in the appropriate pipeline, the + correcty variant must be used. *) + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + |> job_external ~filename_suffix:"release" type bin_package_target = Dpkg | Rpm @@ -679,44 +679,44 @@ let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch ] let job_build_dpkg_amd64 = - job_external - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + () + |> job_external let job_build_rpm_amd64 = - job_external - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + () + |> job_external let _job_build_dpkg_amd64_manual = - job_external ~directory:"build" ~filename_suffix:"manual" - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - ~rules:[job_rule ~when_:Manual ()] - ~stage:Stages.manual - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" let _job_build_rpm_amd64_manual = - job_external ~directory:"build" ~filename_suffix:"manual" - @@ job_build_bin_package - ~__POS__ - ~rules:[job_rule ~when_:Manual ()] - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - ~stage:Stages.manual - () + job_build_bin_package + ~__POS__ + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" (** Type of release tag pipelines. -- GitLab From 6802e16b3202c5fa4d3ae36861b75d60cabf4c5f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 10:46:18 +0100 Subject: [PATCH 02/13] CI-in-OCaml: fix doc comment for [Pipeline.register] --- ci/bin/tezos_ci.mli | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 222f59fa8738..8c5c34281137 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -56,20 +56,20 @@ end (** A facility for registering pipelines. *) module Pipeline : sig - (* Register a pipeline. + (** Register a pipeline. - [register ?variables name rule] will register a pipeline [name] - that runs when [rule] is true. + [register ?variables name rule] will register a pipeline [name] + that runs when [rule] is true. - If [variables] is set, then these variables will be added to the - [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. + If [variables] is set, then these variables will be added to the + [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. - If [jobs] is not set, then the pipeline is a legacy, hand-written - .yml file, expected to be defined in - [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those - jobs will be generated to the same file when {!write} is - called. In both cases, this file will be included from the - top-level [.gitlab-ci.yml]. *) + If [jobs] is not set, then the pipeline is a legacy, hand-written + .yml file, expected to be defined in + [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those + jobs will be generated to the same file when {!write} is + called. In both cases, this file will be included from the + top-level [.gitlab-ci.yml]. *) val register : ?variables:Gitlab_ci.Types.variables -> ?jobs:tezos_job list -> -- GitLab From 8bf3470d85d7a8694a1dd7496459bf052579a33a Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 09:17:39 +0100 Subject: [PATCH 03/13] CI-in-OCaml: add [Tezos_ci.add_artifacts] --- ci/bin/tezos_ci.ml | 76 +++++++++++++++++++++++++++++++++++++++++++++ ci/bin/tezos_ci.mli | 28 +++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index fbc33fab0daa..b56b7a7cdc6a 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -34,6 +34,10 @@ type tezos_job = { source_position : string * int * int * int; } +let map_job (tezos_job : tezos_job) + (f : Gitlab_ci.Types.job -> Gitlab_ci.Types.job) : tezos_job = + {tezos_job with job = f tezos_job.job} + let tezos_job_to_config_elements (j : tezos_job) = let source_comment = if Cli.config.inline_source_info then @@ -425,3 +429,75 @@ let job_external ?directory ?filename_suffix (tezos_job : tezos_job) : tezos_job filename ; Gitlab_ci.To_yaml.to_file ~header ~filename config ; tezos_job) + +let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths + (tezos_job : tezos_job) = + map_job tezos_job @@ fun (job : Gitlab_ci.Types.job) -> + match job.artifacts with + | None -> + { + job with + artifacts = Some (artifacts ?expose_as ?reports ?expire_in ?when_ paths); + } + | Some artifacts -> + let opt_merge_right o1 o2 = + match (o1, o2) with + | _, Some value -> Some value + | Some value, None -> Some value + | None, None -> None + in + let opt_combine o1 o2 f = + match (o1, o2) with + | None, opt | opt, None -> opt + | Some value1, Some value2 -> Some (f value1 value2) + in + let name = opt_merge_right artifacts.name name in + let expose_as = opt_merge_right artifacts.expose_as expose_as in + let expire_in = + opt_combine artifacts.expire_in expire_in @@ fun interval1 interval2 -> + (* This function gives a measure of the size of a duration in seconds. + It is only used to compare durations. *) + let interval_to_seconds = function + | Gitlab_ci.Types.Seconds n -> n + | Minutes n -> n * 60 + | Hours n -> n * 3600 + | Days n -> n * 24 * 3600 + | Weeks n -> n * 7 * 24 * 3600 + | Months n -> n * 31 * 7 * 24 * 3600 + | Years n -> n * 365 * 7 * 24 * 3600 + in + if interval_to_seconds interval1 > interval_to_seconds interval2 then + interval1 + else interval2 + in + let when_ = + opt_combine artifacts.when_ when_ @@ fun when1 when2 -> + if when1 = when2 then when1 else Always + in + let paths = + match paths with + | [] -> artifacts.paths + | _ -> Some (Option.value ~default:[] artifacts.paths @ paths) + in + let reports = + opt_combine artifacts.reports reports @@ fun reports1 reports2 -> + let opt_combine_fail field o1 o2 = + opt_combine o1 o2 @@ fun _v1 _v2 -> + failwith + "[add_artifacts] attempted to override existing %s report" + field + in + { + dotenv = opt_combine_fail "dotenv" reports1.dotenv reports2.dotenv; + junit = opt_combine_fail "junit" reports1.junit reports2.junit; + coverage_report = + opt_combine_fail + "coverage_report" + reports1.coverage_report + reports2.coverage_report; + } + in + { + job with + artifacts = Some {name; expose_as; expire_in; when_; paths; reports}; + } diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 8c5c34281137..997c082832ec 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -225,3 +225,31 @@ val job : used to write another job. *) val job_external : ?directory:string -> ?filename_suffix:string -> tezos_job -> tezos_job + +(** Adds artifacts to a job without overriding, if possible, existing artifacts. + + - If the job already has an artifact with [old_name] and [name] is given, then + [name] is used. + - If the job already has an artifact exposed as [old_exposed_as] and + [exposed_as] is given, then [exposed_as] is used. + - If the job already has an artifact added [old_when] and + [when_] is given, the new artifact will be added: + - at [old_when] if [old_when = when_] + - at [Always] if [old_when <> when_] + - If the job already has an artifact that expires in [old_expires_in] and + [expires_in] is given, then the largest of the two durations is used. Note + that for the purpose of this comparison, we consider that a minute is 60 + seconds, that an hour is 60 minutes, that a week is 7 days, that a + month is 31 days, and that a year is 365 days. + - Individual fields of a {!report} cannot be combined: a run-time error is + raised if [reports] contains a report that is already set in the job. + - [paths] are appended to the set of paths in [job]. *) +val add_artifacts : + ?name:string -> + ?expose_as:string -> + ?reports:Gitlab_ci.Types.reports -> + ?expire_in:Gitlab_ci.Types.time_interval -> + ?when_:Gitlab_ci.Types.when_artifact -> + string list -> + tezos_job -> + tezos_job -- GitLab From 21661777a9d2cdb56e5c3f9ea4bc5e9431df1acd Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 09:21:55 +0100 Subject: [PATCH 04/13] CI-in-OCaml: add [Tezos_ci.append_variables] --- ci/bin/tezos_ci.ml | 28 ++++++++++++++++++++++++++++ ci/bin/tezos_ci.mli | 7 +++++++ 2 files changed, 35 insertions(+) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index b56b7a7cdc6a..7b11ee7ffb82 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -501,3 +501,31 @@ let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths job with artifacts = Some {name; expose_as; expire_in; when_; paths; reports}; } + +let append_variables ?(allow_overwrite = false) new_variables + (tezos_job : tezos_job) : tezos_job = + map_job tezos_job @@ fun job -> + let variables = + let old_variables, new_variables = + List.fold_left + (fun (old_variables, new_variables) (name, value) -> + let old_variables = + match List.assoc_opt name old_variables with + | Some old_value -> + if not allow_overwrite then + failwith + "[Tezos_ci.append_variables] attempted to overwrite the \ + variable '%s' (old value: '%s', new value: '%s')" + name + old_value + value ; + List.remove_assoc name old_variables + | None -> old_variables + in + (old_variables, (name, value) :: new_variables)) + (Option.value ~default:[] job.variables, []) + new_variables + in + old_variables @ List.rev new_variables + in + {job with variables = Some variables} diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 997c082832ec..3c47c00cbc85 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -253,3 +253,10 @@ val add_artifacts : string list -> tezos_job -> tezos_job + +(** Append the variables [variables] to the variables of [job]. + + Raises [Failure] if any of the [variables] is already defined for + [job], unless [allow_overwrite] is true (default is [false]). *) +val append_variables : + ?allow_overwrite:bool -> Gitlab_ci.Types.variables -> tezos_job -> tezos_job -- GitLab From 8876ad9279ab13d70f00290907cb9400b714fe86 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 13:04:55 +0100 Subject: [PATCH 05/13] CI: rename variable for job [trigger] --- ci/bin/main.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index ed0aad356f7b..0c8ab50d5d22 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -243,7 +243,7 @@ let job_dummy : job = necessary and the decision to do so belongs to the developer §2: We also perform some fast sanity checks. *) -let trigger = +let job_trigger = job ~__POS__ ~image:Images.alpine @@ -340,7 +340,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) pipelines. So when requested through [needs_trigger] we start this job as early as possible, without waiting for sanity_ci. *) - if needs_trigger then Dependent [Optional trigger] else Staged [] + if needs_trigger then Dependent [Optional job_trigger] else Staged [] in job ?rules @@ -413,7 +413,7 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = let _job_docker_rust_toolchain_before_merging = job_docker_rust_toolchain ~__POS__ - ~dependencies:(Dependent [Optional trigger]) + ~dependencies:(Dependent [Optional job_trigger]) ~rules: [ job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); -- GitLab From cffd3086ba22b020e30ab3decd2740970c04d713 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 12:56:26 +0100 Subject: [PATCH 06/13] CI: inline [master_branch] jobs Makes it a little bit more obvious to where they belong. --- ci/bin/main.ml | 128 ++++++++++++++++++++++++++----------------------- 1 file changed, 67 insertions(+), 61 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 0c8ab50d5d22..86004c62f64f 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -355,8 +355,6 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ~artifacts ["./scripts/ci/build_static_binaries.sh"] -let rules_static_build_master = [job_rule ~when_:Always ()] - let rules_static_build_other = [job_rule ~changes:changeset_octez ()] let _job_static_arm64_experimental = @@ -367,14 +365,6 @@ let _job_static_arm64_experimental = () |> job_external ~filename_suffix:"experimental" -let _job_static_arm64_master = - job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_master - () - |> job_external ~filename_suffix:"master" - let _job_static_x86_64_experimental = job_build_static_binaries ~__POS__ @@ -384,17 +374,6 @@ let _job_static_x86_64_experimental = () |> job_external ~filename_suffix:"experimental" -let _job_static_x86_64_master = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* TODO: this job doesn't actually need trigger and there is no - need to set it optional since we know this job is only on the master branch. *) - ~needs_trigger:true - ~rules:rules_static_build_master - () - |> job_external ~filename_suffix:"master" - let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = job_docker_authenticated ?rules @@ -422,10 +401,6 @@ let _job_docker_rust_toolchain_before_merging = () |> job_external ~filename_suffix:"before_merging" -let _job_docker_rust_toolchain_master = - job_docker_rust_toolchain ~__POS__ ~rules:[job_rule ~when_:Always ()] () - |> job_external ~filename_suffix:"master" - let _job_docker_rust_toolchain_other = job_docker_rust_toolchain ~__POS__ () |> job_external ~filename_suffix:"other" @@ -534,17 +509,6 @@ let rules_octez_docker_changes_or_master = job_rule ~changes:changeset_octez_docker_changes_or_master (); ] -let job_docker_amd64_experimental : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true - (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. - For now we can set any variant to get the correct name of the need. *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Amd64 - Experimental - let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ @@ -554,15 +518,6 @@ let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = ~arch:Amd64 Test_manual -let job_docker_arm64_experimental : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Arm64 - Experimental - let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ @@ -589,21 +544,6 @@ let job_docker_merge_manifests ~__POS__ ~ci_docker_hub ~job_docker_amd64 ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] ["./scripts/ci/docker_merge_manifests.sh"] -let _job_docker_merge_manifests_release = - job_docker_merge_manifests - ~__POS__ - ~ci_docker_hub:true - (* TODO: In theory, actually uses either release or - experimental variant of docker jobs depending on - pipeline. In practice, this does not matter as these jobs - have the same name in the generated files - ([oc.build:ARCH]). However, when the merge_manifest jobs - are created directly in the appropriate pipeline, the - correcty variant must be used. *) - ~job_docker_amd64:job_docker_amd64_experimental - ~job_docker_arm64:job_docker_arm64_experimental - |> job_external ~filename_suffix:"release" - type bin_package_target = Dpkg | Rpm let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch @@ -869,7 +809,73 @@ let () = "latest_release_test" If.(not_on_tezos_namespace && push && on_branch "latest-release-test") ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:false] ; - register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; + register + "master_branch" + If.(on_tezos_namespace && push && on_branch "master") + ~jobs: + (let job_docker_rust_toolchain = + job_docker_rust_toolchain + ~__POS__ + ~rules:[job_rule ~when_:Always ()] + () + |> job_external ~filename_suffix:"master" + in + let job_docker_amd64_experimental : tezos_job = + job_docker_build + ~__POS__ + ~external_:true + (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. + For now we can set any variant to get the correct name of the need. *) + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Experimental + in + let job_docker_arm64_experimental : tezos_job = + job_docker_build + ~__POS__ + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Experimental + in + let _job_docker_merge_manifests = + job_docker_merge_manifests + ~__POS__ + ~ci_docker_hub:true + (* TODO: In theory, actually uses either release or + experimental variant of docker jobs depending on + pipeline. In practice, this does not matter as these jobs + have the same name in the generated files + ([oc.build:ARCH]). However, when the merge_manifest jobs + are created directly in the appropriate pipeline, the + correcty variant must be used. *) + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + |> job_external ~filename_suffix:"release" + in + let _job_static_arm64 = + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~rules:[job_rule ~when_:Always ()] + () + |> job_external ~filename_suffix:"master" + in + let _job_static_x86_64 = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* TODO: this job doesn't actually need trigger and there is no + need to set it optional since we know this job is only on the master branch. *) + ~needs_trigger:true + ~rules:[job_rule ~when_:Always ()] + () + |> job_external ~filename_suffix:"master" + in + (* The empty list is a placeholder until the full pipeline is generated *) + []) ; register "release_tag" If.(on_tezos_namespace && push && has_tag_match release_tag_re) -- GitLab From 6ddac01af723568188fdbbdc4f7ece772f03f727 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 10 Jan 2024 16:56:35 +0100 Subject: [PATCH 07/13] CI: generate build_arm64 jobs --- .../build/oc.build_arm64-exp-dev-extra.yml | 49 +++++-- .../ci/jobs/build/oc.build_arm64-released.yml | 44 ++++-- ci/bin/main.ml | 129 ++++++++++++++++++ 3 files changed, 203 insertions(+), 19 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml index 01a9c9408876..c6f4f4db5b28 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml @@ -1,14 +1,41 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# The build_arm64 jobs are split in two to keep the artifact size -# under the 1GB hard limit set by GitLab. - -# 'oc.build_arm64-exp-dev-extra' builds the developer and experimental -# executables, as well as the tezt test suite and the TPS evaluation -# tool. oc.build_arm64-exp-dev-extra: - extends: - - .oc.build_arm64 + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh variables: - EXECUTABLE_FILES: "script-inputs/experimental-executables script-inputs/dev-executables" - BUILD_EXTRA: "src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe tezt/tests/main.exe" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released.yml index 4c947e6f3edc..287950d42cd9 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released.yml @@ -1,11 +1,39 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# The build_arm64 jobs are split in two to keep the artifact size -# under the 1GB hard limit set by GitLab. - -# 'oc.build_arm64-released' builds the released executables. oc.build_arm64-released: - extends: - - .oc.build_arm64 + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 86004c62f64f..03a205596bd5 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -207,6 +207,17 @@ let before_script ?(take_ownership = false) ?(source_version = false) @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" @ before_script +(** Add variable for bisect_ppx instrumentation. + + This template should be extended by jobs that build OCaml targets + that should be instrumented for coverage output. This set of job + includes build jobs (like [oc.build_x86_64_*]). It also includes + OCaml unit test jobs like [oc.unit:*-x86_64] as they build the test + runners before their execution. *) +let enable_coverage_instrumentation : tezos_job -> tezos_job = + Tezos_ci.append_variables + [("COVERAGE_OPTIONS", "--instrument-with bisect_ppx")] + let changeset_octez = [ "src/**/*"; @@ -781,6 +792,124 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = | false, Release_tag -> [job_opam_release] | _ -> [] +let arm64_build_extra = + [ + "src/bin_tps_evaluation/main_tps_evaluation.exe"; + "src/bin_octogram/octogram_main.exe tezt/tests/main.exe"; + ] + +let amd64_build_extra = + [ + "src/bin_tps_evaluation/main_tps_evaluation.exe"; + "src/bin_octogram/octogram_main.exe"; + "tezt/tests/main.exe"; + "contrib/octez_injector_server/octez_injector_server.exe"; + ] + +let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) + ?(needs_trigger = false) () = + let arch_string = match arch with Amd64 -> "x86_64" | Arm64 -> "arm64" in + let name = + sf + "oc.build_%s-%s" + arch_string + (if release then "released" else "exp-dev-extra") + in + let executable_files = + if release then "script-inputs/released-executables" + else "script-inputs/experimental-executables script-inputs/dev-executables" + in + let build_extra = + match (release, arch) with + | true, _ -> None + | false, Amd64 -> Some amd64_build_extra + | false, Arm64 -> Some arm64_build_extra + in + let variables = + [("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + @ + match build_extra with + | Some build_extra -> [("BUILD_EXTRA", String.concat " " build_extra)] + | None -> [] + in + let artifacts = + artifacts + ~name:"build-$ARCH-$CI_COMMIT_REF_SLUG" + ~when_:On_success + ~expire_in:(Days 1) + (* TODO: [paths] can be refined based on [release] *) + [ + "octez-*"; + "src/proto_*/parameters/*.json"; + "_build/default/src/lib_protocol_compiler/bin/main_native.exe"; + "_build/default/tezt/tests/main.exe"; + "_build/default/contrib/octez_injector_server/octez_injector_server.exe"; + ] + in + let dependencies = + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration of + pipelines. So when requested through [needs_trigger] we start + this job as early as possible, without waiting for + sanity_ci. *) + if needs_trigger then Dependent [Optional job_trigger] else Staged [] + in + let job = + job + ?rules + ~__POS__ + ~stage:Stages.build + ~arch + ~name + ~image:Images.runtime_build_dependencies + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~variables + ~dependencies + ~artifacts + ["./scripts/ci/build_full_unreleased.sh"] + in + (* Disable coverage for arm64 *) + if arch = Amd64 then enable_coverage_instrumentation job else job + +let build_arm_rules = + [ + job_rule ~if_:Rules.schedule_extended_tests ~when_:Always (); + job_rule ~if_:Rules.(has_mr_label "ci--arm64") ~when_:On_success (); + job_rule + ~changes:["src/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"] + ~when_:Manual + ~allow_failure:Yes + (); + ] + +(* Write external files for build_arm64_jobs. + + Used in external pipelines [before_merging] and [schedule_extended_test]. *) +let _job_build_arm64_release : tezos_job = + job_build_dynamic_binaries + ~__POS__ + ~arch:Arm64 + ~needs_trigger:false + ~release:true + ~rules:build_arm_rules + () + |> job_external + +let _job_build_arm64_exp_dev_extra : tezos_job = + job_build_dynamic_binaries + ~__POS__ + ~arch:Arm64 + ~needs_trigger:false + ~release:false + ~rules:build_arm_rules + () + |> job_external + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From 3d0e479137ec66852693cbe797f5a7cce2536655 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 08:12:47 +0100 Subject: [PATCH 08/13] CI: generate [oc.unified_coverage-default.yml] --- .../coverage/oc.unified_coverage-default.yml | 50 +++++++++------- ci/bin/main.ml | 58 ++++++++++++++++++- 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml b/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml index 9a57fa61af70..40d4195f5798 100644 --- a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml +++ b/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml @@ -1,29 +1,35 @@ -# This job fetches coverage files from the most recently merged MR on the default branch. -# It creates the html, summary and cobertura reports. It also provide a coverage % for the -# merge request. - -include: .gitlab/ci/jobs/coverage/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.unified_coverage: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .oc.template__coverage_report + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test_coverage + tags: + - gcp dependencies: [] + allow_failure: true + before_script: + - . ./scripts/version.sh + script: + - mkdir -p _coverage_report + - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline + --info --log-file _coverage_report/download_coverage.log + - ./scripts/ci/report_coverage.sh variables: - # The GitLab project to use in the coverage downloader. PROJECT: $CI_PROJECT_PATH - # The default branch where the coverage downloader will search for - # merge commits. DEFAULT_BRANCH: $CI_COMMIT_SHA - before_script: - - . ./scripts/version.sh # sets COVERAGE_OUTPUT - script: - # On the project default branch, we fetch coverage from the last merged MR - - mkdir -p _coverage_report - - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline --info --log-file _coverage_report/download_coverage.log - - ./scripts/ci/report_coverage.sh - # This job will fail if coverage is not found, but that is not - # reason to stop the pipeline. - allow_failure: true + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + SLACK_COVERAGE_CHANNEL: C02PHBE7W73 + artifacts: + expire_in: 15 days + paths: + - _coverage_report/ + - $BISECT_FILE + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + when: always + expose_as: Coverage report when: always + coverage: '/Coverage: ([^%]+%)/' diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 03a205596bd5..160c4957cd5f 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -33,7 +33,7 @@ module Stages = struct let _test = Stage.register "test" - let _test_coverage = Stage.register "test_coverage" + let test_coverage = Stage.register "test_coverage" let _packaging = Stage.register "packaging" @@ -218,6 +218,17 @@ let enable_coverage_instrumentation : tezos_job -> tezos_job = Tezos_ci.append_variables [("COVERAGE_OPTIONS", "--instrument-with bisect_ppx")] +(** Add variable specifying coverage trace storage. + + This function should be applied to jobs that either produce (like + test jobs) or consume (like the [unified_coverage] job) coverage + traces. In addition to specifying the location of traces, setting + this variable also _enables_ coverage trace output for + instrumented binaries. *) +let enable_coverage_location : tezos_job -> tezos_job = + Tezos_ci.append_variables + [("BISECT_FILE", "$CI_PROJECT_DIR/_coverage_output/")] + let changeset_octez = [ "src/**/*"; @@ -910,6 +921,23 @@ let _job_build_arm64_exp_dev_extra : tezos_job = () |> job_external +let enable_coverage_report job : tezos_job = + job + |> Tezos_ci.add_artifacts + ~expose_as:"Coverage report" + ~reports: + (reports + ~coverage_report: + { + coverage_format = Cobertura; + path = "_coverage_report/cobertura.xml"; + } + ()) + ~expire_in:(Days 15) + ~when_:Always + ["_coverage_report/"; "$BISECT_FILE"] + |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -1003,6 +1031,34 @@ let () = () |> job_external ~filename_suffix:"master" in + let _job_unified_coverage_default : tezos_job = + job + ~__POS__ + ~image:Images.runtime_build_test_dependencies + ~name:"oc.unified_coverage" + ~stage:Stages.test_coverage + ~variables: + [ + ("PROJECT", Predefined_vars.(show ci_project_path)); + ("DEFAULT_BRANCH", Predefined_vars.(show ci_commit_sha)); + ] + ~allow_failure:Yes + ~before_script: + ((* sets COVERAGE_OUTPUT *) + before_script ~source_version:true []) + ~when_:Always + ~coverage:"/Coverage: ([^%]+%)/" + [ + (* On the project default branch, we fetch coverage from the last merged MR *) + "mkdir -p _coverage_report"; + "dune exec scripts/ci/download_coverage/download.exe -- -a \ + from=last-merged-pipeline --info --log-file \ + _coverage_report/download_coverage.log"; + "./scripts/ci/report_coverage.sh"; + ] + |> enable_coverage_location |> enable_coverage_report + |> job_external ~directory:"coverage" ~filename_suffix:"default" + in (* The empty list is a placeholder until the full pipeline is generated *) []) ; register -- GitLab From c2d277aaea27e417e80098dedc4d46cd8f796512 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 10:46:04 +0100 Subject: [PATCH 09/13] CI: generate [publish:documentation.yml] --- .gitlab/ci/jobs/doc/publish:documentation.yml | 51 ++++++++++--------- ci/bin/main.ml | 40 ++++++++++++++- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/.gitlab/ci/jobs/doc/publish:documentation.yml b/.gitlab/ci/jobs/doc/publish:documentation.yml index 0b92cdda5521..78d2a443bc58 100644 --- a/.gitlab/ci/jobs/doc/publish:documentation.yml +++ b/.gitlab/ci/jobs/doc/publish:documentation.yml @@ -1,30 +1,33 @@ -include: .gitlab/ci/jobs/shared/templates.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# here we use this hack to publish the tezos documentation on -# gitlab.io because we want to publish the doc for the project -# tezos under https://tezos.gitlab.io and not https://tezos.gitlab.io/tezos -# The latter follows the gitlab url convention of -# https://.gitlab.io// -# Notice that we push only if CI_COMMIT_REF_NAME is really master . -# This allows to test the release workflow publish:documentation: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .rules__octez_docs_changes + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} stage: doc - # Make the publish_documentation run in the beginning of the master - # pipeline to ensure it has time to run before the next merge. + tags: + - gcp + rules: + - changes: + - scripts/**/*/ + - script-inputs/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - docs/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success needs: [] + dependencies: [] + interruptible: false before_script: - - eval $(opam env) - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 - - echo "${CI_KH}" > ~/.ssh/known_hosts - - chmod 400 ~/.ssh/id_ed25519 + - eval $(opam env) + - . $HOME/.venv/bin/activate + - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 + - echo "${CI_KH}" > ~/.ssh/known_hosts + - chmod 400 ~/.ssh/id_ed25519 script: - - ./scripts/ci/doc_publish.sh - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false + - ./scripts/ci/doc_publish.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 160c4957cd5f..6dfdaf3247f9 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -37,7 +37,7 @@ module Stages = struct let _packaging = Stage.register "packaging" - let _doc = Stage.register "doc" + let doc = Stage.register "doc" let prepare_release = Stage.register "prepare_release" @@ -244,6 +244,21 @@ let changeset_octez_or_kernels = ["images/**/*"; "scripts/ci/**/*"; "kernels.mk"; "etherlink.mk"] @ changeset_octez +let changeset_octez_docs = + [ + "scripts/**/*/"; + "script-inputs/**/*/"; + "src/**/*"; + "tezt/**/*"; + "vendors/**/*"; + "dune"; + "dune-project"; + "dune-workspace"; + "docs/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (* Dummy job. This fixes the "configuration must contain at least one @@ -1059,6 +1074,29 @@ let () = |> enable_coverage_location |> enable_coverage_report |> job_external ~directory:"coverage" ~filename_suffix:"default" in + let _job_publish_documentation : tezos_job = + job + ~__POS__ + ~name:"publish:documentation" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:(Dependent []) + ~before_script: + (before_script + ~eval_opam:true + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables. *) + ~init_python_venv:true + [ + {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; + {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; + {|chmod 400 ~/.ssh/id_ed25519|}; + ]) + ~interruptible:false + ~rules:[job_rule ~changes:changeset_octez_docs ~when_:On_success ()] + ["./scripts/ci/doc_publish.sh"] + |> job_external + in (* The empty list is a placeholder until the full pipeline is generated *) []) ; register -- GitLab From bc0dc6fec736d3b1086f9d972b565f608c556ec8 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 13:41:48 +0100 Subject: [PATCH 10/13] CI: generate job [publish_kernel_sdk] --- .../ci/jobs/publish/publish_kernel_sdk.yml | 31 +++++++++-------- ci/bin/main.ml | 34 ++++++++++++++++++- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml b/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml index 32fd14baae48..369eda3cf42d 100644 --- a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml +++ b/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml @@ -1,25 +1,26 @@ -# -# Smart Rollup: Kernel SDK -# -# See src/kernel_sdk/RELEASE.md for more information +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. publish_kernel_sdk: - extends: - - .default_settings_template - - .image_template__rust_toolchain - - .needs__rust_toolchain + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: manual + tags: + - gcp rules: - when: manual + allow_failure: false + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain allow_failure: true + cache: + key: kernels + paths: + - cargo/ interruptible: false script: - - make -f kernels.mk publish-sdk-deps - # Manually set SSL_CERT_DIR as default setting points to empty dir - - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk + - make -f kernels.mk publish-sdk-deps + - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk variables: CARGO_HOME: $CI_PROJECT_DIR/cargo - cache: - - key: kernels - paths: - - cargo/ diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 6dfdaf3247f9..cb9ba846116d 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -150,7 +150,7 @@ module Images = struct ~image_path: "${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag}" - let _rust_toolchain = + let rust_toolchain = (* Warning: we are relying on ill-specified behavior from GitLab that allows the expansion of dotenv variables (here: $rust_toolchain_image_tag) in the image field. @@ -1097,6 +1097,38 @@ let () = ["./scripts/ci/doc_publish.sh"] |> job_external in + (* Smart Rollup: Kernel SDK + + See [src/kernel_sdk/RELEASE.md] for more information. *) + let _job_publish_kernel_sdk : tezos_job = + job + ~__POS__ + ~name:"publish_kernel_sdk" + ~image:Images.rust_toolchain + ~stage:Stages.manual + ~rules: + [ + (* This job is in the last stage {!Stages.manual} so we + can disallow failure without blocking the pipeline. + Furthermore, unlike other manual jobs, this is not + an "optional" job for which failures are + tolerated. *) + job_rule ~when_:Manual ~allow_failure:No (); + ] + ~allow_failure:Yes + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~interruptible:false + ~variables: + [("CARGO_HOME", Predefined_vars.(show ci_project_dir) // "cargo")] + ~cache:[{key = "kernels"; paths = ["cargo/"]}] + [ + "make -f kernels.mk publish-sdk-deps"; + (* Manually set SSL_CERT_DIR as default setting points to empty dir *) + "SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk \ + publish-sdk"; + ] + |> job_external ~directory:"publish" + in (* The empty list is a placeholder until the full pipeline is generated *) []) ; register -- GitLab From 1bfa1e253e905183c7a8d10d954664b776c28cc3 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 13:58:26 +0100 Subject: [PATCH 11/13] CI: generate [master_branch] pipeline --- .gitlab/ci/pipelines/master_branch.yml | 353 +++++++++++++++++++++++-- ci/bin/main.ml | 47 ++-- 2 files changed, 358 insertions(+), 42 deletions(-) diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 4f28d8a75d5a..5ba80ed29911 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -1,21 +1,332 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml - - # Stage: test_coverage - - .gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml - - # Stage: doc - - .gitlab/ci/jobs/doc/publish:documentation.yml - - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: manual - - .gitlab/ci/jobs/publish/publish_kernel_sdk.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env + +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - when: always + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* + +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* + +oc.build_arm64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success + +oc.build_arm64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success + +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +oc.unified_coverage: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test_coverage + tags: + - gcp + dependencies: [] + allow_failure: true + before_script: + - . ./scripts/version.sh + script: + - mkdir -p _coverage_report + - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline + --info --log-file _coverage_report/download_coverage.log + - ./scripts/ci/report_coverage.sh + variables: + PROJECT: $CI_PROJECT_PATH + DEFAULT_BRANCH: $CI_COMMIT_SHA + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + SLACK_COVERAGE_CHANNEL: C02PHBE7W73 + artifacts: + expire_in: 15 days + paths: + - _coverage_report/ + - $BISECT_FILE + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + when: always + expose_as: Coverage report + when: always + coverage: '/Coverage: ([^%]+%)/' + +publish:documentation: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - changes: + - scripts/**/*/ + - script-inputs/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - docs/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: [] + dependencies: [] + interruptible: false + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 + - echo "${CI_KH}" > ~/.ssh/known_hosts + - chmod 400 ~/.ssh/id_ed25519 + script: + - ./scripts/ci/doc_publish.sh + +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" + +publish_kernel_sdk: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} + stage: manual + tags: + - gcp + rules: + - when: manual + allow_failure: false + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + cache: + key: kernels + paths: + - cargo/ + interruptible: false + script: + - make -f kernels.mk publish-sdk-deps + - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk + variables: + CARGO_HOME: $CI_PROJECT_DIR/cargo diff --git a/ci/bin/main.ml b/ci/bin/main.ml index cb9ba846116d..d2316aaf4d59 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -916,7 +916,7 @@ let build_arm_rules = (* Write external files for build_arm64_jobs. Used in external pipelines [before_merging] and [schedule_extended_test]. *) -let _job_build_arm64_release : tezos_job = +let job_build_arm64_release : tezos_job = job_build_dynamic_binaries ~__POS__ ~arch:Arm64 @@ -926,7 +926,7 @@ let _job_build_arm64_release : tezos_job = () |> job_external -let _job_build_arm64_exp_dev_extra : tezos_job = +let job_build_arm64_exp_dev_extra : tezos_job = job_build_dynamic_binaries ~__POS__ ~arch:Arm64 @@ -990,14 +990,10 @@ let () = ~__POS__ ~rules:[job_rule ~when_:Always ()] () - |> job_external ~filename_suffix:"master" in let job_docker_amd64_experimental : tezos_job = job_docker_build ~__POS__ - ~external_:true - (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. - For now we can set any variant to get the correct name of the need. *) ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) ~rules:rules_octez_docker_changes_or_master ~arch:Amd64 @@ -1006,13 +1002,12 @@ let () = let job_docker_arm64_experimental : tezos_job = job_docker_build ~__POS__ - ~external_:true (* TODO: see above *) ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) ~rules:rules_octez_docker_changes_or_master ~arch:Arm64 Experimental in - let _job_docker_merge_manifests = + let job_docker_merge_manifests = job_docker_merge_manifests ~__POS__ ~ci_docker_hub:true @@ -1025,17 +1020,15 @@ let () = correcty variant must be used. *) ~job_docker_amd64:job_docker_amd64_experimental ~job_docker_arm64:job_docker_arm64_experimental - |> job_external ~filename_suffix:"release" in - let _job_static_arm64 = + let job_static_arm64 = job_build_static_binaries ~__POS__ ~arch:Arm64 ~rules:[job_rule ~when_:Always ()] () - |> job_external ~filename_suffix:"master" in - let _job_static_x86_64 = + let job_static_x86_64 = job_build_static_binaries ~__POS__ ~arch:Amd64 @@ -1044,9 +1037,8 @@ let () = ~needs_trigger:true ~rules:[job_rule ~when_:Always ()] () - |> job_external ~filename_suffix:"master" in - let _job_unified_coverage_default : tezos_job = + let job_unified_coverage_default : tezos_job = job ~__POS__ ~image:Images.runtime_build_test_dependencies @@ -1072,9 +1064,8 @@ let () = "./scripts/ci/report_coverage.sh"; ] |> enable_coverage_location |> enable_coverage_report - |> job_external ~directory:"coverage" ~filename_suffix:"default" in - let _job_publish_documentation : tezos_job = + let job_publish_documentation : tezos_job = job ~__POS__ ~name:"publish:documentation" @@ -1095,12 +1086,11 @@ let () = ~interruptible:false ~rules:[job_rule ~changes:changeset_octez_docs ~when_:On_success ()] ["./scripts/ci/doc_publish.sh"] - |> job_external in (* Smart Rollup: Kernel SDK See [src/kernel_sdk/RELEASE.md] for more information. *) - let _job_publish_kernel_sdk : tezos_job = + let job_publish_kernel_sdk : tezos_job = job ~__POS__ ~name:"publish_kernel_sdk" @@ -1127,10 +1117,25 @@ let () = "SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk \ publish-sdk"; ] - |> job_external ~directory:"publish" in - (* The empty list is a placeholder until the full pipeline is generated *) - []) ; + [ + (* Stage: build *) + job_docker_rust_toolchain; + job_static_x86_64; + job_static_arm64; + job_build_arm64_release; + job_build_arm64_exp_dev_extra; + job_docker_amd64_experimental; + job_docker_arm64_experimental; + (* Stage: test_coverage *) + job_unified_coverage_default; + (* Stage: doc *) + job_publish_documentation; + (* Stage: prepare_release *) + job_docker_merge_manifests; + (* Stage: manual *) + job_publish_kernel_sdk; + ]) ; register "release_tag" If.(on_tezos_namespace && push && has_tag_match release_tag_re) -- GitLab From 4a92749ecdf6a99528001ea8966f90062e1b900b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 5 Mar 2024 14:24:31 +0100 Subject: [PATCH 12/13] CI: Remove unused external files --- ...ild:static-arm64-linux-binaries-master.yml | 22 ---------- ...ld:static-x86_64-linux-binaries-master.yml | 25 ----------- .../build/oc.docker:amd64-experimental.yml | 44 ------------------- .../build/oc.docker:arm64-experimental.yml | 44 ------------------- .../build/oc.docker:rust-toolchain-master.yml | 22 ---------- .../coverage/oc.unified_coverage-default.yml | 35 --------------- .gitlab/ci/jobs/doc/publish:documentation.yml | 33 -------------- .../docker:merge_manifests-release.yml | 21 --------- .../ci/jobs/publish/publish_kernel_sdk.yml | 26 ----------- 9 files changed, 272 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml delete mode 100644 .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml delete mode 100644 .gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml delete mode 100644 .gitlab/ci/jobs/doc/publish:documentation.yml delete mode 100644 .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml delete mode 100644 .gitlab/ci/jobs/publish/publish_kernel_sdk.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml deleted file mode 100644 index 8633e3c6de5f..000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-arm64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp_arm64 - rules: - - when: always - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - artifacts: - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml deleted file mode 100644 index 8e721cb125d4..000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-x86_64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp - rules: - - when: always - needs: - - job: trigger - optional: true - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - artifacts: - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml deleted file mode 100644 index 4b1fc6f26702..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:amd64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: with-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml deleted file mode 100644 index 31ae139c30ed..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:arm64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp_arm64 - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml deleted file mode 100644 index 6bce59a27e94..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:rust-toolchain: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - when: always - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml b/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml deleted file mode 100644 index 40d4195f5798..000000000000 --- a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.unified_coverage: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} - stage: test_coverage - tags: - - gcp - dependencies: [] - allow_failure: true - before_script: - - . ./scripts/version.sh - script: - - mkdir -p _coverage_report - - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline - --info --log-file _coverage_report/download_coverage.log - - ./scripts/ci/report_coverage.sh - variables: - PROJECT: $CI_PROJECT_PATH - DEFAULT_BRANCH: $CI_COMMIT_SHA - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - SLACK_COVERAGE_CHANNEL: C02PHBE7W73 - artifacts: - expire_in: 15 days - paths: - - _coverage_report/ - - $BISECT_FILE - reports: - coverage_report: - coverage_format: cobertura - path: _coverage_report/cobertura.xml - when: always - expose_as: Coverage report - when: always - coverage: '/Coverage: ([^%]+%)/' diff --git a/.gitlab/ci/jobs/doc/publish:documentation.yml b/.gitlab/ci/jobs/doc/publish:documentation.yml deleted file mode 100644 index 78d2a443bc58..000000000000 --- a/.gitlab/ci/jobs/doc/publish:documentation.yml +++ /dev/null @@ -1,33 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -publish:documentation: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} - stage: doc - tags: - - gcp - rules: - - changes: - - scripts/**/*/ - - script-inputs/**/*/ - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - docs/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: [] - dependencies: [] - interruptible: false - before_script: - - eval $(opam env) - - . $HOME/.venv/bin/activate - - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 - - echo "${CI_KH}" > ~/.ssh/known_hosts - - chmod 400 ~/.ssh/id_ed25519 - script: - - ./scripts/ci/doc_publish.sh diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml deleted file mode 100644 index d1318ef9c8b4..000000000000 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -docker:merge_manifests: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "true" diff --git a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml b/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml deleted file mode 100644 index 369eda3cf42d..000000000000 --- a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml +++ /dev/null @@ -1,26 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -publish_kernel_sdk: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - stage: manual - tags: - - gcp - rules: - - when: manual - allow_failure: false - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - allow_failure: true - cache: - key: kernels - paths: - - cargo/ - interruptible: false - script: - - make -f kernels.mk publish-sdk-deps - - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk - variables: - CARGO_HOME: $CI_PROJECT_DIR/cargo -- GitLab From a6d2237c0e43583a3c6e540ec4f52d9cf899a432 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 15:00:41 +0100 Subject: [PATCH 13/13] CI: remove unused templates --- .gitlab/ci/jobs/build/common.yml | 84 -------------------------------- 1 file changed, 84 deletions(-) diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml index 063dbd7d87ff..98a8b9b274ec 100644 --- a/.gitlab/ci/jobs/build/common.yml +++ b/.gitlab/ci/jobs/build/common.yml @@ -17,61 +17,6 @@ expire_in: 1 day when: on_success -.oc.build_arm64: - extends: - - .oc.build - - .tags_template__build_arm64 - variables: - ARCH: "arm64" - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/' - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual - allow_failure: true - -# similar to the build template above, this template -# compiles all binaries using the static profile. -.oc.build_static_binaries_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - stage: build - variables: - ARCH: "" - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - artifacts: - paths: - - "octez-binaries/$ARCH/*" - -.oc.docker:rust-toolchain_template: - extends: - - .image_template__docker - - .docker_auth_template - stage: build - variables: - # These image are not built for external use. - CI_DOCKER_HUB: "false" - # Handle docker initialization, if necessary, in - # './scripts/ci/docker_rust_toolchain_build.sh'. - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env - .oc.docker:client-libs-dependencies_template: extends: - .image_template__docker @@ -89,17 +34,6 @@ reports: dotenv: client_libs_dependencies_image_tag.env -.oc.build_docker_release_template: - extends: - - .image_template__docker - - .docker_auth_template - - .needs__rust_toolchain - variables: - DOCKER_BUILD_TARGET: "without-evm-artifacts" - IMAGE_ARCH_PREFIX: "" - script: - - ./scripts/ci/docker_release.sh - .oc.build_x86_64: extends: - .oc.build @@ -110,21 +44,3 @@ - .needs__trigger variables: ARCH: "x86_64" - -.oc.build:static-x86_64-linux-binaries: - extends: - - .tags_template__build - - .oc.build_static_binaries_template - # Even though not many tests depend on static executables, some of those that do - # are limiting factors in the total duration of pipelines. - # So we start this job as early as possible, without waiting for sanity_ci. - - .needs__trigger - variables: - ARCH: "x86_64" - -.oc.build:static-arm64-linux-binaries: - extends: - - .oc.build_static_binaries_template - - .tags_template__build_arm64 - variables: - ARCH: "arm64" -- GitLab