From 177203b06bfcb8e248140fcdb2d8ed6042ede434 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 5 Jan 2024 19:48:17 +0100 Subject: [PATCH 01/15] CI-in-OCaml: allow the registration of external jobs --- ci/bin/tezos_ci.ml | 29 +++++++++++++++++++++++++++++ ci/bin/tezos_ci.mli | 16 ++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 5fe34f097aa2..4ffd1e41b742 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -326,3 +326,32 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache retry; parallel; } + +let external_jobs = ref String_set.empty + +let job_external ?directory ?filename_suffix (job : Gitlab_ci.Types.job) : + Gitlab_ci.Types.job = + let stage = + match job.stage with + | Some stage -> stage + | None -> + (* Test is the name of the default stage in GitLab CI *) + "test" + in + let basename = + match filename_suffix with + | None -> job.name + | Some suffix -> job.name ^ "-" ^ suffix + in + let directory = Option.value ~default:stage directory in + let filename = sf ".gitlab/ci/jobs/%s/%s.yml" directory basename in + if String_set.mem filename !external_jobs then + failwith + "Attempted to write external job %s twice -- perhaps you need to set \ + filename_suffix?" + filename + else ( + external_jobs := String_set.add filename !external_jobs ; + let config = [Gitlab_ci.Types.Job job] in + Gitlab_ci.To_yaml.to_file ~header ~filename config ; + job) diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 50721847630d..d65d1836ce11 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -186,3 +186,19 @@ val job : name:string -> string list -> Gitlab_ci.Types.job + +(** Generates a job to an external file. + + This function is meant to be used in the transition to CI-in-OCaml. + It writes {!header} and the given job to the file + [.gitlab/ci/jobs/DIRECTORY/NAME(-FILENAME_SUFFIX).yml]. + Directory defaults to the stage name if not set. + + This allows migrating all the jobs of a given pipeline, and + including the generated definition of those jobs in other + pipelines where it appears. *) +val job_external : + ?directory:string -> + ?filename_suffix:string -> + Gitlab_ci.Types.job -> + Gitlab_ci.Types.job -- GitLab From 2685dc27ae50c6c4fa01573bedcbaec3805737c5 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 22 Feb 2024 13:45:02 +0100 Subject: [PATCH 02/15] CI-in-OCaml: do not allow empty lists in [artifacts:paths] This is not allowed by the gitlab-ci JSON schema. --- ci/bin/tezos_ci.ml | 2 +- ci/lib_gitlab_ci/to_yaml.ml | 2 +- ci/lib_gitlab_ci/types.ml | 2 +- ci/lib_gitlab_ci/util.ml | 9 ++++++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 4ffd1e41b742..e171dae28093 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -114,7 +114,7 @@ module Pipeline = struct (* Check that all [dependencies:] are on jobs that produce artifacts *) ( Fun.flip String_set.iter dependencies @@ fun dependency -> match Hashtbl.find_opt job_by_name dependency with - | Some {artifacts = Some {paths = _ :: _; _}; _} + | Some {artifacts = Some {paths = Some (_ :: _); _}; _} | Some {artifacts = Some {reports = Some {dotenv = Some _; _}; _}; _} -> (* This is fine: we depend on a job that define non-report artifacts, or a dotenv file. *) () diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 97cdd86df5a1..feab427704ff 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -158,7 +158,7 @@ let enc_artifacts : artifacts -> value = [ opt "name" string name; opt "expire_in" enc_time_interval expire_in; - key "paths" strings paths; + opt "paths" strings paths; opt "reports" enc_report reports; opt "when" enc_when_artifact when_; opt "expose_as" string expose_as; diff --git a/ci/lib_gitlab_ci/types.ml b/ci/lib_gitlab_ci/types.ml index dbc6fd1db684..54e28eb412c6 100644 --- a/ci/lib_gitlab_ci/types.ml +++ b/ci/lib_gitlab_ci/types.ml @@ -78,7 +78,7 @@ type when_artifact = Always | On_success | On_failure type artifacts = { expire_in : time_interval option; - paths : string list; + paths : string list option; reports : reports option; when_ : when_artifact option; expose_as : string option; diff --git a/ci/lib_gitlab_ci/util.ml b/ci/lib_gitlab_ci/util.ml index 62a3c9e62949..1a4a1389373c 100644 --- a/ci/lib_gitlab_ci/util.ml +++ b/ci/lib_gitlab_ci/util.ml @@ -65,7 +65,14 @@ let artifacts ?expire_in ?reports ?when_ ?expose_as ?name paths = "Attempted to register an artifact with no reports or paths -- this \ doesn't make any sense" | _ -> ()) ; - {expire_in; paths; reports; when_; expose_as; name} + { + expire_in; + paths = (if paths = [] then None else Some paths); + reports; + when_; + expose_as; + name; + } let reports ?dotenv ?junit ?coverage_report () = (match (dotenv, junit, coverage_report) with -- GitLab From 1812b59415bc70acd7e720c5a98e4f1bf1e3ab11 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 15:06:39 +0100 Subject: [PATCH 03/15] CI-in-OCaml: improve error messages in [Tezos_ci.job_external] --- ci/bin/tezos_ci.ml | 10 ++++++++-- ci/bin/tezos_ci.mli | 8 ++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index e171dae28093..84d2f09a0584 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -343,8 +343,14 @@ let job_external ?directory ?filename_suffix (job : Gitlab_ci.Types.job) : | None -> job.name | Some suffix -> job.name ^ "-" ^ suffix in - let directory = Option.value ~default:stage directory in - let filename = sf ".gitlab/ci/jobs/%s/%s.yml" directory basename in + let directory = ".gitlab/ci/jobs" // Option.value ~default:stage directory in + if not (Sys.file_exists directory && Sys.is_directory directory) then + failwith + "[job_external] attempted to write job '%s' to non-existing directory \ + '%s'" + job.name + directory ; + let filename = (directory // basename) ^ ".yml" in if String_set.mem filename !external_jobs then failwith "Attempted to write external job %s twice -- perhaps you need to set \ diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index d65d1836ce11..76da952ff890 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -190,13 +190,17 @@ val job : (** Generates a job to an external file. This function is meant to be used in the transition to CI-in-OCaml. - It writes {!header} and the given job to the file + It writes {!header} and the given job to the destination path [.gitlab/ci/jobs/DIRECTORY/NAME(-FILENAME_SUFFIX).yml]. Directory defaults to the stage name if not set. This allows migrating all the jobs of a given pipeline, and including the generated definition of those jobs in other - pipelines where it appears. *) + pipelines where it appears. + + Raises [Failure] if [.gitlab/ci/jobs/DIRECTORY] is not an existing + directory. Also [Failure] if destination path has already been + used to write another job. *) val job_external : ?directory:string -> ?filename_suffix:string -> -- GitLab From 013a2923ca8189bcf358ac9e8b6f82e0f12dc080 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 9 Feb 2024 15:35:37 +0100 Subject: [PATCH 04/15] CI: generate job [trigger] --- .gitlab/ci/jobs/trigger/trigger.yml | 30 +++++++++-------------- ci/bin/main.ml | 38 +++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.gitlab/ci/jobs/trigger/trigger.yml b/.gitlab/ci/jobs/trigger/trigger.yml index 715b589f7712..164632e04224 100644 --- a/.gitlab/ci/jobs/trigger/trigger.yml +++ b/.gitlab/ci/jobs/trigger/trigger.yml @@ -1,26 +1,20 @@ -# Trigger -# -# §1: The purpose of this job is to launch the CI manually in certain cases. -# The objective is not to run computing when it is not -# necessary and the decision to do so belongs to the developer -# -# §2: We also perform some fast sanity checks. +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + trigger: - extends: - - .default_settings_template - - .image_template__alpine + image: alpine:3.18 stage: trigger + tags: + - gcp rules: - if: $CI_MERGE_REQUEST_ASSIGNEES !~ /nomadic-margebot/ when: manual + allow_failure: false - when: always + dependencies: [] allow_failure: false - timeout: "10m" + timeout: 10 minutes script: - - echo 'Trigger pipeline 🤠' - # Check that .gitlab-ci.yml's 'build_deps_image_version' and - # 'scripts/version.sh's 'opam_repository_tag' are the same. - - ./scripts/ci/check_opam_repository_tag.sh - # Check that the Alpine version of the trigger job's image - # corresponds to the value in scripts/version.sh. - - ./scripts/ci/check_alpine_version.sh + - echo 'Trigger pipeline!' + - ./scripts/ci/check_opam_repository_tag.sh + - ./scripts/ci/check_alpine_version.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 1a8ad4d7d7ab..5d0096244274 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -23,7 +23,7 @@ let default = default ~interruptible:true () The "manual" stage exists to fix a UI problem that occurs when mixing manual and non-manual jobs. *) module Stages = struct - let _trigger = Stage.register "trigger" + let trigger = Stage.register "trigger" let _sanity = Stage.register "sanity" @@ -175,7 +175,7 @@ module Images = struct used for the [build_deps_image_name] images and specified in the variable [alpine_version] in [scripts/version.sh]. This is checked by the jobs [trigger] and [sanity_ci]. *) - let _alpine = + let alpine = Image.register ~name:"alpine" ~image_path:("alpine:" ^ alpine_version) end @@ -193,6 +193,40 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () +(* Define the [trigger] job + + §1: The purpose of this job is to launch the CI manually in certain cases. + The objective is not to run computing when it is not + necessary and the decision to do so belongs to the developer + + §2: We also perform some fast sanity checks. *) +let _trigger = + job_external + @@ job + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + (** Helper to create jobs that uses the docker deamon. It: -- GitLab From d51f187ba23f3989eb4784da6f314af6ce3c2a6a Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 9 Feb 2024 15:55:28 +0100 Subject: [PATCH 05/15] CI: generate static binary build jobs --- ...atic-arm64-linux-binaries-experimental.yml | 32 ++++- ...ild:static-arm64-linux-binaries-master.yml | 26 +++- ...ld:static-arm64-linux-binaries-release.yml | 32 ++++- ...tic-x86_64-linux-binaries-experimental.yml | 44 ++++-- ...ld:static-x86_64-linux-binaries-master.yml | 31 +++-- ...d:static-x86_64-linux-binaries-release.yml | 35 ++++- ci/bin/main.ml | 125 +++++++++++++++++- 7 files changed, 275 insertions(+), 50 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml index 829729ad4f1e..f1a171f820f6 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml @@ -1,8 +1,30 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml index 399f32d1287d..8633e3c6de5f 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml @@ -1,10 +1,22 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 rules: - - when: always + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml index 9be030fc3b58..297a0f71ae97 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml @@ -1,11 +1,31 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables" - # Extend the lifespan to prevent failure for external tools using artifacts. + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables artifacts: expire_in: 90 days + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml index 5bebb10ee004..6b9a9dcca0aa 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml @@ -1,17 +1,33 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# This version of the job builds both released and experimental executables. -# It is used in the following pipelines: -# - Before merging: check whether static executables still compile, -# i.e. that we do pass the -static flag and that when we do it does compile -# - Master branch: executables (including experimental ones) are used in some test networks -# Variants: -# - an arm64 variant exist, but is only used in the master branch pipeline -# (no need to test that we pass the -static flag twice) -# - released variants exist, that are used in release tag pipelines -# (they do not build experimental executables) oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml index 93b06b8fc24d..8e721cb125d4 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml @@ -1,12 +1,25 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# this is the same as -# oc.build:static-x86_64-linux-binaries-experimental.yml, but for the -# master branch pipeline only oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp rules: - - when: always + - when: always + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml index 5f79eb896192..9d3f888608fa 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml @@ -1,11 +1,34 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables" - # Extend the lifespan to prevent failure for external tools using artifacts. + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables artifacts: expire_in: 90 days + paths: + - octez-binaries/$ARCH/* diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 5d0096244274..8b7784973726 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -27,7 +27,7 @@ module Stages = struct let _sanity = Stage.register "sanity" - let _build = Stage.register "build" + let build = Stage.register "build" let _test = Stage.register "test" @@ -130,7 +130,7 @@ module Images = struct ~image_path: "${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version}" - let _runtime_build_dependencies = + let runtime_build_dependencies = Image.register ~name:"runtime_build_dependencies" ~image_path: @@ -179,6 +179,33 @@ module Images = struct Image.register ~name:"alpine" ~image_path:("alpine:" ^ alpine_version) end +let before_script ?(take_ownership = false) ?(source_version = false) + ?(eval_opam = false) ?(init_python_venv = false) ?(install_js_deps = false) + before_script = + let toggle t x = if t then [x] else [] in + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 *) + toggle take_ownership "./scripts/ci/take_ownership.sh" + @ toggle source_version ". ./scripts/version.sh" + (* TODO: this must run in the before_script of all jobs that use the opam environment. + how to enforce? *) + @ toggle eval_opam "eval $(opam env)" + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables *) + @ toggle init_python_venv ". $HOME/.venv/bin/activate" + @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" + @ before_script + +let changeset_octez = + [ + "src/**/*"; + "etherlink/**/*"; + "tezt/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + "michelson_test_scripts/**/*"; + "tzt_reference_test_suite/**/*"; + ] + (* Dummy job. This fixes the "configuration must contain at least one @@ -200,7 +227,7 @@ let job_dummy : job = necessary and the decision to do so belongs to the developer §2: We also perform some fast sanity checks. *) -let _trigger = +let trigger = job_external @@ job ~image:Images.alpine @@ -252,6 +279,98 @@ let job_docker_promote_to_latest ~ci_docker_hub : job = ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] ["./scripts/ci/docker_promote_to_latest.sh"] +(* This version of the job builds both released and experimental executables. + It is used in the following pipelines: + - Before merging: check whether static executables still compile, + i.e. that we do pass the -static flag and that when we do it does compile + - Master branch: executables (including experimental ones) are used in some test networks + Variants: + - an arm64 variant exist, but is only used in the master branch pipeline + (no need to test that we pass the -static flag twice) + - released variants exist, that are used in release tag pipelines + (they do not build experimental executables) *) +let job_build_static_binaries ~arch ?(release = false) ?(needs_trigger = false) + ?rules () = + let arch_string = + match arch with Tezos_ci.Amd64 -> "x86_64" | Arm64 -> "arm64" + in + let name = "oc.build:static-" ^ arch_string ^ "-linux-binaries" in + let artifacts = + (* Extend the lifespan to prevent failure for external tools using artifacts. *) + let expire_in = if release then Some (Days 90) else None in + artifacts ?expire_in ["octez-binaries/$ARCH/*"] + in + let executable_files = + "script-inputs/released-executables" + ^ if not release then " script-inputs/experimental-executables" else "" + in + let dependencies = + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration of + pipelines. So when requested through [needs_trigger] we start + this job as early as possible, without waiting for + sanity_ci. *) + if needs_trigger then Dependent [Optional trigger] else Staged [] + in + job + ?rules + ~stage:Stages.build + ~arch + ~name + ~image:Images.runtime_build_dependencies + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + ~variables:[("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + ~dependencies + ~artifacts + ["./scripts/ci/build_static_binaries.sh"] + +let rules_static_build_master = [job_rule ~when_:Always ()] + +let rules_static_build_other = [job_rule ~changes:changeset_octez ()] + +let _job_static_arm64_experimental = + job_external ~filename_suffix:"experimental" + @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_other () + +let _job_static_arm64_master = + job_external ~filename_suffix:"master" + @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_master () + +let _job_static_arm64_release = + job_external ~filename_suffix:"release" + @@ job_build_static_binaries + ~arch:Arm64 + ~release:true + ~rules:rules_static_build_other + () + +let _job_static_x86_64_experimental = + job_external ~filename_suffix:"experimental" + @@ job_build_static_binaries + ~arch:Amd64 + ~needs_trigger:true + ~rules:rules_static_build_other + () + +let _job_static_x86_64_master = + job_external ~filename_suffix:"master" + @@ job_build_static_binaries + ~arch:Amd64 + (* TODO: this job doesn't actually need trigger and there is no + need to set it optional since we know this job is only on the master branch. *) + ~needs_trigger:true + ~rules:rules_static_build_master + () + +let _job_static_x86_64_release = + job_external ~filename_suffix:"release" + @@ job_build_static_binaries + ~arch:Amd64 + ~needs_trigger:true + ~release:true + ~rules:rules_static_build_other + () + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From a311d14de7c3c63e3baafd8fd80119390015cc0d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 22 Feb 2024 13:49:12 +0100 Subject: [PATCH 06/15] CI: refactor, remove redundant [paths] in [changes:] clause --- .gitlab/ci/jobs/shared/templates.yml | 46 +++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 138c733b64bc..283f293124fa 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -76,36 +76,34 @@ .rules__octez_changes_or_kernels: rules: - changes: - paths: - - images/**/* - - scripts/ci/**/* - - kernels.mk - - etherlink.mk - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* when: on_success # As .rules__octez_changes_or_kernels, but allows a manual trigger. .rules__octez_changes_or_kernels_or_manual: rules: - changes: - paths: - - images/**/* - - scripts/ci/**/* - - kernels.mk - - etherlink.mk - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* when: on_success - when: manual allow_failure: true -- GitLab From 3a853fa8d9d71f96f2ccc63c30a28a2707e9ba7c Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 9 Feb 2024 16:11:01 +0100 Subject: [PATCH 07/15] CI: generate [rust-toolchain] build jobs --- ...c.docker:rust-toolchain-before_merging.yml | 44 ++++++++++++++--- .../build/oc.docker:rust-toolchain-master.yml | 23 +++++++-- .../build/oc.docker:rust-toolchain-other.yml | 20 +++++++- ci/bin/main.ml | 47 ++++++++++++++++++- 4 files changed, 120 insertions(+), 14 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml index 84a87185fc2c..61062a248501 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml @@ -1,9 +1,39 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: - - .oc.docker:rust-toolchain_template - - .needs__trigger - # This job runs whenever the image itself changes or when the - # `build_kernels` or `test_kernels` runs. - - .rules__octez_changes_or_kernels_or_manual + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - changes: + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + - when: manual + allow_failure: true + needs: + - job: trigger + optional: true + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml index 12abcd0e7985..6bce59a27e94 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml @@ -1,5 +1,22 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: .oc.docker:rust-toolchain_template - when: always + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml index 0b0e7ed77685..aa236b57b019 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml @@ -1,4 +1,20 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: .oc.docker:rust-toolchain_template + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 8b7784973726..c09f23bd8694 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -206,6 +206,10 @@ let changeset_octez = "tzt_reference_test_suite/**/*"; ] +let changeset_octez_or_kernels = + ["images/**/*"; "scripts/ci/**/*"; "kernels.mk"; "etherlink.mk"] + @ changeset_octez + (* Dummy job. This fixes the "configuration must contain at least one @@ -260,13 +264,20 @@ let trigger = - Sets the appropriate image. - Activates the Docker daemon as a service. - It sets up authentification with docker registries *) -let job_docker_authenticated ?variables ~stage ~name script : job = +let job_docker_authenticated ?(skip_docker_initialization = false) ?variables + ?rules ?dependencies ?artifacts ~stage ~name script : job = let docker_version = "24.0.6" in job + ?rules + ?dependencies + ?artifacts ~image:Images.docker ~variables: ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) - ~before_script:["./scripts/ci/docker_initialize.sh"] + ~before_script: + (if not skip_docker_initialization then + ["./scripts/ci/docker_initialize.sh"] + else []) ~services:[{name = "docker:${DOCKER_VERSION}-dind"}] ~stage ~name @@ -371,6 +382,38 @@ let _job_static_x86_64_release = ~rules:rules_static_build_other () +let job_docker_rust_toolchain ?rules ?dependencies () = + job_docker_authenticated + ?rules + ?dependencies + ~skip_docker_initialization:true + ~stage:Stages.build + ~name:"oc.docker:rust-toolchain" + ~variables:[("CI_DOCKER_HUB", "false")] + ~artifacts: + (artifacts + ~reports:(reports ~dotenv:"rust_toolchain_image_tag.env" ()) + []) + ["./scripts/ci/docker_rust_toolchain_build.sh"] + +let _job_docker_rust_toolchain_before_merging = + job_external ~filename_suffix:"before_merging" + @@ job_docker_rust_toolchain + ~dependencies:(Dependent [Optional trigger]) + ~rules: + [ + job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); + job_rule ~when_:Manual (); + ] + () + +let _job_docker_rust_toolchain_master = + job_external ~filename_suffix:"master" + @@ job_docker_rust_toolchain ~rules:[job_rule ~when_:Always ()] () + +let _job_docker_rust_toolchain_other = + job_external ~filename_suffix:"other" @@ job_docker_rust_toolchain () + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From 472f6987a460b666af9900745d37aa279e12102c Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 9 Feb 2024 16:23:08 +0100 Subject: [PATCH 08/15] CI: generate Docker build jobs --- .../build/oc.docker:amd64-experimental.yml | 47 ++++- .../ci/jobs/build/oc.docker:amd64-release.yml | 46 ++++- .../ci/jobs/build/oc.docker:amd64-test.yml | 46 ++++- .../build/oc.docker:amd64-test_manual.yml | 28 ++- .../build/oc.docker:arm64-experimental.yml | 46 ++++- .../ci/jobs/build/oc.docker:arm64-release.yml | 46 ++++- .../ci/jobs/build/oc.docker:arm64-test.yml | 46 ++++- .../build/oc.docker:arm64-test_manual.yml | 27 ++- ci/bin/main.ml | 180 +++++++++++++++++- 9 files changed, 451 insertions(+), 61 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml index cfe08fb03555..4b1fc6f26702 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml @@ -1,13 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" - DOCKER_BUILD_TARGET: "with-evm-artifacts" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml index dbf019201b7c..2b04dbe07ebe 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables" + EXECUTABLE_FILES: script-inputs/released-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml index d4cf2ee66b54..d0e011163c14 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml index 2c54627cd762..2d78099119a0 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml @@ -1,14 +1,26 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: manual + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" - DOCKER_BUILD_TARGET: "with-evm-artifacts" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual - allow_failure: true diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml index 884cd596ab45..31ae139c30ed 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml index ee947b59a01e..baa9bb637c80 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables" + EXECUTABLE_FILES: script-inputs/released-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml index 75a152429a77..117f923bab08 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml index cba0b1a2f373..793e520008c1 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml @@ -1,13 +1,26 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: manual + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual - allow_failure: true diff --git a/ci/bin/main.ml b/ci/bin/main.ml index c09f23bd8694..e911207e3476 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -45,7 +45,7 @@ module Stages = struct let _publish_package_gitlab = Stage.register "publish_package_gitlab" - let _manual = Stage.register "manual" + let manual = Stage.register "manual" end (* Get the [build_deps_image_version] from the environment, which is @@ -264,13 +264,17 @@ let trigger = - Sets the appropriate image. - Activates the Docker daemon as a service. - It sets up authentification with docker registries *) -let job_docker_authenticated ?(skip_docker_initialization = false) ?variables - ?rules ?dependencies ?artifacts ~stage ~name script : job = +let job_docker_authenticated ?(skip_docker_initialization = false) ?artifacts + ?variables ?rules ?dependencies ?arch ?when_ ?allow_failure ~stage ~name + script : job = let docker_version = "24.0.6" in job ?rules ?dependencies ?artifacts + ?arch + ?when_ + ?allow_failure ~image:Images.docker ~variables: ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) @@ -414,6 +418,176 @@ let _job_docker_rust_toolchain_master = let _job_docker_rust_toolchain_other = job_external ~filename_suffix:"other" @@ job_docker_rust_toolchain () +(** Type of Docker build jobs. + + The semantics of the type is summed up in this table: + + | | Release | Experimental | Test | Test_manual | + |-----------------------+------------+--------------+--------+-------------| + | Image registry | Docker hub | Docker hub | GitLab | GitLab | + | Experimental binaries | no | yes | yes | yes | + | EVM Kernels | no | On amd64 | no | On amd64 | + | Manual job | no | no | no | yes | + + - [Release] Docker builds include only released executables whereas other + types also includes experimental ones. + - [Test_manual] and [Experimental] Docker builds include the EVM kernels in + amd64 builds. + - [Release] and [Experimental] Docker builds are pushed to Docker hub, + whereas other types are pushed to the GitLab registry. + - [Test_manual] Docker builds are triggered manually, put in the stage + [manual] and their failure is allowed. The other types are in the build + stage, run [on_success] and are not allowed to fail. *) +type docker_build_type = Experimental | Release | Test | Test_manual + +(** Creates a Docker build job of the given [arch] and [docker_build_type]. + + If [external_] is set to true (default [false]), then the job is + also written to an external file. *) +let job_docker_build ?rules ?dependencies ~arch ?(external_ = false) + docker_build_type : job = + let arch_string = + match arch with Tezos_ci.Amd64 -> "amd64" | Arm64 -> "arm64" + in + let variables = + [ + ( "DOCKER_BUILD_TARGET", + match (arch, docker_build_type) with + | Amd64, (Test_manual | Experimental) -> "with-evm-artifacts" + | _ -> "without-evm-artifacts" ); + ("IMAGE_ARCH_PREFIX", arch_string ^ "_"); + ( "CI_DOCKER_HUB", + Bool.to_string + (match docker_build_type with + | Release | Experimental -> true + | Test | Test_manual -> false) ); + ( "EXECUTABLE_FILES", + match docker_build_type with + | Release -> "script-inputs/released-executables" + | Test | Test_manual | Experimental -> + "script-inputs/released-executables \ + script-inputs/experimental-executables" ); + ] + in + let stage, when_, (allow_failure : allow_failure_job option) = + match docker_build_type with + | Test_manual -> (Stages.manual, Some Manual, Some Yes) + | _ -> (Stages.build, None, None) + in + let name = "oc.docker:" ^ arch_string in + let filename_suffix = + match docker_build_type with + | Release -> "release" + | Experimental -> "experimental" + | Test -> "test" + | Test_manual -> "test_manual" + in + let job = + job_docker_authenticated + ?when_ + ?allow_failure + ?rules + ?dependencies + ~stage + ~arch + ~name + ~variables + ["./scripts/ci/docker_release.sh"] + in + if external_ then job_external ~directory:"build" ~filename_suffix job + else job + +let changeset_octez_docker_changes_or_master = + [ + "scripts/**/*"; + "script-inputs/**/*"; + "src/**/*"; + "tezt/**/*"; + "vendors/**/*"; + "dune"; + "dune-project"; + "dune-workspace"; + "opam"; + "Makefile"; + "kernels.mk"; + "build.Dockerfile"; + "Dockerfile"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + +let rules_octez_docker_changes_or_master = + [ + job_rule ~if_:Rules.on_master ~when_:Always (); + job_rule ~changes:changeset_octez_docker_changes_or_master (); + ] + +let _job_docker_amd64_experimental : job = + job_docker_build + ~external_:true + (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. + For now we can set any variant to get the correct name of the need. *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Experimental + +let _job_docker_amd64_release : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Release + +let _job_docker_amd64_test_manual : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies: + (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) + ~arch:Amd64 + Test_manual + +let _job_docker_amd64_test : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Test + +let _job_docker_arm64_experimental : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Experimental + +let _job_docker_arm64_release : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Release + +let _job_docker_arm64_test_manual : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies: + (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) + ~arch:Arm64 + Test_manual + +let _job_docker_arm64_test : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Test + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From 3d5533f1136c3f078466918b991af4a21c9a52b6 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 15 Jan 2024 17:18:52 +0100 Subject: [PATCH 09/15] CI: generate jobs [docker:merge_manifests] --- .../docker:merge_manifests-release.yml | 31 ++++++------ .../docker:merge_manifests-test.yml | 31 ++++++------ ci/bin/main.ml | 47 +++++++++++++++++-- 3 files changed, 74 insertions(+), 35 deletions(-) diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml index d8a8fa57aabe..d1318ef9c8b4 100644 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml +++ b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml @@ -1,20 +1,21 @@ -# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. -# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. -# $IMAGE_ARCH_PREFIX is only used when building Docker images, -# here we handle all architectures so there is no such variable. +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. docker:merge_manifests: - # This job merges the images produced in the jobs - # docker:{amd64,arm64} into a single multi-architecture image, and - # so must be run after these jobs. - extends: - - .docker_auth_template - - .image_template__docker - needs: - - oc.docker:amd64 - - oc.docker:arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind variables: + DOCKER_VERSION: 24.0.6 CI_DOCKER_HUB: "true" - script: - - ./scripts/ci/docker_merge_manifests.sh diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml index 37f70b7c661c..634537335556 100644 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml +++ b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml @@ -1,20 +1,21 @@ -# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. -# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. -# $IMAGE_ARCH_PREFIX is only used when building Docker images, -# here we handle all architectures so there is no such variable. +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. docker:merge_manifests: - # This job merges the images produced in the jobs - # docker:{amd64,arm64} into a single multi-architecture image, and - # so must be run after these jobs. - extends: - - .docker_auth_template - - .image_template__docker - needs: - - oc.docker:amd64 - - oc.docker:arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind variables: + DOCKER_VERSION: 24.0.6 CI_DOCKER_HUB: "false" - script: - - ./scripts/ci/docker_merge_manifests.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index e911207e3476..6aa616692d03 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -37,7 +37,7 @@ module Stages = struct let _doc = Stage.register "doc" - let _prepare_release = Stage.register "prepare_release" + let prepare_release = Stage.register "prepare_release" let _publish_release_gitlab = Stage.register "publish_release_gitlab" @@ -522,7 +522,7 @@ let rules_octez_docker_changes_or_master = job_rule ~changes:changeset_octez_docker_changes_or_master (); ] -let _job_docker_amd64_experimental : job = +let job_docker_amd64_experimental : job = job_docker_build ~external_:true (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. @@ -548,7 +548,7 @@ let _job_docker_amd64_test_manual : job = ~arch:Amd64 Test_manual -let _job_docker_amd64_test : job = +let job_docker_amd64_test : job = job_docker_build ~external_:true (* TODO: see above *) ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) @@ -556,7 +556,7 @@ let _job_docker_amd64_test : job = ~arch:Amd64 Test -let _job_docker_arm64_experimental : job = +let job_docker_arm64_experimental : job = job_docker_build ~external_:true (* TODO: see above *) ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) @@ -580,7 +580,7 @@ let _job_docker_arm64_test_manual : job = ~arch:Arm64 Test_manual -let _job_docker_arm64_test : job = +let job_docker_arm64_test : job = job_docker_build ~external_:true (* TODO: see above *) ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) @@ -588,6 +588,43 @@ let _job_docker_arm64_test : job = ~arch:Arm64 Test +(* Note: here we rely on [$IMAGE_ARCH_PREFIX] to be empty. + Otherwise, [$DOCKER_IMAGE_TAG] would contain [$IMAGE_ARCH_PREFIX] too. + [$IMAGE_ARCH_PREFIX] is only used when building Docker images, + here we handle all architectures so there is no such variable. *) +let job_docker_merge_manifests ~ci_docker_hub ~job_docker_amd64 + ~job_docker_arm64 : job = + job_docker_authenticated + ~stage:Stages.prepare_release + ~name:"docker:merge_manifests" + (* This job merges the images produced in the jobs + [docker:{amd64,arm64}] into a single multi-architecture image, and + so must be run after these jobs. *) + ~dependencies:(Dependent [Job job_docker_amd64; Job job_docker_arm64]) + ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ["./scripts/ci/docker_merge_manifests.sh"] + +let _job_docker_merge_manifests_release = + job_external ~filename_suffix:"release" + @@ job_docker_merge_manifests + ~ci_docker_hub:true + (* TODO: In theory, actually uses either release or + experimental variant of docker jobs depending on + pipeline. In practice, this does not matter as these jobs + have the same name in the generated files + ([oc.build:ARCH]). However, when the merge_manifest jobs + are created directly in the appropriate pipeline, the + correcty variant must be used. *) + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + +let _job_docker_merge_manifests_test = + job_external ~filename_suffix:"test" + @@ job_docker_merge_manifests + ~ci_docker_hub:false + ~job_docker_amd64:job_docker_amd64_test + ~job_docker_arm64:job_docker_arm64_test + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From e5f0095efc9d3e02db3c9c7ec40454615dcbf44b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 14:32:31 +0100 Subject: [PATCH 10/15] CI: refactor, split [bin_packages{,-manual}.yml] --- ...ackages_manual.yml => oc.build:dpkg:amd64-manual.yml} | 8 -------- .../build/{bin_packages.yml => oc.build:dpkg:amd64.yml} | 7 ------- .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml | 9 +++++++++ .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml | 8 ++++++++ .gitlab/ci/pipelines/before_merging.yml | 3 ++- .gitlab/ci/pipelines/beta_release_tag.yml | 3 ++- .gitlab/ci/pipelines/non_release_tag.yml | 3 ++- .gitlab/ci/pipelines/non_release_tag_test.yml | 4 +++- .gitlab/ci/pipelines/release_tag.yml | 3 ++- .gitlab/ci/pipelines/release_tag_test.yml | 3 ++- .gitlab/ci/pipelines/schedule_extended_test.yml | 3 ++- 11 files changed, 32 insertions(+), 22 deletions(-) rename .gitlab/ci/jobs/build/{bin_packages_manual.yml => oc.build:dpkg:amd64-manual.yml} (58%) rename .gitlab/ci/jobs/build/{bin_packages.yml => oc.build:dpkg:amd64.yml} (59%) create mode 100644 .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml create mode 100644 .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml diff --git a/.gitlab/ci/jobs/build/bin_packages_manual.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml similarity index 58% rename from .gitlab/ci/jobs/build/bin_packages_manual.yml rename to .gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml index 03354af36e3d..9debb856babf 100644 --- a/.gitlab/ci/jobs/build/bin_packages_manual.yml +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml @@ -7,11 +7,3 @@ oc.build:dpkg:amd64: - .bin_packages_dpkg stage: manual when: manual - -oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm - stage: manual - when: manual diff --git a/.gitlab/ci/jobs/build/bin_packages.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml similarity index 59% rename from .gitlab/ci/jobs/build/bin_packages.yml rename to .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml index 70889f0fb6ed..8165750e16dc 100644 --- a/.gitlab/ci/jobs/build/bin_packages.yml +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml @@ -6,10 +6,3 @@ oc.build:dpkg:amd64: - .bin_packages_common - .bin_packages_dpkg stage: build - -oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm - stage: build diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml new file mode 100644 index 000000000000..69df1eb4cee0 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml @@ -0,0 +1,9 @@ +include: .gitlab/ci/jobs/build/bin_packages_common.yml + +oc.build:rpm:amd64: + extends: + - .tags_template__build + - .bin_packages_common + - .bin_packages_rpm + stage: manual + when: manual diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml new file mode 100644 index 000000000000..33d9ee7c083e --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml @@ -0,0 +1,8 @@ +include: .gitlab/ci/jobs/build/bin_packages_common.yml + +oc.build:rpm:amd64: + extends: + - .tags_template__build + - .bin_packages_common + - .bin_packages_rpm + stage: build diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d632bf641a67..6b486f1a1ca3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -24,7 +24,8 @@ include: - .gitlab/ci/jobs/doc/documentation.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml - - .gitlab/ci/jobs/build/bin_packages_manual.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml - .gitlab/ci/jobs/build/select_tezts.yml diff --git a/.gitlab/ci/pipelines/beta_release_tag.yml b/.gitlab/ci/pipelines/beta_release_tag.yml index 4ba1d341e12c..b709709a3307 100644 --- a/.gitlab/ci/pipelines/beta_release_tag.yml +++ b/.gitlab/ci/pipelines/beta_release_tag.yml @@ -5,7 +5,8 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml # Stage: prepare_release - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index fd0e94dd03cb..04c8b5518cb1 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -5,7 +5,8 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml # Stage: prepare - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 5b6164e503a4..8c45ce89a928 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -5,7 +5,9 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml + # Stage: prepare - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index 25ee59e0dd2b..0ba12b9e28a4 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -5,7 +5,8 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml # Stage: prepare_release - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index ab7af747cac3..ee70f4657a2f 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -5,7 +5,8 @@ include: - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml # Stage: prepare_release - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 1c7113ab3189..278ab933b4ea 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -7,7 +7,8 @@ include: - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_arm64-released.yml # Check that binary packages build - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml - .gitlab/ci/jobs/build/select_tezts.yml # These jobs are necessary to run flaky tezts -- GitLab From 0c659f3400800676ffb14d1b26ba2f7150198f78 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 15:11:18 +0100 Subject: [PATCH 11/15] CI: generate binary package builds [oc.build:{rpm,dpkg}] --- .../jobs/build/oc.build:dpkg:amd64-manual.yml | 44 ++++++- .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml | 40 +++++- .../jobs/build/oc.build:rpm:amd64-manual.yml | 45 ++++++- .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml | 41 ++++++- .gitlab/ci/jobs/shared/images.yml | 4 + ci/bin/main.ml | 114 ++++++++++++++++++ 6 files changed, 266 insertions(+), 22 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml index 9debb856babf..872920329286 100644 --- a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml @@ -1,9 +1,41 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.build:dpkg:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_dpkg + image: debian:bookworm stage: manual - when: manual + tags: + - gcp + rules: + - when: manual + allow_failure: true + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml index 8165750e16dc..93672e210d23 100644 --- a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml @@ -1,8 +1,38 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.build:dpkg:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_dpkg + image: debian:bookworm stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml index 69df1eb4cee0..1ee52c5e81d6 100644 --- a/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml @@ -1,9 +1,42 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm + image: fedora:39 stage: manual - when: manual + tags: + - gcp + rules: + - when: manual + allow_failure: true + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml index 33d9ee7c083e..899abeb6bd7a 100644 --- a/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml @@ -1,8 +1,39 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm + image: fedora:39 stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index db81563733e1..492934221169 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -5,8 +5,12 @@ image: alpine:3.18 .image_template__client_libs_dependencies: image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +.image_template__debian_bookworm: + image: debian:bookworm .image_template__docker: image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 +.image_template__fedora_39: + image: fedora:39 .image_template__runtime_build_dependencies: image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} .image_template__runtime_build_test_dependencies: diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 6aa616692d03..5d80c499bfa0 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -177,6 +177,11 @@ module Images = struct checked by the jobs [trigger] and [sanity_ci]. *) let alpine = Image.register ~name:"alpine" ~image_path:("alpine:" ^ alpine_version) + + let debian_bookworm = + Image.register ~name:"debian_bookworm" ~image_path:"debian:bookworm" + + let fedora_39 = Image.register ~name:"fedora_39" ~image_path:"fedora:39" end let before_script ?(take_ownership = false) ?(source_version = false) @@ -625,6 +630,115 @@ let _job_docker_merge_manifests_test = ~job_docker_amd64:job_docker_amd64_test ~job_docker_arm64:job_docker_arm64_test +type bin_package_target = Dpkg | Rpm + +let job_build_bin_package ?rules ~name ?(stage = Stages.build) ~arch ~target () + : job = + let arch_string = + match arch with Tezos_ci.Amd64 -> "amd64" | Arm64 -> "arm64" + in + let target_string = match target with Dpkg -> "dpkg" | Rpm -> "rpm" in + let image = + match target with Dpkg -> Images.debian_bookworm | Rpm -> Images.fedora_39 + in + let artifacts = + let artifact_path = + "octez-*." ^ match target with Dpkg -> "deb" | Rpm -> "rpm" + in + artifacts + ~expire_in:(Days 1) + ~when_:On_success + ~name:"${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG" + [artifact_path] + in + let before_script = + ". ./scripts/version.sh" + :: + (match target with + | Dpkg -> + [ + "apt update"; + "apt-get install -y rsync git m4 build-essential patch unzip wget \ + opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ + libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev \ + protobuf-compiler libsqlite3-dev"; + ] + | Rpm -> + [ + "dnf update -y"; + "dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel \ + zlib-devel libpq-devel m4 perl git pkg-config rpmdevtools \ + python3-devel python3-setuptools wget opam rsync which cargo \ + autoconf mock systemd systemd-rpm-macros cmake python3-wheel \ + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel \ + sqlite-devel"; + ]) + in + job + ?rules + ~name + ~arch + ~image + ~stage + ~dependencies:(Dependent []) + ~variables: + [ + ("TARGET", target_string); + ("OCTEZ_PKGMAINTAINER", "nomadic-labs"); + ("BLST_PORTABLE", "yes"); + ("ARCH", arch_string); + ] + ~artifacts + ~before_script + [ + "wget https://sh.rustup.rs/rustup-init.sh"; + "chmod +x rustup-init.sh"; + "./rustup-init.sh --profile minimal --default-toolchain \ + $recommended_rust_version -y"; + ". $HOME/.cargo/env"; + "export OPAMYES=\"true\""; + "opam init --bare --disable-sandboxing"; + "make build-deps"; + "eval $(opam env)"; + "make $TARGET"; + ] + +let _job_build_dpkg_amd64 = + job_external + @@ job_build_bin_package + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + () + +let _job_build_rpm_amd64 = + job_external + @@ job_build_bin_package + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + () + +let _job_build_dpkg_amd64_manual = + job_external ~directory:"build" ~filename_suffix:"manual" + @@ job_build_bin_package + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + +let _job_build_rpm_amd64_manual = + job_external ~directory:"build" ~filename_suffix:"manual" + @@ job_build_bin_package + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From 1f32e7fd78ecfaecb3f66562ba47e4267c1bdafa Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 15:45:59 +0100 Subject: [PATCH 12/15] CI: refactor, remove redundant [dependencies:] from [opam:release] The job [gitlab:release] does not produce any artifacts so this [dependencies] clause is noop. --- .gitlab/ci/jobs/publish/opam:release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab/ci/jobs/publish/opam:release.yml b/.gitlab/ci/jobs/publish/opam:release.yml index 6708a7ee424f..beabf2261c6a 100644 --- a/.gitlab/ci/jobs/publish/opam:release.yml +++ b/.gitlab/ci/jobs/publish/opam:release.yml @@ -5,8 +5,6 @@ opam:release: stage: publish_release # Publish jobs are uninterruptible to avoid publishing partial results. interruptible: false - dependencies: - - gitlab:release script: # create opam release and push to github (also pushes latest master branch) - ./scripts/ci/opam-release.sh -- GitLab From 41f842648abfcaae125035b124593f2232ba42f2 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 15:27:27 +0100 Subject: [PATCH 13/15] CI: generate jobs [opam:release], [gitlab:{publish,release}] --- .gitlab/ci/jobs/publish/gitlab:publish.yml | 27 +++++--- .gitlab/ci/jobs/publish/gitlab:release.yml | 32 +++++----- .gitlab/ci/jobs/publish/opam:release.yml | 12 ++-- .gitlab/ci/jobs/shared/images.yml | 2 + ci/bin/main.ml | 73 +++++++++++++++++++--- 5 files changed, 110 insertions(+), 36 deletions(-) diff --git a/.gitlab/ci/jobs/publish/gitlab:publish.yml b/.gitlab/ci/jobs/publish/gitlab:publish.yml index eb0b1c127ae7..4d950b0f4aae 100644 --- a/.gitlab/ci/jobs/publish/gitlab:publish.yml +++ b/.gitlab/ci/jobs/publish/gitlab:publish.yml @@ -1,14 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + gitlab:publish: - extends: - - .default_settings_template - image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 stage: publish_package_gitlab - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false script: - - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh + - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/jobs/publish/gitlab:release.yml b/.gitlab/ci/jobs/publish/gitlab:release.yml index cf9487b52925..1f42a06690ce 100644 --- a/.gitlab/ci/jobs/publish/gitlab:release.yml +++ b/.gitlab/ci/jobs/publish/gitlab:release.yml @@ -1,18 +1,22 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + gitlab:release: - extends: - - .default_settings_template - # https://gitlab.com/tezos/docker-images/ci-release - image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 stage: publish_release_gitlab - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false script: - # Set .gitattributes to restrict the tarball create - # by gitlab-release.sh to Octez. - - ./scripts/ci/restrict_export_to_octez_source.sh - - ./scripts/ci/gitlab-release.sh + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/jobs/publish/opam:release.yml b/.gitlab/ci/jobs/publish/opam:release.yml index beabf2261c6a..c0ac8ebdcd08 100644 --- a/.gitlab/ci/jobs/publish/opam:release.yml +++ b/.gitlab/ci/jobs/publish/opam:release.yml @@ -1,10 +1,12 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + opam:release: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} stage: publish_release - # Publish jobs are uninterruptible to avoid publishing partial results. + tags: + - gcp + dependencies: [] interruptible: false script: - # create opam release and push to github (also pushes latest master branch) - ./scripts/ci/opam-release.sh diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 492934221169..bbde7b2256da 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -3,6 +3,8 @@ .image_template__alpine: image: alpine:3.18 +.image_template__ci_release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 .image_template__client_libs_dependencies: image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} .image_template__debian_bookworm: diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 5d80c499bfa0..7bcd7310e2c1 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -39,11 +39,11 @@ module Stages = struct let prepare_release = Stage.register "prepare_release" - let _publish_release_gitlab = Stage.register "publish_release_gitlab" + let publish_release_gitlab = Stage.register "publish_release_gitlab" let publish_release = Stage.register "publish_release" - let _publish_package_gitlab = Stage.register "publish_package_gitlab" + let publish_package_gitlab = Stage.register "publish_package_gitlab" let manual = Stage.register "manual" end @@ -124,7 +124,7 @@ module Images = struct ~image_path: "${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version}" - let _runtime_build_test_dependencies = + let runtime_build_test_dependencies = Image.register ~name:"runtime_build_test_dependencies" ~image_path: @@ -182,6 +182,11 @@ module Images = struct Image.register ~name:"debian_bookworm" ~image_path:"debian:bookworm" let fedora_39 = Image.register ~name:"fedora_39" ~image_path:"fedora:39" + + let ci_release = + Image.register + ~name:"ci_release" + ~image_path:"${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" end let before_script ?(take_ownership = false) ?(source_version = false) @@ -348,7 +353,7 @@ let rules_static_build_master = [job_rule ~when_:Always ()] let rules_static_build_other = [job_rule ~changes:changeset_octez ()] -let _job_static_arm64_experimental = +let job_static_arm64_experimental = job_external ~filename_suffix:"experimental" @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_other () @@ -364,7 +369,7 @@ let _job_static_arm64_release = ~rules:rules_static_build_other () -let _job_static_x86_64_experimental = +let job_static_x86_64_experimental = job_external ~filename_suffix:"experimental" @@ job_build_static_binaries ~arch:Amd64 @@ -703,7 +708,7 @@ let job_build_bin_package ?rules ~name ?(stage = Stages.build) ~arch ~target () "make $TARGET"; ] -let _job_build_dpkg_amd64 = +let job_build_dpkg_amd64 = job_external @@ job_build_bin_package ~name:"oc.build:dpkg:amd64" @@ -711,7 +716,7 @@ let _job_build_dpkg_amd64 = ~arch:Tezos_ci.Amd64 () -let _job_build_rpm_amd64 = +let job_build_rpm_amd64 = job_external @@ job_build_bin_package ~name:"oc.build:rpm:amd64" @@ -739,6 +744,60 @@ let _job_build_rpm_amd64_manual = ~stage:Stages.manual () +let job_gitlab_release ~dependencies : job = + job + ~name:"gitlab:release" + ~image:Images.ci_release + ~stage:Stages.publish_release_gitlab + ~interruptible:false + ~dependencies + [ + "./scripts/ci/restrict_export_to_octez_source.sh"; + "./scripts/ci/gitlab-release.sh"; + ] + +let job_gitlab_publish ~dependencies : job = + job + ~name:"gitlab:publish" + ~image:Images.ci_release + ~stage:Stages.publish_package_gitlab + ~interruptible:false + ~dependencies + ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] + +let _job_gitlab_release : job = + job_external ~directory:"publish" + @@ job_gitlab_release + ~dependencies: + (Dependent + [ + Artifacts job_static_x86_64_experimental; + Artifacts job_static_arm64_experimental; + Artifacts job_build_dpkg_amd64; + Artifacts job_build_rpm_amd64; + ]) + +let _job_gitlab_publish : job = + job_external ~directory:"publish" + @@ job_gitlab_publish + ~dependencies: + (Dependent + [ + Artifacts job_static_x86_64_experimental; + Artifacts job_static_arm64_experimental; + Artifacts job_build_dpkg_amd64; + Artifacts job_build_rpm_amd64; + ]) + +let _job_opam_release : job = + job_external ~directory:"publish" + @@ job + ~name:"opam:release" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.publish_release + ~interruptible:false + ["./scripts/ci/opam-release.sh"] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined -- GitLab From 0e18db0e10b894ab1db02f804701738f4055b898 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 23 Feb 2024 15:47:55 +0100 Subject: [PATCH 14/15] CI: generate [release_tag] pipelines --- ...ld:static-arm64-linux-binaries-release.yml | 31 --- ...d:static-x86_64-linux-binaries-release.yml | 34 --- .../ci/jobs/build/oc.docker:amd64-release.yml | 44 ---- .../ci/jobs/build/oc.docker:amd64-test.yml | 44 ---- .../ci/jobs/build/oc.docker:arm64-release.yml | 44 ---- .../ci/jobs/build/oc.docker:arm64-test.yml | 44 ---- .../docker:merge_manifests-test.yml | 21 -- .gitlab/ci/jobs/publish/gitlab:publish.yml | 21 -- .gitlab/ci/jobs/publish/gitlab:release.yml | 22 -- .gitlab/ci/jobs/publish/opam:release.yml | 12 - .gitlab/ci/pipelines/beta_release_tag.yml | 222 ++++++++++++++-- .gitlab/ci/pipelines/non_release_tag.yml | 221 ++++++++++++++-- .gitlab/ci/pipelines/non_release_tag_test.yml | 222 ++++++++++++++-- .gitlab/ci/pipelines/release_tag.yml | 234 +++++++++++++++-- .gitlab/ci/pipelines/release_tag_test.yml | 222 ++++++++++++++-- ci/bin/main.ml | 237 +++++++++--------- 16 files changed, 1169 insertions(+), 506 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml delete mode 100644 .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:amd64-release.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:amd64-test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:arm64-release.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:arm64-test.yml delete mode 100644 .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml delete mode 100644 .gitlab/ci/jobs/publish/gitlab:publish.yml delete mode 100644 .gitlab/ci/jobs/publish/gitlab:release.yml delete mode 100644 .gitlab/ci/jobs/publish/opam:release.yml diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml deleted file mode 100644 index 297a0f71ae97..000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml +++ /dev/null @@ -1,31 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-arm64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp_arm64 - rules: - - changes: - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/released-executables - artifacts: - expire_in: 90 days - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml deleted file mode 100644 index 9d3f888608fa..000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml +++ /dev/null @@ -1,34 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-x86_64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp - rules: - - changes: - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success - needs: - - job: trigger - optional: true - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/released-executables - artifacts: - expire_in: 90 days - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml deleted file mode 100644 index 2b04dbe07ebe..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:amd64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml deleted file mode 100644 index d0e011163c14..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:amd64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml deleted file mode 100644 index baa9bb637c80..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:arm64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp_arm64 - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml deleted file mode 100644 index 117f923bab08..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:arm64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp_arm64 - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml deleted file mode 100644 index 634537335556..000000000000 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -docker:merge_manifests: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" diff --git a/.gitlab/ci/jobs/publish/gitlab:publish.yml b/.gitlab/ci/jobs/publish/gitlab:publish.yml deleted file mode 100644 index 4d950b0f4aae..000000000000 --- a/.gitlab/ci/jobs/publish/gitlab:publish.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -gitlab:publish: - image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 - stage: publish_package_gitlab - tags: - - gcp - needs: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - interruptible: false - script: - - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/jobs/publish/gitlab:release.yml b/.gitlab/ci/jobs/publish/gitlab:release.yml deleted file mode 100644 index 1f42a06690ce..000000000000 --- a/.gitlab/ci/jobs/publish/gitlab:release.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -gitlab:release: - image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 - stage: publish_release_gitlab - tags: - - gcp - needs: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - interruptible: false - script: - - ./scripts/ci/restrict_export_to_octez_source.sh - - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/jobs/publish/opam:release.yml b/.gitlab/ci/jobs/publish/opam:release.yml deleted file mode 100644 index c0ac8ebdcd08..000000000000 --- a/.gitlab/ci/jobs/publish/opam:release.yml +++ /dev/null @@ -1,12 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -opam:release: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} - stage: publish_release - tags: - - gcp - dependencies: [] - interruptible: false - script: - - ./scripts/ci/opam-release.sh diff --git a/.gitlab/ci/pipelines/beta_release_tag.yml b/.gitlab/ci/pipelines/beta_release_tag.yml index b709709a3307..c9fc68ea6b96 100644 --- a/.gitlab/ci/pipelines/beta_release_tag.yml +++ b/.gitlab/ci/pipelines/beta_release_tag.yml @@ -1,15 +1,209 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 04c8b5518cb1..8a5a05e5cb2d 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -1,15 +1,208 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish - - .gitlab/ci/jobs/publish/gitlab:publish.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:publish: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_package_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 8c45ce89a928..ad72f018dd9d 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -1,16 +1,208 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - - # Stage: prepare - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml - - # Stage: publish - - .gitlab/ci/jobs/publish/gitlab:publish.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" +gitlab:publish: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_package_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index 0ba12b9e28a4..f1b7d9585380 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -1,18 +1,218 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml - - # Stage: publish_release - - .gitlab/ci/jobs/publish/opam:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh +opam:release: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: publish_release + tags: + - gcp + dependencies: [] + interruptible: false + script: + - ./scripts/ci/opam-release.sh diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index ee70f4657a2f..2b9018c2fd5e 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -1,15 +1,209 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml - - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 7bcd7310e2c1..1c9f3c8cd25b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -353,7 +353,7 @@ let rules_static_build_master = [job_rule ~when_:Always ()] let rules_static_build_other = [job_rule ~changes:changeset_octez ()] -let job_static_arm64_experimental = +let _job_static_arm64_experimental = job_external ~filename_suffix:"experimental" @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_other () @@ -361,15 +361,7 @@ let _job_static_arm64_master = job_external ~filename_suffix:"master" @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_master () -let _job_static_arm64_release = - job_external ~filename_suffix:"release" - @@ job_build_static_binaries - ~arch:Arm64 - ~release:true - ~rules:rules_static_build_other - () - -let job_static_x86_64_experimental = +let _job_static_x86_64_experimental = job_external ~filename_suffix:"experimental" @@ job_build_static_binaries ~arch:Amd64 @@ -387,15 +379,6 @@ let _job_static_x86_64_master = ~rules:rules_static_build_master () -let _job_static_x86_64_release = - job_external ~filename_suffix:"release" - @@ job_build_static_binaries - ~arch:Amd64 - ~needs_trigger:true - ~release:true - ~rules:rules_static_build_other - () - let job_docker_rust_toolchain ?rules ?dependencies () = job_docker_authenticated ?rules @@ -542,14 +525,6 @@ let job_docker_amd64_experimental : job = ~arch:Amd64 Experimental -let _job_docker_amd64_release : job = - job_docker_build - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Amd64 - Release - let _job_docker_amd64_test_manual : job = job_docker_build ~external_:true (* TODO: see above *) @@ -558,14 +533,6 @@ let _job_docker_amd64_test_manual : job = ~arch:Amd64 Test_manual -let job_docker_amd64_test : job = - job_docker_build - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Amd64 - Test - let job_docker_arm64_experimental : job = job_docker_build ~external_:true (* TODO: see above *) @@ -574,14 +541,6 @@ let job_docker_arm64_experimental : job = ~arch:Arm64 Experimental -let _job_docker_arm64_release : job = - job_docker_build - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Arm64 - Release - let _job_docker_arm64_test_manual : job = job_docker_build ~external_:true (* TODO: see above *) @@ -590,14 +549,6 @@ let _job_docker_arm64_test_manual : job = ~arch:Arm64 Test_manual -let job_docker_arm64_test : job = - job_docker_build - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Arm64 - Test - (* Note: here we rely on [$IMAGE_ARCH_PREFIX] to be empty. Otherwise, [$DOCKER_IMAGE_TAG] would contain [$IMAGE_ARCH_PREFIX] too. [$IMAGE_ARCH_PREFIX] is only used when building Docker images, @@ -628,13 +579,6 @@ let _job_docker_merge_manifests_release = ~job_docker_amd64:job_docker_amd64_experimental ~job_docker_arm64:job_docker_arm64_experimental -let _job_docker_merge_manifests_test = - job_external ~filename_suffix:"test" - @@ job_docker_merge_manifests - ~ci_docker_hub:false - ~job_docker_amd64:job_docker_amd64_test - ~job_docker_arm64:job_docker_arm64_test - type bin_package_target = Dpkg | Rpm let job_build_bin_package ?rules ~name ?(stage = Stages.build) ~arch ~target () @@ -744,59 +688,117 @@ let _job_build_rpm_amd64_manual = ~stage:Stages.manual () -let job_gitlab_release ~dependencies : job = - job - ~name:"gitlab:release" - ~image:Images.ci_release - ~stage:Stages.publish_release_gitlab - ~interruptible:false - ~dependencies - [ - "./scripts/ci/restrict_export_to_octez_source.sh"; - "./scripts/ci/gitlab-release.sh"; - ] +(** Type of release tag pipelines. -let job_gitlab_publish ~dependencies : job = - job - ~name:"gitlab:publish" - ~image:Images.ci_release - ~stage:Stages.publish_package_gitlab - ~interruptible:false - ~dependencies - ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] - -let _job_gitlab_release : job = - job_external ~directory:"publish" - @@ job_gitlab_release - ~dependencies: - (Dependent - [ - Artifacts job_static_x86_64_experimental; - Artifacts job_static_arm64_experimental; - Artifacts job_build_dpkg_amd64; - Artifacts job_build_rpm_amd64; - ]) - -let _job_gitlab_publish : job = - job_external ~directory:"publish" - @@ job_gitlab_publish - ~dependencies: - (Dependent - [ - Artifacts job_static_x86_64_experimental; - Artifacts job_static_arm64_experimental; - Artifacts job_build_dpkg_amd64; - Artifacts job_build_rpm_amd64; - ]) - -let _job_opam_release : job = - job_external ~directory:"publish" - @@ job - ~name:"opam:release" - ~image:Images.runtime_build_test_dependencies - ~stage:Stages.publish_release - ~interruptible:false - ["./scripts/ci/opam-release.sh"] + The semantics of the type is summed up in this table: + + | | Release_tag | Beta_release_tag | Non_release_tag | + |-----------------------+-------------+------------------+-----------------| + | GitLab release type | Release | Release | Create | + | Experimental binaries | No | No | No | + | Docker build type | Release | Release | Release | + | Publishes to opam | Yes | No | No | + + - All release tag pipelines types publish [Release] type Docker builds. + - No release tag pipelines include experimental binaries. + - [Release_tag] and [Beta_release_tag] pipelines creates GitLab + and publishes releases. [Non_release_tag] pipelines creates the + GitLab release but do not publish them. + - Only [Release_tag] pipelines publish to opam. *) +type release_tag_pipeline_type = + | Release_tag + | Beta_release_tag + | Non_release_tag + +(** Create a release tag pipeline of type {!release_tag_pipeline_type}. + + If [test] is true (default is [false]), then the Docker images are + built of the [Test] type and are published to the GitLab registry + instead of Docker hub. *) +let release_tag_pipeline ?(test = false) release_tag_pipeline_type = + let job_docker_rust_toolchain = job_docker_rust_toolchain () in + let job_docker_amd64 = + job_docker_build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Amd64 + (if test then Test else Release) + in + let job_docker_arm64 = + job_docker_build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Arm64 + (if test then Test else Release) + in + let job_docker_merge = + job_docker_merge_manifests + ~ci_docker_hub:(not test) + ~job_docker_amd64 + ~job_docker_arm64 + in + let job_static_arm64_release = + job_build_static_binaries ~arch:Arm64 ~release:true () + in + let job_static_x86_64_release = + job_build_static_binaries ~arch:Amd64 ~release:true ~needs_trigger:true () + in + let job_gitlab_release ~dependencies : job = + job + ~image:Images.ci_release + ~stage:Stages.publish_release_gitlab + ~interruptible:false + ~dependencies + ~name:"gitlab:release" + [ + "./scripts/ci/restrict_export_to_octez_source.sh"; + "./scripts/ci/gitlab-release.sh"; + ] + in + let job_gitlab_publish ~dependencies : job = + job + ~image:Images.ci_release + ~stage:Stages.publish_package_gitlab + ~interruptible:false + ~dependencies + ~name:"gitlab:publish" + ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] + in + let job_gitlab_release_or_publish = + let dependencies = + Dependent + [ + Artifacts job_static_x86_64_release; + Artifacts job_static_arm64_release; + Artifacts job_build_dpkg_amd64; + Artifacts job_build_rpm_amd64; + ] + in + match release_tag_pipeline_type with + | Non_release_tag -> job_gitlab_publish ~dependencies + | _ -> job_gitlab_release ~dependencies + in + let job_opam_release : job = + job + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.publish_release + ~interruptible:false + ~name:"opam:release" + ["./scripts/ci/opam-release.sh"] + in + [ + job_docker_rust_toolchain; + job_static_x86_64_release; + job_static_arm64_release; + job_docker_amd64; + job_docker_arm64; + job_build_dpkg_amd64; + job_build_rpm_amd64; + job_docker_merge; + job_gitlab_release_or_publish; + ] + @ + match (test, release_tag_pipeline_type) with + | false, Release_tag -> [job_opam_release] + | _ -> [] (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the @@ -829,19 +831,24 @@ let () = register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; register "release_tag" - If.(on_tezos_namespace && push && has_tag_match release_tag_re) ; + If.(on_tezos_namespace && push && has_tag_match release_tag_re) + ~jobs:(release_tag_pipeline Release_tag) ; register "beta_release_tag" - If.(on_tezos_namespace && push && has_tag_match beta_release_tag_re) ; + If.(on_tezos_namespace && push && has_tag_match beta_release_tag_re) + ~jobs:(release_tag_pipeline Beta_release_tag) ; register "release_tag_test" - If.(not_on_tezos_namespace && push && has_any_release_tag) ; + If.(not_on_tezos_namespace && push && has_any_release_tag) + ~jobs:(release_tag_pipeline ~test:true Release_tag) ; register "non_release_tag" - If.(on_tezos_namespace && push && has_non_release_tag) ; + If.(on_tezos_namespace && push && has_non_release_tag) + ~jobs:(release_tag_pipeline Non_release_tag) ; register "non_release_tag_test" - If.(not_on_tezos_namespace && push && has_non_release_tag) ; + If.(not_on_tezos_namespace && push && has_non_release_tag) + ~jobs:(release_tag_pipeline ~test:true Non_release_tag) ; register "schedule_extended_test" schedule_extended_tests (* Split pipelines and writes image templates *) -- GitLab From 5697605217e49a1cc8d7e8191459899cd1c90a76 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 28 Feb 2024 14:24:59 +0100 Subject: [PATCH 15/15] CI-in-OCaml: documentation updates --- ci/README.md | 4 ++-- ci/bin/main.ml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/README.md b/ci/README.md index 621b8f22c115..0914c078ec55 100644 --- a/ci/README.md +++ b/ci/README.md @@ -13,10 +13,10 @@ This directory is structured like this: ## Usage -To regenerate `.gitlab-ci.yml` (from the root of the repo): +To regenerate the GitLab CI configuration, run (from the root of the repo): make -C ci all -To check that `.gitlab-ci.yml` is up-to-date (from the root of the repo): +To check that the GitLab CI configuration is up-to-date, run (from the root of the repo): make -C ci check diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 1c9f3c8cd25b..a48b0f88d24b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -8,7 +8,7 @@ (* Main entrypoint of CI-in-OCaml. Here we register the set of pipelines, stages and images and - generate the top-level [.gitlab-ci.yml] file. *) + generate the GitLab CI configuration file. *) open Gitlab_ci open Gitlab_ci.Types -- GitLab