From 156ec7fdd817bd0f406509af097e3ee88f116127 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Fri, 6 Dec 2024 15:02:03 +0100 Subject: [PATCH 1/3] CI: Add [inherit_] option to [register_child] function --- ci/bin/tezos_ci.ml | 15 +++++++++++---- ci/bin/tezos_ci.mli | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index a41acac3064a..0f9de57b3c98 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -143,6 +143,7 @@ module Pipeline = struct name : string; description : string; auto_cancel : Gitlab_ci.Types.auto_cancel option; + inherit_ : Gitlab_ci.Types.inherit_ option; jobs : tezos_job list; } @@ -178,8 +179,8 @@ module Pipeline = struct register_raw (Pipeline {variables; if_; name; jobs; auto_cancel; description}) - let register_child ?auto_cancel ~description ~jobs name = - let child_pipeline = {name; jobs; auto_cancel; description} in + let register_child ?auto_cancel ?inherit_ ~description ~jobs name = + let child_pipeline = {name; inherit_; jobs; auto_cancel; description} in register_raw (Child_pipeline child_pipeline) ; child_pipeline @@ -868,8 +869,13 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage Pipeline. - {name = child_pipeline_name; jobs = _; auto_cancel = _; description = _} : - tezos_job = + { + name = child_pipeline_name; + inherit_; + jobs = _; + auto_cancel = _; + description = _; + } : tezos_job = let job_name = "trigger:" ^ child_pipeline_name in let needs, dependencies = resolve_dependencies job_name dependencies in if dependencies != [] then @@ -880,6 +886,7 @@ let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage let trigger_job = Gitlab_ci.Util.trigger_job ?needs + ?inherit_ ?rules ~stage:(Stage.name stage) ~name:job_name diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index defb85251ca8..6130ce40919c 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -119,6 +119,7 @@ module Pipeline : sig included in a regular pipeline (see {!trigger_job}). *) val register_child : ?auto_cancel:Gitlab_ci.Types.auto_cancel -> + ?inherit_:Gitlab_ci.Types.inherit_ -> description:string -> jobs:tezos_job list -> string -> -- GitLab From b9ecbac2105e18f8491a64b3157e4ca23664acb6 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Fri, 6 Dec 2024 15:41:06 +0100 Subject: [PATCH 2/3] CI: Move [job_build_profiling_layer1] to [Common] This will be used also in [release_tag.ml] --- ci/bin/code_verification.ml | 17 ----------------- ci/bin/common.ml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index df1908d12bd5..edcd6bffc09e 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -750,23 +750,6 @@ let jobs pipeline_type = | Schedule_extended_test -> Common.job_build_grafazos ~rules:[job_rule ~when_:Always ()] () in - let job_build_layer1_profiling = - job - ~__POS__ - ~stage - ~image:Images.CI.build - ~name:"build-layer1-profiling" - ~artifacts:(artifacts ~expire_in:(Duration (Days 1)) ["./octez-node"]) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ~variables:[("TEZOS_PPX_PROFILER", "profiling")] - ["make octez-layer1"] - |> enable_cargo_cache |> enable_sccache - in [ job_build_arm64_release; job_build_arm64_exp_dev_extra; diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 83c4ea58851c..05e0572bc67b 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1326,6 +1326,23 @@ let job_build_grafazos ?rules () = ] ["make"] +let job_build_layer1_profiling = + job + ~__POS__ + ~stage:Stages.build + ~image:Images.CI.build + ~name:"build-layer1-profiling" + ~artifacts:(artifacts ~expire_in:(Duration (Days 1)) ["./octez-node"]) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~variables:[("TEZOS_PPX_PROFILER", "profiling")] + ["make octez-layer1"] + |> enable_cargo_cache |> enable_sccache + module Tezt = struct (** Create a tezt job. -- GitLab From 1c091cb4c8fe7b1d8e1c5e79b445df3317a64a30 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Fri, 6 Dec 2024 15:44:49 +0100 Subject: [PATCH 3/3] CI, Release: Introduce [Octez_monitoring] release child pipeline --- .gitlab/ci/pipelines/non_release_tag.yml | 10 ++ .gitlab/ci/pipelines/non_release_tag_test.yml | 10 ++ .../ci/pipelines/octez_beta_release_tag.yml | 10 ++ .gitlab/ci/pipelines/octez_monitoring.yml | 124 ++++++++++++++++++ .gitlab/ci/pipelines/octez_release_tag.yml | 10 ++ .../ci/pipelines/octez_release_tag_test.yml | 10 ++ .../schedule_master_test_release.yml | 10 ++ ci/bin/release_tag.ml | 21 +++ 8 files changed, 205 insertions(+) create mode 100644 .gitlab/ci/pipelines/octez_monitoring.yml diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 0ac43ef7a3fa..b08a8d7d0818 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -317,6 +317,16 @@ gitlab:publish: script: - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index b8856e031c45..cc5840b47092 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -317,6 +317,16 @@ gitlab:publish: script: - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index fa3c0e78b2e2..eb43dfe28c1e 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -318,6 +318,16 @@ gitlab:release: - ./scripts/ci/restrict_export_to_octez_source.sh - ./scripts/ci/gitlab-release.sh +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/.gitlab/ci/pipelines/octez_monitoring.yml b/.gitlab/ci/pipelines/octez_monitoring.yml new file mode 100644 index 000000000000..161f734cd9d2 --- /dev/null +++ b/.gitlab/ci/pipelines/octez_monitoring.yml @@ -0,0 +1,124 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +workflow: + rules: + - if: $foo != "bar" || $foo == "bar" + when: always + +variables: + PIPELINE_TYPE: octez_monitoring + +stages: +- start +- images +- build + +oc.docker:ci:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 60 minutes + before_script: [] + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: amd64 + artifacts: + reports: + dotenv: ci_image_tag.env + +oc.docker:jsonnet:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_jsonnet_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: jsonnet_image_tag.env + +datadog_pipeline_trace: + image: datadog/ci:v2.44.0 + stage: start + tags: + - gcp + dependencies: [] + allow_failure: true + timeout: 60 minutes + script: + - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} + - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE + --tags mr_number:$CI_MERGE_REQUEST_IID + +build_grafazos_dashboards: + image: ${jsonnet_image_name}:${jsonnet_image_tag} + stage: build + tags: + - gcp + dependencies: + - oc.docker:jsonnet:amd64 + timeout: 60 minutes + before_script: + - cd grafazos/ + - jb install github.com/grafana/grafonnet/gen/grafonnet-v11.1.0@1ce5aec + script: + - make + artifacts: + name: grafazos-dashboards + expire_in: 1 day + paths: + - grafazos/output/**/*.json + when: on_success + +build-layer1-profiling: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make octez-layer1 + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + TEZOS_PPX_PROFILER: profiling + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 1 day + paths: + - ./octez-node diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index 2c386f66386c..5247900d6c2a 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -319,6 +319,16 @@ gitlab:release: - ./scripts/ci/restrict_export_to_octez_source.sh - ./scripts/ci/gitlab-release.sh +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 41561be49c59..3ae54af7eaa4 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -319,6 +319,16 @@ gitlab:release: - ./scripts/ci/restrict_export_to_octez_source.sh - ./scripts/ci/gitlab-release.sh +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/.gitlab/ci/pipelines/schedule_master_test_release.yml b/.gitlab/ci/pipelines/schedule_master_test_release.yml index 018f95dede30..36bc04f9c41e 100644 --- a/.gitlab/ci/pipelines/schedule_master_test_release.yml +++ b/.gitlab/ci/pipelines/schedule_master_test_release.yml @@ -321,6 +321,16 @@ gitlab:publish: variables: CI_COMMIT_TAG: octez-v0.0 +trigger:octez_monitoring: + stage: build + inherit: + variables: + - ci_image_name + - jsonnet_image_name + needs: [] + trigger: + include: .gitlab/ci/pipelines/octez_monitoring.yml + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 88ad7dc754b1..7a39808e6563 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -41,6 +41,19 @@ type release_tag_pipeline_type = | Non_release_tag | Schedule_test +let monitoring_child_pipeline = + Pipeline.register_child + "octez_monitoring" + ~description:"Octez monitoring jobs" + ~inherit_: + (Gitlab_ci.Types.Variable_list ["ci_image_name"; "jsonnet_image_name"]) + ~jobs: + [ + job_datadog_pipeline_trace; + job_build_grafazos (); + job_build_layer1_profiling; + ] + (** Create an Octez release tag pipeline of type {!release_tag_pipeline_type}. If [test] is true (default is [false]), then the Docker images are @@ -196,6 +209,13 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = ~dependencies:(Dependent [Job job_docker_merge]) () in + let job_trigger_monitoring = + trigger_job + ~__POS__ + ~dependencies:(Dependent []) + ~stage:Stages.build + monitoring_child_pipeline + in [ (* Stage: start *) job_datadog_pipeline_trace; @@ -208,6 +228,7 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = job_build_homebrew_release; job_docker_merge; job_gitlab_release_or_publish; + job_trigger_monitoring; ] @ jobs_debian_repository @ -- GitLab