diff --git a/.gitlab/ci/jobs/build/bin_packages.yml b/.gitlab/ci/jobs/build/bin_packages.yml deleted file mode 100644 index 70889f0fb6ed768b300bc21be0fa4ce01d62357f..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/bin_packages.yml +++ /dev/null @@ -1,15 +0,0 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml - -oc.build:dpkg:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_dpkg - stage: build - -oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm - stage: build diff --git a/.gitlab/ci/jobs/build/bin_packages_manual.yml b/.gitlab/ci/jobs/build/bin_packages_manual.yml deleted file mode 100644 index 03354af36e3d890465c56336af0850578cf57e00..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/bin_packages_manual.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/build/bin_packages_common.yml - -oc.build:dpkg:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_dpkg - stage: manual - when: manual - -oc.build:rpm:amd64: - extends: - - .tags_template__build - - .bin_packages_common - - .bin_packages_rpm - stage: manual - when: manual diff --git a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml new file mode 100644 index 0000000000000000000000000000000000000000..8729203292860d527967abebb16547f19f3b2842 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml @@ -0,0 +1,41 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build:dpkg:amd64: + image: debian:bookworm + stage: manual + tags: + - gcp + rules: + - when: manual + allow_failure: true + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..93672e210d236c1ef8cc8e03aa797b043e278647 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml @@ -0,0 +1,38 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml new file mode 100644 index 0000000000000000000000000000000000000000..1ee52c5e81d60443cdac6dfebcbe8920524d7c49 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml @@ -0,0 +1,42 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build:rpm:amd64: + image: fedora:39 + stage: manual + tags: + - gcp + rules: + - when: manual + allow_failure: true + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..899abeb6bd7a14f7df8e53f2618f05c4b91c85fd --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build:rpm:amd64.yml @@ -0,0 +1,39 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml index 829729ad4f1e2f8593ee8c5052d66141a5add9ad..f1a171f820f6f73be98af6da376e44897decec8e 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-experimental.yml @@ -1,8 +1,30 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml index 399f32d1287d72c8719c0a243831b9935c37c169..8633e3c6de5f9940a3c3df117ea8cabb4624aa00 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml @@ -1,10 +1,22 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 rules: - - when: always + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml deleted file mode 100644 index 9be030fc3b581221c8b0f4bc5981282993e63710..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml +++ /dev/null @@ -1,11 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. -oc.build:static-arm64-linux-binaries: - extends: - - .oc.build:static-arm64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables" - # Extend the lifespan to prevent failure for external tools using artifacts. - artifacts: - expire_in: 90 days diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml index 5bebb10ee004dd3d4ce3d68c243716583e32bc30..6b9a9dcca0aac03bacd754a8aeda1921cd2dc7ea 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-experimental.yml @@ -1,17 +1,33 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# This version of the job builds both released and experimental executables. -# It is used in the following pipelines: -# - Before merging: check whether static executables still compile, -# i.e. that we do pass the -static flag and that when we do it does compile -# - Master branch: executables (including experimental ones) are used in some test networks -# Variants: -# - an arm64 variant exist, but is only used in the master branch pipeline -# (no need to test that we pass the -static flag twice) -# - released variants exist, that are used in release tag pipelines -# (they do not build experimental executables) oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml index 93b06b8fc24dd1bd3bbd831cdaa99e16bf8a87a9..8e721cb125d42bb8d61b90f7ab6678486eb6917d 100644 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml +++ b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml @@ -1,12 +1,25 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# this is the same as -# oc.build:static-x86_64-linux-binaries-experimental.yml, but for the -# master branch pipeline only oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp rules: - - when: always + - when: always + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml deleted file mode 100644 index 5f79eb896192b3c0039a880abc37df0f52b6915a..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml +++ /dev/null @@ -1,11 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# See comments in oc.build:static-x86_64-linux-binaries-experimental.yml. -oc.build:static-x86_64-linux-binaries: - extends: - - .oc.build:static-x86_64-linux-binaries - variables: - EXECUTABLE_FILES: "script-inputs/released-executables" - # Extend the lifespan to prevent failure for external tools using artifacts. - artifacts: - expire_in: 90 days diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml index cfe08fb0355571b4194ccc183a0b4c722fcdee60..4b1fc6f267026b6eea850b54c903fbcf9753ce43 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml @@ -1,13 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" - DOCKER_BUILD_TARGET: "with-evm-artifacts" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml deleted file mode 100644 index dbf019201b7cbb738ad4862208bcccb87740c796..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-release.yml +++ /dev/null @@ -1,12 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build - stage: build - variables: - IMAGE_ARCH_PREFIX: "amd64_" - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables" diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml deleted file mode 100644 index d4cf2ee66b54d8d377fbbe9d488d52414aaf8cd0..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test.yml +++ /dev/null @@ -1,12 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build - stage: build - variables: - IMAGE_ARCH_PREFIX: "amd64_" - CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml index 2c54627cd7620e9271bbd2cd6cdfc50fb99cdfc8..2d78099119a090668b37b75c6a747bd5206e88da 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml @@ -1,14 +1,26 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:amd64: - extends: - - .oc.build_docker_release_template - - .tags_template__build + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: manual + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "amd64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" - DOCKER_BUILD_TARGET: "with-evm-artifacts" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual - allow_failure: true diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml index 884cd596ab456b9edceedd58c89ec42a4bcfe604..31ae139c30ed1dcac67dadf5553b2673b83003f2 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml @@ -1,12 +1,44 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml deleted file mode 100644 index ee947b59a01e6ff245e0fd130ea6d215a6a49f1a..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-release.yml +++ /dev/null @@ -1,12 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 - stage: build - variables: - IMAGE_ARCH_PREFIX: "arm64_" - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: "script-inputs/released-executables" diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml deleted file mode 100644 index 75a152429a778097002be855f3f2a69f0868f95d..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test.yml +++ /dev/null @@ -1,12 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .rules__octez_docker_changes_or_master - - .tags_template__build_arm64 - stage: build - variables: - IMAGE_ARCH_PREFIX: "arm64_" - CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml index cba0b1a2f3732e4e1031a82df37a9810167fa7c3..793e520008c171c799ebeaaf3bf7bf266bb1df73 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml @@ -1,13 +1,26 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:arm64: - extends: - - .oc.build_docker_release_template - - .tags_template__build_arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: manual + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - IMAGE_ARCH_PREFIX: "arm64_" + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ CI_DOCKER_HUB: "false" - EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual - allow_failure: true diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml index 84a87185fc2ce3ccd4dd08280b2b89c070c6c834..61062a24850164d586794bdcac65c0547f1d2c06 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml @@ -1,9 +1,39 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: - - .oc.docker:rust-toolchain_template - - .needs__trigger - # This job runs whenever the image itself changes or when the - # `build_kernels` or `test_kernels` runs. - - .rules__octez_changes_or_kernels_or_manual + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - changes: + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + - when: manual + allow_failure: true + needs: + - job: trigger + optional: true + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml index 12abcd0e7985c6e6a79d9dbe3ae9ee3e5283110f..6bce59a27e949c97c62df38765d9a4649e0cf338 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml @@ -1,5 +1,22 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: .oc.docker:rust-toolchain_template - when: always + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml index 0b0e7ed77685578b9909c2d3703e44f333f1d861..aa236b57b0191079b2c336edbcf84eb00dee43b1 100644 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml @@ -1,4 +1,20 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - extends: .oc.docker:rust-toolchain_template + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml index d8a8fa57aabe1e914272fc629f88fb4136eec0f8..d1318ef9c8b4fd8a12e553284febd4805278e3bd 100644 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml +++ b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml @@ -1,20 +1,21 @@ -# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. -# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. -# $IMAGE_ARCH_PREFIX is only used when building Docker images, -# here we handle all architectures so there is no such variable. +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. docker:merge_manifests: - # This job merges the images produced in the jobs - # docker:{amd64,arm64} into a single multi-architecture image, and - # so must be run after these jobs. - extends: - - .docker_auth_template - - .image_template__docker - needs: - - oc.docker:amd64 - - oc.docker:arm64 + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind variables: + DOCKER_VERSION: 24.0.6 CI_DOCKER_HUB: "true" - script: - - ./scripts/ci/docker_merge_manifests.sh diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml deleted file mode 100644 index 37f70b7c661cb62e7b0a40dab3fa5580ce3cf868..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. -# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. -# $IMAGE_ARCH_PREFIX is only used when building Docker images, -# here we handle all architectures so there is no such variable. - -docker:merge_manifests: - # This job merges the images produced in the jobs - # docker:{amd64,arm64} into a single multi-architecture image, and - # so must be run after these jobs. - extends: - - .docker_auth_template - - .image_template__docker - needs: - - oc.docker:amd64 - - oc.docker:arm64 - stage: prepare_release - variables: - CI_DOCKER_HUB: "false" - script: - - ./scripts/ci/docker_merge_manifests.sh diff --git a/.gitlab/ci/jobs/publish/gitlab:publish.yml b/.gitlab/ci/jobs/publish/gitlab:publish.yml deleted file mode 100644 index eb0b1c127ae79d18a61f241b0091f7ef81caafa7..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/gitlab:publish.yml +++ /dev/null @@ -1,14 +0,0 @@ -gitlab:publish: - extends: - - .default_settings_template - image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" - stage: publish_package_gitlab - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false - dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - script: - - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/jobs/publish/gitlab:release.yml b/.gitlab/ci/jobs/publish/gitlab:release.yml deleted file mode 100644 index cf9487b529254b8308ffff26ba70be75cd0b8932..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/gitlab:release.yml +++ /dev/null @@ -1,18 +0,0 @@ -gitlab:release: - extends: - - .default_settings_template - # https://gitlab.com/tezos/docker-images/ci-release - image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" - stage: publish_release_gitlab - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false - dependencies: - - oc.build:static-x86_64-linux-binaries - - oc.build:static-arm64-linux-binaries - - oc.build:dpkg:amd64 - - oc.build:rpm:amd64 - script: - # Set .gitattributes to restrict the tarball create - # by gitlab-release.sh to Octez. - - ./scripts/ci/restrict_export_to_octez_source.sh - - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/jobs/publish/opam:release.yml b/.gitlab/ci/jobs/publish/opam:release.yml deleted file mode 100644 index 6708a7ee424f954810c44bd3d7a70ae63f38f425..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/opam:release.yml +++ /dev/null @@ -1,12 +0,0 @@ -opam:release: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - stage: publish_release - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false - dependencies: - - gitlab:release - script: - # create opam release and push to github (also pushes latest master branch) - - ./scripts/ci/opam-release.sh diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index db81563733e15ce853a9b5dd6552987899af3edc..bbde7b2256dab12c6c26b85c5138620caba9df10 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -3,10 +3,16 @@ .image_template__alpine: image: alpine:3.18 +.image_template__ci_release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 .image_template__client_libs_dependencies: image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +.image_template__debian_bookworm: + image: debian:bookworm .image_template__docker: image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 +.image_template__fedora_39: + image: fedora:39 .image_template__runtime_build_dependencies: image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} .image_template__runtime_build_test_dependencies: diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 138c733b64bcfd651c58e02a38c32166e57bc8c4..283f293124fad98431c60112144b5d9c35d453b4 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -76,36 +76,34 @@ .rules__octez_changes_or_kernels: rules: - changes: - paths: - - images/**/* - - scripts/ci/**/* - - kernels.mk - - etherlink.mk - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* when: on_success # As .rules__octez_changes_or_kernels, but allows a manual trigger. .rules__octez_changes_or_kernels_or_manual: rules: - changes: - paths: - - images/**/* - - scripts/ci/**/* - - kernels.mk - - etherlink.mk - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* when: on_success - when: manual allow_failure: true diff --git a/.gitlab/ci/jobs/trigger/trigger.yml b/.gitlab/ci/jobs/trigger/trigger.yml index 715b589f7712424564cc75ae0586908335f095e1..164632e04224d514d00ee24abebef95c6e0dfcc3 100644 --- a/.gitlab/ci/jobs/trigger/trigger.yml +++ b/.gitlab/ci/jobs/trigger/trigger.yml @@ -1,26 +1,20 @@ -# Trigger -# -# §1: The purpose of this job is to launch the CI manually in certain cases. -# The objective is not to run computing when it is not -# necessary and the decision to do so belongs to the developer -# -# §2: We also perform some fast sanity checks. +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + trigger: - extends: - - .default_settings_template - - .image_template__alpine + image: alpine:3.18 stage: trigger + tags: + - gcp rules: - if: $CI_MERGE_REQUEST_ASSIGNEES !~ /nomadic-margebot/ when: manual + allow_failure: false - when: always + dependencies: [] allow_failure: false - timeout: "10m" + timeout: 10 minutes script: - - echo 'Trigger pipeline 🤠' - # Check that .gitlab-ci.yml's 'build_deps_image_version' and - # 'scripts/version.sh's 'opam_repository_tag' are the same. - - ./scripts/ci/check_opam_repository_tag.sh - # Check that the Alpine version of the trigger job's image - # corresponds to the value in scripts/version.sh. - - ./scripts/ci/check_alpine_version.sh + - echo 'Trigger pipeline!' + - ./scripts/ci/check_opam_repository_tag.sh + - ./scripts/ci/check_alpine_version.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d632bf641a676cc2f4540051cef0b456524e5bb8..6b486f1a1ca315c48032a7bef85c1e489ce122f0 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -24,7 +24,8 @@ include: - .gitlab/ci/jobs/doc/documentation.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml - - .gitlab/ci/jobs/build/bin_packages_manual.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml - .gitlab/ci/jobs/build/select_tezts.yml diff --git a/.gitlab/ci/pipelines/beta_release_tag.yml b/.gitlab/ci/pipelines/beta_release_tag.yml index 4ba1d341e12c064fccc3a6fbab6914e3863f501f..c9fc68ea6b96d40dc53656bfd882d3a69705a0a0 100644 --- a/.gitlab/ci/pipelines/beta_release_tag.yml +++ b/.gitlab/ci/pipelines/beta_release_tag.yml @@ -1,14 +1,209 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index fd0e94dd03cb4e2eacc57efd34a4d7f5750d78f9..8a5a05e5cb2df1ba9d74489fef3b708aa8eff106 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -1,14 +1,208 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish - - .gitlab/ci/jobs/publish/gitlab:publish.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:publish: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_package_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 5b6164e503a4a2f26f3cf15b8c87ec6df46e55b5..ad72f018dd9d56003964aaf2919621ecd907880b 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -1,14 +1,208 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/bin_packages.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml - - # Stage: publish - - .gitlab/ci/jobs/publish/gitlab:publish.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" +gitlab:publish: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_package_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index 25ee59e0dd2bed5eafe0e4a3220d4a0be560b005..f1b7d9585380b988a6f2c853a09679a7c435d817 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -1,17 +1,218 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-release.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-release.yml - - .gitlab/ci/jobs/build/bin_packages.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml - - # Stage: publish_release - - .gitlab/ci/jobs/publish/opam:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh +opam:release: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: publish_release + tags: + - gcp + dependencies: [] + interruptible: false + script: + - ./scripts/ci/opam-release.sh diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index ab7af747cac3adec6d7001d5c0aa16a5aa89314e..2b9018c2fd5e45f62423d493d6776447218886c4 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -1,14 +1,209 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-release.yml - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-other.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-test.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-test.yml - - .gitlab/ci/jobs/build/bin_packages.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-test.yml - - # Stage: publish_release_gitlab - - .gitlab/ci/jobs/publish/gitlab:release.yml +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables +oc.build:dpkg:amd64: + image: debian:bookworm + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf + cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev + protobuf-compiler libsqlite3-dev + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: dpkg + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.deb + when: on_success +oc.build:rpm:amd64: + image: fedora:39 + stage: build + tags: + - gcp + needs: [] + dependencies: [] + before_script: + - . ./scripts/version.sh + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel + m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam + rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel + script: + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version + -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make $TARGET + variables: + TARGET: rpm + OCTEZ_PKGMAINTAINER: nomadic-labs + BLST_PORTABLE: "yes" + ARCH: amd64 + artifacts: + name: ${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-*.rpm + when: on_success +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" +gitlab:release: + image: ${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + - oc.build:dpkg:amd64 + - oc.build:rpm:amd64 + interruptible: false + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/gitlab-release.sh diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 1c7113ab31891ebafab4ca0cca848a9d1298d02c..278ab933b4eafc81a600c3b1d50a619c1b2c3eb7 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -7,7 +7,8 @@ include: - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_arm64-released.yml # Check that binary packages build - - .gitlab/ci/jobs/build/bin_packages.yml + - .gitlab/ci/jobs/build/oc.build:rpm:amd64.yml + - .gitlab/ci/jobs/build/oc.build:dpkg:amd64.yml - .gitlab/ci/jobs/build/select_tezts.yml # These jobs are necessary to run flaky tezts diff --git a/ci/README.md b/ci/README.md index 621b8f22c115b0dce0c0a556a8cc2f2062b285d2..0914c078ec55474559e5349ba7479060cdfaf9fd 100644 --- a/ci/README.md +++ b/ci/README.md @@ -13,10 +13,10 @@ This directory is structured like this: ## Usage -To regenerate `.gitlab-ci.yml` (from the root of the repo): +To regenerate the GitLab CI configuration, run (from the root of the repo): make -C ci all -To check that `.gitlab-ci.yml` is up-to-date (from the root of the repo): +To check that the GitLab CI configuration is up-to-date, run (from the root of the repo): make -C ci check diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 1a8ad4d7d7ab436091238f78466e2e7d0246ae4b..a48b0f88d24b6aa7d829417bd717a32b65071a6d 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -8,7 +8,7 @@ (* Main entrypoint of CI-in-OCaml. Here we register the set of pipelines, stages and images and - generate the top-level [.gitlab-ci.yml] file. *) + generate the GitLab CI configuration file. *) open Gitlab_ci open Gitlab_ci.Types @@ -23,11 +23,11 @@ let default = default ~interruptible:true () The "manual" stage exists to fix a UI problem that occurs when mixing manual and non-manual jobs. *) module Stages = struct - let _trigger = Stage.register "trigger" + let trigger = Stage.register "trigger" let _sanity = Stage.register "sanity" - let _build = Stage.register "build" + let build = Stage.register "build" let _test = Stage.register "test" @@ -37,15 +37,15 @@ module Stages = struct let _doc = Stage.register "doc" - let _prepare_release = Stage.register "prepare_release" + let prepare_release = Stage.register "prepare_release" - let _publish_release_gitlab = Stage.register "publish_release_gitlab" + let publish_release_gitlab = Stage.register "publish_release_gitlab" let publish_release = Stage.register "publish_release" - let _publish_package_gitlab = Stage.register "publish_package_gitlab" + let publish_package_gitlab = Stage.register "publish_package_gitlab" - let _manual = Stage.register "manual" + let manual = Stage.register "manual" end (* Get the [build_deps_image_version] from the environment, which is @@ -124,13 +124,13 @@ module Images = struct ~image_path: "${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version}" - let _runtime_build_test_dependencies = + let runtime_build_test_dependencies = Image.register ~name:"runtime_build_test_dependencies" ~image_path: "${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version}" - let _runtime_build_dependencies = + let runtime_build_dependencies = Image.register ~name:"runtime_build_dependencies" ~image_path: @@ -175,10 +175,51 @@ module Images = struct used for the [build_deps_image_name] images and specified in the variable [alpine_version] in [scripts/version.sh]. This is checked by the jobs [trigger] and [sanity_ci]. *) - let _alpine = + let alpine = Image.register ~name:"alpine" ~image_path:("alpine:" ^ alpine_version) + + let debian_bookworm = + Image.register ~name:"debian_bookworm" ~image_path:"debian:bookworm" + + let fedora_39 = Image.register ~name:"fedora_39" ~image_path:"fedora:39" + + let ci_release = + Image.register + ~name:"ci_release" + ~image_path:"${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" end +let before_script ?(take_ownership = false) ?(source_version = false) + ?(eval_opam = false) ?(init_python_venv = false) ?(install_js_deps = false) + before_script = + let toggle t x = if t then [x] else [] in + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 *) + toggle take_ownership "./scripts/ci/take_ownership.sh" + @ toggle source_version ". ./scripts/version.sh" + (* TODO: this must run in the before_script of all jobs that use the opam environment. + how to enforce? *) + @ toggle eval_opam "eval $(opam env)" + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables *) + @ toggle init_python_venv ". $HOME/.venv/bin/activate" + @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" + @ before_script + +let changeset_octez = + [ + "src/**/*"; + "etherlink/**/*"; + "tezt/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + "michelson_test_scripts/**/*"; + "tzt_reference_test_suite/**/*"; + ] + +let changeset_octez_or_kernels = + ["images/**/*"; "scripts/ci/**/*"; "kernels.mk"; "etherlink.mk"] + @ changeset_octez + (* Dummy job. This fixes the "configuration must contain at least one @@ -193,19 +234,64 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () +(* Define the [trigger] job + + §1: The purpose of this job is to launch the CI manually in certain cases. + The objective is not to run computing when it is not + necessary and the decision to do so belongs to the developer + + §2: We also perform some fast sanity checks. *) +let trigger = + job_external + @@ job + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + (** Helper to create jobs that uses the docker deamon. It: - Sets the appropriate image. - Activates the Docker daemon as a service. - It sets up authentification with docker registries *) -let job_docker_authenticated ?variables ~stage ~name script : job = +let job_docker_authenticated ?(skip_docker_initialization = false) ?artifacts + ?variables ?rules ?dependencies ?arch ?when_ ?allow_failure ~stage ~name + script : job = let docker_version = "24.0.6" in job + ?rules + ?dependencies + ?artifacts + ?arch + ?when_ + ?allow_failure ~image:Images.docker ~variables: ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) - ~before_script:["./scripts/ci/docker_initialize.sh"] + ~before_script: + (if not skip_docker_initialization then + ["./scripts/ci/docker_initialize.sh"] + else []) ~services:[{name = "docker:${DOCKER_VERSION}-dind"}] ~stage ~name @@ -218,6 +304,502 @@ let job_docker_promote_to_latest ~ci_docker_hub : job = ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] ["./scripts/ci/docker_promote_to_latest.sh"] +(* This version of the job builds both released and experimental executables. + It is used in the following pipelines: + - Before merging: check whether static executables still compile, + i.e. that we do pass the -static flag and that when we do it does compile + - Master branch: executables (including experimental ones) are used in some test networks + Variants: + - an arm64 variant exist, but is only used in the master branch pipeline + (no need to test that we pass the -static flag twice) + - released variants exist, that are used in release tag pipelines + (they do not build experimental executables) *) +let job_build_static_binaries ~arch ?(release = false) ?(needs_trigger = false) + ?rules () = + let arch_string = + match arch with Tezos_ci.Amd64 -> "x86_64" | Arm64 -> "arm64" + in + let name = "oc.build:static-" ^ arch_string ^ "-linux-binaries" in + let artifacts = + (* Extend the lifespan to prevent failure for external tools using artifacts. *) + let expire_in = if release then Some (Days 90) else None in + artifacts ?expire_in ["octez-binaries/$ARCH/*"] + in + let executable_files = + "script-inputs/released-executables" + ^ if not release then " script-inputs/experimental-executables" else "" + in + let dependencies = + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration of + pipelines. So when requested through [needs_trigger] we start + this job as early as possible, without waiting for + sanity_ci. *) + if needs_trigger then Dependent [Optional trigger] else Staged [] + in + job + ?rules + ~stage:Stages.build + ~arch + ~name + ~image:Images.runtime_build_dependencies + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + ~variables:[("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + ~dependencies + ~artifacts + ["./scripts/ci/build_static_binaries.sh"] + +let rules_static_build_master = [job_rule ~when_:Always ()] + +let rules_static_build_other = [job_rule ~changes:changeset_octez ()] + +let _job_static_arm64_experimental = + job_external ~filename_suffix:"experimental" + @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_other () + +let _job_static_arm64_master = + job_external ~filename_suffix:"master" + @@ job_build_static_binaries ~arch:Arm64 ~rules:rules_static_build_master () + +let _job_static_x86_64_experimental = + job_external ~filename_suffix:"experimental" + @@ job_build_static_binaries + ~arch:Amd64 + ~needs_trigger:true + ~rules:rules_static_build_other + () + +let _job_static_x86_64_master = + job_external ~filename_suffix:"master" + @@ job_build_static_binaries + ~arch:Amd64 + (* TODO: this job doesn't actually need trigger and there is no + need to set it optional since we know this job is only on the master branch. *) + ~needs_trigger:true + ~rules:rules_static_build_master + () + +let job_docker_rust_toolchain ?rules ?dependencies () = + job_docker_authenticated + ?rules + ?dependencies + ~skip_docker_initialization:true + ~stage:Stages.build + ~name:"oc.docker:rust-toolchain" + ~variables:[("CI_DOCKER_HUB", "false")] + ~artifacts: + (artifacts + ~reports:(reports ~dotenv:"rust_toolchain_image_tag.env" ()) + []) + ["./scripts/ci/docker_rust_toolchain_build.sh"] + +let _job_docker_rust_toolchain_before_merging = + job_external ~filename_suffix:"before_merging" + @@ job_docker_rust_toolchain + ~dependencies:(Dependent [Optional trigger]) + ~rules: + [ + job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); + job_rule ~when_:Manual (); + ] + () + +let _job_docker_rust_toolchain_master = + job_external ~filename_suffix:"master" + @@ job_docker_rust_toolchain ~rules:[job_rule ~when_:Always ()] () + +let _job_docker_rust_toolchain_other = + job_external ~filename_suffix:"other" @@ job_docker_rust_toolchain () + +(** Type of Docker build jobs. + + The semantics of the type is summed up in this table: + + | | Release | Experimental | Test | Test_manual | + |-----------------------+------------+--------------+--------+-------------| + | Image registry | Docker hub | Docker hub | GitLab | GitLab | + | Experimental binaries | no | yes | yes | yes | + | EVM Kernels | no | On amd64 | no | On amd64 | + | Manual job | no | no | no | yes | + + - [Release] Docker builds include only released executables whereas other + types also includes experimental ones. + - [Test_manual] and [Experimental] Docker builds include the EVM kernels in + amd64 builds. + - [Release] and [Experimental] Docker builds are pushed to Docker hub, + whereas other types are pushed to the GitLab registry. + - [Test_manual] Docker builds are triggered manually, put in the stage + [manual] and their failure is allowed. The other types are in the build + stage, run [on_success] and are not allowed to fail. *) +type docker_build_type = Experimental | Release | Test | Test_manual + +(** Creates a Docker build job of the given [arch] and [docker_build_type]. + + If [external_] is set to true (default [false]), then the job is + also written to an external file. *) +let job_docker_build ?rules ?dependencies ~arch ?(external_ = false) + docker_build_type : job = + let arch_string = + match arch with Tezos_ci.Amd64 -> "amd64" | Arm64 -> "arm64" + in + let variables = + [ + ( "DOCKER_BUILD_TARGET", + match (arch, docker_build_type) with + | Amd64, (Test_manual | Experimental) -> "with-evm-artifacts" + | _ -> "without-evm-artifacts" ); + ("IMAGE_ARCH_PREFIX", arch_string ^ "_"); + ( "CI_DOCKER_HUB", + Bool.to_string + (match docker_build_type with + | Release | Experimental -> true + | Test | Test_manual -> false) ); + ( "EXECUTABLE_FILES", + match docker_build_type with + | Release -> "script-inputs/released-executables" + | Test | Test_manual | Experimental -> + "script-inputs/released-executables \ + script-inputs/experimental-executables" ); + ] + in + let stage, when_, (allow_failure : allow_failure_job option) = + match docker_build_type with + | Test_manual -> (Stages.manual, Some Manual, Some Yes) + | _ -> (Stages.build, None, None) + in + let name = "oc.docker:" ^ arch_string in + let filename_suffix = + match docker_build_type with + | Release -> "release" + | Experimental -> "experimental" + | Test -> "test" + | Test_manual -> "test_manual" + in + let job = + job_docker_authenticated + ?when_ + ?allow_failure + ?rules + ?dependencies + ~stage + ~arch + ~name + ~variables + ["./scripts/ci/docker_release.sh"] + in + if external_ then job_external ~directory:"build" ~filename_suffix job + else job + +let changeset_octez_docker_changes_or_master = + [ + "scripts/**/*"; + "script-inputs/**/*"; + "src/**/*"; + "tezt/**/*"; + "vendors/**/*"; + "dune"; + "dune-project"; + "dune-workspace"; + "opam"; + "Makefile"; + "kernels.mk"; + "build.Dockerfile"; + "Dockerfile"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + +let rules_octez_docker_changes_or_master = + [ + job_rule ~if_:Rules.on_master ~when_:Always (); + job_rule ~changes:changeset_octez_docker_changes_or_master (); + ] + +let job_docker_amd64_experimental : job = + job_docker_build + ~external_:true + (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. + For now we can set any variant to get the correct name of the need. *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Experimental + +let _job_docker_amd64_test_manual : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies: + (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) + ~arch:Amd64 + Test_manual + +let job_docker_arm64_experimental : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Experimental + +let _job_docker_arm64_test_manual : job = + job_docker_build + ~external_:true (* TODO: see above *) + ~dependencies: + (Dependent [Artifacts _job_docker_rust_toolchain_before_merging]) + ~arch:Arm64 + Test_manual + +(* Note: here we rely on [$IMAGE_ARCH_PREFIX] to be empty. + Otherwise, [$DOCKER_IMAGE_TAG] would contain [$IMAGE_ARCH_PREFIX] too. + [$IMAGE_ARCH_PREFIX] is only used when building Docker images, + here we handle all architectures so there is no such variable. *) +let job_docker_merge_manifests ~ci_docker_hub ~job_docker_amd64 + ~job_docker_arm64 : job = + job_docker_authenticated + ~stage:Stages.prepare_release + ~name:"docker:merge_manifests" + (* This job merges the images produced in the jobs + [docker:{amd64,arm64}] into a single multi-architecture image, and + so must be run after these jobs. *) + ~dependencies:(Dependent [Job job_docker_amd64; Job job_docker_arm64]) + ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ["./scripts/ci/docker_merge_manifests.sh"] + +let _job_docker_merge_manifests_release = + job_external ~filename_suffix:"release" + @@ job_docker_merge_manifests + ~ci_docker_hub:true + (* TODO: In theory, actually uses either release or + experimental variant of docker jobs depending on + pipeline. In practice, this does not matter as these jobs + have the same name in the generated files + ([oc.build:ARCH]). However, when the merge_manifest jobs + are created directly in the appropriate pipeline, the + correcty variant must be used. *) + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + +type bin_package_target = Dpkg | Rpm + +let job_build_bin_package ?rules ~name ?(stage = Stages.build) ~arch ~target () + : job = + let arch_string = + match arch with Tezos_ci.Amd64 -> "amd64" | Arm64 -> "arm64" + in + let target_string = match target with Dpkg -> "dpkg" | Rpm -> "rpm" in + let image = + match target with Dpkg -> Images.debian_bookworm | Rpm -> Images.fedora_39 + in + let artifacts = + let artifact_path = + "octez-*." ^ match target with Dpkg -> "deb" | Rpm -> "rpm" + in + artifacts + ~expire_in:(Days 1) + ~when_:On_success + ~name:"${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG" + [artifact_path] + in + let before_script = + ". ./scripts/version.sh" + :: + (match target with + | Dpkg -> + [ + "apt update"; + "apt-get install -y rsync git m4 build-essential patch unzip wget \ + opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ + libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev \ + protobuf-compiler libsqlite3-dev"; + ] + | Rpm -> + [ + "dnf update -y"; + "dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel \ + zlib-devel libpq-devel m4 perl git pkg-config rpmdevtools \ + python3-devel python3-setuptools wget opam rsync which cargo \ + autoconf mock systemd systemd-rpm-macros cmake python3-wheel \ + python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel \ + sqlite-devel"; + ]) + in + job + ?rules + ~name + ~arch + ~image + ~stage + ~dependencies:(Dependent []) + ~variables: + [ + ("TARGET", target_string); + ("OCTEZ_PKGMAINTAINER", "nomadic-labs"); + ("BLST_PORTABLE", "yes"); + ("ARCH", arch_string); + ] + ~artifacts + ~before_script + [ + "wget https://sh.rustup.rs/rustup-init.sh"; + "chmod +x rustup-init.sh"; + "./rustup-init.sh --profile minimal --default-toolchain \ + $recommended_rust_version -y"; + ". $HOME/.cargo/env"; + "export OPAMYES=\"true\""; + "opam init --bare --disable-sandboxing"; + "make build-deps"; + "eval $(opam env)"; + "make $TARGET"; + ] + +let job_build_dpkg_amd64 = + job_external + @@ job_build_bin_package + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + () + +let job_build_rpm_amd64 = + job_external + @@ job_build_bin_package + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + () + +let _job_build_dpkg_amd64_manual = + job_external ~directory:"build" ~filename_suffix:"manual" + @@ job_build_bin_package + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + +let _job_build_rpm_amd64_manual = + job_external ~directory:"build" ~filename_suffix:"manual" + @@ job_build_bin_package + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + +(** Type of release tag pipelines. + + The semantics of the type is summed up in this table: + + | | Release_tag | Beta_release_tag | Non_release_tag | + |-----------------------+-------------+------------------+-----------------| + | GitLab release type | Release | Release | Create | + | Experimental binaries | No | No | No | + | Docker build type | Release | Release | Release | + | Publishes to opam | Yes | No | No | + + - All release tag pipelines types publish [Release] type Docker builds. + - No release tag pipelines include experimental binaries. + - [Release_tag] and [Beta_release_tag] pipelines creates GitLab + and publishes releases. [Non_release_tag] pipelines creates the + GitLab release but do not publish them. + - Only [Release_tag] pipelines publish to opam. *) +type release_tag_pipeline_type = + | Release_tag + | Beta_release_tag + | Non_release_tag + +(** Create a release tag pipeline of type {!release_tag_pipeline_type}. + + If [test] is true (default is [false]), then the Docker images are + built of the [Test] type and are published to the GitLab registry + instead of Docker hub. *) +let release_tag_pipeline ?(test = false) release_tag_pipeline_type = + let job_docker_rust_toolchain = job_docker_rust_toolchain () in + let job_docker_amd64 = + job_docker_build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Amd64 + (if test then Test else Release) + in + let job_docker_arm64 = + job_docker_build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~arch:Arm64 + (if test then Test else Release) + in + let job_docker_merge = + job_docker_merge_manifests + ~ci_docker_hub:(not test) + ~job_docker_amd64 + ~job_docker_arm64 + in + let job_static_arm64_release = + job_build_static_binaries ~arch:Arm64 ~release:true () + in + let job_static_x86_64_release = + job_build_static_binaries ~arch:Amd64 ~release:true ~needs_trigger:true () + in + let job_gitlab_release ~dependencies : job = + job + ~image:Images.ci_release + ~stage:Stages.publish_release_gitlab + ~interruptible:false + ~dependencies + ~name:"gitlab:release" + [ + "./scripts/ci/restrict_export_to_octez_source.sh"; + "./scripts/ci/gitlab-release.sh"; + ] + in + let job_gitlab_publish ~dependencies : job = + job + ~image:Images.ci_release + ~stage:Stages.publish_package_gitlab + ~interruptible:false + ~dependencies + ~name:"gitlab:publish" + ["${CI_PROJECT_DIR}/scripts/ci/create_gitlab_package.sh"] + in + let job_gitlab_release_or_publish = + let dependencies = + Dependent + [ + Artifacts job_static_x86_64_release; + Artifacts job_static_arm64_release; + Artifacts job_build_dpkg_amd64; + Artifacts job_build_rpm_amd64; + ] + in + match release_tag_pipeline_type with + | Non_release_tag -> job_gitlab_publish ~dependencies + | _ -> job_gitlab_release ~dependencies + in + let job_opam_release : job = + job + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.publish_release + ~interruptible:false + ~name:"opam:release" + ["./scripts/ci/opam-release.sh"] + in + [ + job_docker_rust_toolchain; + job_static_x86_64_release; + job_static_arm64_release; + job_docker_amd64; + job_docker_arm64; + job_build_dpkg_amd64; + job_build_rpm_amd64; + job_docker_merge; + job_gitlab_release_or_publish; + ] + @ + match (test, release_tag_pipeline_type) with + | false, Release_tag -> [job_opam_release] + | _ -> [] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -249,19 +831,24 @@ let () = register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; register "release_tag" - If.(on_tezos_namespace && push && has_tag_match release_tag_re) ; + If.(on_tezos_namespace && push && has_tag_match release_tag_re) + ~jobs:(release_tag_pipeline Release_tag) ; register "beta_release_tag" - If.(on_tezos_namespace && push && has_tag_match beta_release_tag_re) ; + If.(on_tezos_namespace && push && has_tag_match beta_release_tag_re) + ~jobs:(release_tag_pipeline Beta_release_tag) ; register "release_tag_test" - If.(not_on_tezos_namespace && push && has_any_release_tag) ; + If.(not_on_tezos_namespace && push && has_any_release_tag) + ~jobs:(release_tag_pipeline ~test:true Release_tag) ; register "non_release_tag" - If.(on_tezos_namespace && push && has_non_release_tag) ; + If.(on_tezos_namespace && push && has_non_release_tag) + ~jobs:(release_tag_pipeline Non_release_tag) ; register "non_release_tag_test" - If.(not_on_tezos_namespace && push && has_non_release_tag) ; + If.(not_on_tezos_namespace && push && has_non_release_tag) + ~jobs:(release_tag_pipeline ~test:true Non_release_tag) ; register "schedule_extended_test" schedule_extended_tests (* Split pipelines and writes image templates *) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 5fe34f097aa2b8367faf091258d734fd504c21f4..84d2f09a058424bbbcb41a86736937942badf0af 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -114,7 +114,7 @@ module Pipeline = struct (* Check that all [dependencies:] are on jobs that produce artifacts *) ( Fun.flip String_set.iter dependencies @@ fun dependency -> match Hashtbl.find_opt job_by_name dependency with - | Some {artifacts = Some {paths = _ :: _; _}; _} + | Some {artifacts = Some {paths = Some (_ :: _); _}; _} | Some {artifacts = Some {reports = Some {dotenv = Some _; _}; _}; _} -> (* This is fine: we depend on a job that define non-report artifacts, or a dotenv file. *) () @@ -326,3 +326,38 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache retry; parallel; } + +let external_jobs = ref String_set.empty + +let job_external ?directory ?filename_suffix (job : Gitlab_ci.Types.job) : + Gitlab_ci.Types.job = + let stage = + match job.stage with + | Some stage -> stage + | None -> + (* Test is the name of the default stage in GitLab CI *) + "test" + in + let basename = + match filename_suffix with + | None -> job.name + | Some suffix -> job.name ^ "-" ^ suffix + in + let directory = ".gitlab/ci/jobs" // Option.value ~default:stage directory in + if not (Sys.file_exists directory && Sys.is_directory directory) then + failwith + "[job_external] attempted to write job '%s' to non-existing directory \ + '%s'" + job.name + directory ; + let filename = (directory // basename) ^ ".yml" in + if String_set.mem filename !external_jobs then + failwith + "Attempted to write external job %s twice -- perhaps you need to set \ + filename_suffix?" + filename + else ( + external_jobs := String_set.add filename !external_jobs ; + let config = [Gitlab_ci.Types.Job job] in + Gitlab_ci.To_yaml.to_file ~header ~filename config ; + job) diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 50721847630d3c054c6a74fdcf54294d1d6cde13..76da952ff890ace048bf49058c1d0e215edcfd91 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -186,3 +186,23 @@ val job : name:string -> string list -> Gitlab_ci.Types.job + +(** Generates a job to an external file. + + This function is meant to be used in the transition to CI-in-OCaml. + It writes {!header} and the given job to the destination path + [.gitlab/ci/jobs/DIRECTORY/NAME(-FILENAME_SUFFIX).yml]. + Directory defaults to the stage name if not set. + + This allows migrating all the jobs of a given pipeline, and + including the generated definition of those jobs in other + pipelines where it appears. + + Raises [Failure] if [.gitlab/ci/jobs/DIRECTORY] is not an existing + directory. Also [Failure] if destination path has already been + used to write another job. *) +val job_external : + ?directory:string -> + ?filename_suffix:string -> + Gitlab_ci.Types.job -> + Gitlab_ci.Types.job diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 97cdd86df5a162c9b980c39daa0d5010d1877472..feab427704ff59370f3ab577d64c5b5083703fb1 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -158,7 +158,7 @@ let enc_artifacts : artifacts -> value = [ opt "name" string name; opt "expire_in" enc_time_interval expire_in; - key "paths" strings paths; + opt "paths" strings paths; opt "reports" enc_report reports; opt "when" enc_when_artifact when_; opt "expose_as" string expose_as; diff --git a/ci/lib_gitlab_ci/types.ml b/ci/lib_gitlab_ci/types.ml index dbc6fd1db684c7c73529cd8aa8ec833407e62691..54e28eb412c6cd2eaae13088ca52ca69b5db7da4 100644 --- a/ci/lib_gitlab_ci/types.ml +++ b/ci/lib_gitlab_ci/types.ml @@ -78,7 +78,7 @@ type when_artifact = Always | On_success | On_failure type artifacts = { expire_in : time_interval option; - paths : string list; + paths : string list option; reports : reports option; when_ : when_artifact option; expose_as : string option; diff --git a/ci/lib_gitlab_ci/util.ml b/ci/lib_gitlab_ci/util.ml index 62a3c9e62949840686a732cf9b1a20c6931444e3..1a4a1389373c91edd4b2e01b472183d0d78e4261 100644 --- a/ci/lib_gitlab_ci/util.ml +++ b/ci/lib_gitlab_ci/util.ml @@ -65,7 +65,14 @@ let artifacts ?expire_in ?reports ?when_ ?expose_as ?name paths = "Attempted to register an artifact with no reports or paths -- this \ doesn't make any sense" | _ -> ()) ; - {expire_in; paths; reports; when_; expose_as; name} + { + expire_in; + paths = (if paths = [] then None else Some paths); + reports; + when_; + expose_as; + name; + } let reports ?dotenv ?junit ?coverage_report () = (match (dotenv, junit, coverage_report) with