From 602150fb9bb19e17b1db584c4c0c55085db246cb Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Tue, 18 Jun 2024 10:00:31 +0200 Subject: [PATCH 1/2] scripts: Add kisscache wrapper --- scripts/kiss-fetch.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 scripts/kiss-fetch.sh diff --git a/scripts/kiss-fetch.sh b/scripts/kiss-fetch.sh new file mode 100755 index 000000000000..1786726559f8 --- /dev/null +++ b/scripts/kiss-fetch.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script is a wrapper for opam to use +# a kisscache server, for example located at +# http://localhost:8001/api/v1/fetch/?url= + +# kisscache: https://gitlab.com/Linaro/kisscache + +# To enable this wrapper +# +# export OPAMFETCH="scripts/kiss-fetch.sh" +# +# before invoking opam +# +# Use the evn var KISSCACHE to specify a remote kisscache server. + +KISSCACHE=${KISSCACHE:-http://localhost:8001} + +# Original URL passed by opam +original_url=${*: -1} +arguments=${*:1:$#-1} + +# Mangle the URL by prepending the base URL +mangled_url="${KISSCACHE}/api/v1/fetch/?url=${original_url}" + +# shellcheck disable=SC2086 +if curl --output /dev/null --silent --head --fail "$KISSCACHE"; then + # Use curl to fetch the mangled URL + curl $arguments $mangled_url +else + # fallback + curl $arguments $original_url +fi -- GitLab From 08b70404f38f801ef030cc1ce970e8088dcda93e Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Tue, 16 Jul 2024 10:36:31 +0200 Subject: [PATCH 2/2] ci: add kisscache env vars --- .gitlab-ci.yml | 2 ++ .gitlab/ci/pipelines/before_merging.yml | 10 ++++++++++ .gitlab/ci/pipelines/non_release_tag.yml | 10 ++++++++++ .gitlab/ci/pipelines/non_release_tag_test.yml | 10 ++++++++++ .gitlab/ci/pipelines/octez_beta_release_tag.yml | 10 ++++++++++ .gitlab/ci/pipelines/octez_release_tag.yml | 10 ++++++++++ .gitlab/ci/pipelines/octez_release_tag_test.yml | 10 ++++++++++ .gitlab/ci/pipelines/schedule_extended_test.yml | 10 ++++++++++ ci/bin/common.ml | 6 ++++++ ci/bin/main.ml | 2 ++ scripts/ci/bin_packages_deb_dependencies.sh | 2 +- 11 files changed, 81 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 555fd97e305d..0848e9856f23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,6 +75,8 @@ variables: RUNTEZTALIAS: "false" CARGO_HOME: $CI_PROJECT_DIR/.cargo CARGO_NET_OFFLINE: "true" + OPAMFETCH: scripts/kiss-fetch.sh + KISSCACHE: http://p2p-monitoring-sandbox.nomadic-labs.eu dummy_job: stage: .pre diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index fb0cb137d8f2..4d3f132ab303 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -3473,11 +3473,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -3521,11 +3526,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 1a58eca019f7..0b45029ff2de 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -164,11 +164,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -209,11 +214,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index ad4dfd55c328..3ae55f2819cf 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -164,11 +164,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -209,11 +214,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 58d8cec34806..b4d6e9feab6f 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -164,11 +164,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -209,11 +214,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index 9a4efc3fcb5f..b0798f6b8823 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -165,11 +165,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -210,11 +215,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 26254bb74d71..f32a586b2908 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -166,11 +166,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -211,11 +216,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index f025c47034b5..da59ad417e97 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -506,11 +506,16 @@ oc.build:dpkg:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_deb_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps @@ -551,11 +556,16 @@ oc.build:rpm:amd64: - . ./scripts/version.sh - ./scripts/ci/bin_packages_rpm_dependencies.sh script: + - wget https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux + -O /usr/local/bin/opam + - chmod +x /usr/local/bin/opam - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - . $HOME/.cargo/env + - export OPAMFETCH="scripts/kiss-fetch.sh" + - export KISSCACHE="http://p2p-monitoring-sandbox.nomadic-labs.eu" - export OPAMYES="true" - opam init --bare --disable-sandboxing - make build-deps diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 665b15eb640a..30b97a3f357a 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -904,11 +904,17 @@ let job_build_bin_package ?dependencies ?rules ~__POS__ ~name ~parallel ~before_script [ + "wget \ + https://github.com/ocaml/opam/releases/download/2.2.0/opam-2.2.0-x86_64-linux \ + -O /usr/local/bin/opam"; + "chmod +x /usr/local/bin/opam"; "wget https://sh.rustup.rs/rustup-init.sh"; "chmod +x rustup-init.sh"; "./rustup-init.sh --profile minimal --default-toolchain \ $recommended_rust_version -y"; ". $HOME/.cargo/env"; + "export OPAMFETCH=\"scripts/kiss-fetch.sh\""; + "export KISSCACHE=\"http://p2p-monitoring-sandbox.nomadic-labs.eu\""; "export OPAMYES=\"true\""; "opam init --bare --disable-sandboxing"; "make build-deps"; diff --git a/ci/bin/main.ml b/ci/bin/main.ml index da69c382a686..c8edc60ff11e 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -61,6 +61,8 @@ let variables : variables = (* To avoid Cargo accessing the network in jobs without caching (see {!Common.enable_cargo_cache}), we turn of net access by default. *) ("CARGO_NET_OFFLINE", "true"); + ("OPAMFETCH", "scripts/kiss-fetch.sh"); + ("KISSCACHE", "http://p2p-monitoring-sandbox.nomadic-labs.eu"); ] (* Register pipelines types. Pipelines types are used to generate diff --git a/scripts/ci/bin_packages_deb_dependencies.sh b/scripts/ci/bin_packages_deb_dependencies.sh index 1ad51685c769..b69e8233dbac 100755 --- a/scripts/ci/bin_packages_deb_dependencies.sh +++ b/scripts/ci/bin_packages_deb_dependencies.sh @@ -4,6 +4,6 @@ export DEBIAN_FRONTEND=noninteractive apt update apt-get install -y rsync git m4 build-essential patch unzip wget \ - opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ + curl opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev \ protobuf-compiler libsqlite3-dev -- GitLab