From 5043090b9a28f9f9b194c2ea99da309655fd2ba2 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Wed, 23 Oct 2024 15:28:34 +0200 Subject: [PATCH 1/4] packages: add tool to create rpm packages --- scripts/packaging/deb-to-rpm/README.md | 0 scripts/packaging/deb-to-rpm/deb-to-rpm.py | 279 ++++++++++++++++++++ scripts/packaging/deb-to-rpm/poetry.lock | 85 ++++++ scripts/packaging/deb-to-rpm/pyproject.toml | 18 ++ 4 files changed, 382 insertions(+) create mode 100644 scripts/packaging/deb-to-rpm/README.md create mode 100755 scripts/packaging/deb-to-rpm/deb-to-rpm.py create mode 100644 scripts/packaging/deb-to-rpm/poetry.lock create mode 100644 scripts/packaging/deb-to-rpm/pyproject.toml diff --git a/scripts/packaging/deb-to-rpm/README.md b/scripts/packaging/deb-to-rpm/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/packaging/deb-to-rpm/deb-to-rpm.py b/scripts/packaging/deb-to-rpm/deb-to-rpm.py new file mode 100755 index 000000000000..f9e370af76cd --- /dev/null +++ b/scripts/packaging/deb-to-rpm/deb-to-rpm.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 + +import os +import sys +from datetime import datetime +from debian import deb822 +from specfile import Specfile +import argparse + + +def generate_spec_file(file_path): + content = """Name: test +Version: 1.0.0 +Release: a +Summary: test +License: MIT +Requires: a +Recommends: a +Suggests: a +Requires(pre): a +Conflicts: a +Enhances: a +%description +%install +%files +""" + + # Create and write the content to the spec file + with open(file_path, 'w') as spec_file: + spec_file.write(content) + + +# List of Debian-specific dependencies to remove or skip +DEBIAN_SPECIFIC_DEPS = [ + "debconf", "${misc:Depends}", "${shlibs:Depends}", + "dpkg", "dpkg-dev", "apt", "apt-utils" + # Add other Debian-specific dependencies here +] + +# Table for mapping Debian packages to RPM packages +DEB_TO_RPM_MAPPING = { + "adduser": "shadow-utils", + "zlib1g": "zlib", + # Add other mappings as needed +} + + +def remove_if_contains(items, substrings): + return [item.strip() for item in items if + not any(sub in item for sub in substrings)] + +def strip_specific_version_constraint(dep): + """ + Remove the specific version constraint of the form "(= ${source:Version})". + Example: "zlib1g (= ${source:Version})" -> "zlib1g" + """ + # Remove the specific pattern if it matches + return dep.replace("(= ${source:Version})", "").strip() + +def parse_debian_dependencies(deps): + """ + Parse a string of Debian dependencies and return a logical representation + with AND (,) and OR (|) operators as nested lists. Then remove any + sublists that contain Debian-specific dependencies. + + Returns: + List of lists: AND/OR representation of dependencies. + """ + dependencies = [] + + # Split dependencies by comma (AND) + and_dependencies = deps.split(',') + + for dep_group in and_dependencies: + dep_group = dep_group.strip() + + if '|' in dep_group: + # Handle OR conditions by splitting with | + or_dependencies = [strip_specific_version_constraint(dep.strip()) for dep in dep_group.split('|')] + if or_dependencies: # Only append if something is left + dependencies.append(or_dependencies) + else: + dep_group = strip_specific_version_constraint(dep_group) + # Handle single dependency directly (AND) + if dep_group: # Only append if something is left + dependencies.append([dep_group]) + + return dependencies + +def convert_deps(deps): + """Convert Debian dependencies to RPM format, remove Debian-specific dependencies.""" + + deps_list = [] + for d in parse_debian_dependencies(deps): + if isinstance(d,list): + deps_list.append(remove_if_contains(d,DEBIAN_SPECIFIC_DEPS)) + else: + deps_list.append(remove_if_contains([d],DEBIAN_SPECIFIC_DEPS)) + + # Create a list to store RPM dependencies + rpm_deps = [] + + for deps in deps_list: + for deb_pkg in deps: + + rpm_pkg = DEB_TO_RPM_MAPPING.get(deb_pkg, deb_pkg) # Map Debian package to RPM + # Add the RPM dependency to the list if it's not empty + if rpm_pkg: + rpm_deps.append(rpm_pkg) + + # Return the cleaned and mapped dependencies as a string + return " ".join(rpm_deps) + +def process_debian_control(debian_dir, spec_dir): + """Process the Debian control file and generate .spec files.""" + + control_file = os.path.join(debian_dir, 'control') + + # Ensure the spec directory exists + os.makedirs(spec_dir, exist_ok=True) + + with open(control_file, 'r') as f: + # Parse the control file with multiple stanzas + control_data = deb822.Packages.iter_paragraphs(f) + + for package in control_data: + # We're only interested in binary packages + if package.get('Source'): + continue + + package_name = package.get('Package', '') + version = package.get('Version', '1.0.0') + arch = package.get('Architecture', 'noarch') + depends = package.get('Depends', '') + pre_depends = package.get('Pre-Depends', '') + recommends = package.get('Recommends', '') + suggests = package.get('Suggests', '') + breaks = package.get('Breaks', '') + conflicts = package.get('Conflicts', '') + enhances = package.get('Enhances', '') + + full_description = package.get('Description', '') + + # Split description into first line and the rest + if full_description: + description_lines = full_description.splitlines() + summury = description_lines[0] # First line + description = description_lines[1:] + else: + summury = '' + description = '' + + if not package_name: + continue + + # Create a new .spec file using packit.specfile.Specfile + spec_path = os.path.join(spec_dir, f"{package_name}.spec") + generate_spec_file(spec_path) + spec = Specfile( + spec_path, sourcedir='/tmp/sources' + ) # ,sources=None, changelog_entry=None) + + # Basic info for spec file + spec.name = package_name + spec.version = version + spec.release = '1%{?dist}' + spec.summary = summury + spec.license = 'MIT' + spec.buildArch = arch if arch != 'all' else 'noarch' + + with spec.tags() as tags: + print(spec.name) + print([t.name for t in tags]) + + # Define a mapping of RPM tags to the corresponding Debian fields + deps_mapping = { + 'requires': depends, + 'requires(pre)': pre_depends, + 'recommends': recommends, + 'suggests': suggests, + 'conflicts': f"{conflicts} {breaks}".strip(), + 'enhances': enhances, + } + + # Convert and set the corresponding RPM tags if they have values + for tag, deb_field in deps_mapping.items(): + rpm_deps = convert_deps(deb_field) + if rpm_deps: + spec.update_tag(tag, rpm_deps) + else: + with spec.tags() as tags: + delattr(tags, tag) + + # %description + with spec.sections() as sections: + sections.description = description + files_section = [] + install_section = [] + + install_file = os.path.join( + debian_dir, f"{package_name}.install" + ) + if os.path.exists(install_file): + with open(install_file, 'r') as install_f: + print(f"Considering install file {install_file}") + for line in install_f: + # Split each line into source and destination + line = line.strip() + if not line: + continue # Skip empty lines + src, dest = line.split() + name = os.path.basename(src) + + # Add the installation commands to the %install section + install_section.append( + f"mkdir -p %{{buildroot}}{dest}") + + install_section.append( + f"install -m 0755 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}{dest}") + + # Add the destination to the %files section + files_section.append(f"{dest}{name}") + + # Handle .manpage files if they exist + manpage_file = os.path.join( + debian_dir, f"{package_name}.manpage" + ) + if os.path.exists(manpage_file): + with open(manpage_file, 'r') as man_f: + for man_line in man_f: + files_section.append(f"{man_line.strip()}") + + # Add the install and files sections to the spec file + sections.install = install_section + sections.files = files_section + print(sections.install) + + # Save the generated .spec file + spec.save() + + print(f"Spec file for {package_name} created at {spec_path}") + + +def parse_args(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Generate RPM spec files from Debian control files." + ) + parser.add_argument( + '-d', + '--debian-dir', + required=True, + help="Directory containing Debian control, .install, and .manpage files.", + ) + parser.add_argument( + '-s', + '--spec-dir', + required=True, + help="Directory where the generated spec files will be saved.", + ) + return parser.parse_args() + + +def main(): + """Main function to handle argument parsing and processing.""" + args = parse_args() + + debian_dir = args.debian_dir + spec_dir = args.spec_dir + + if not os.path.isdir(debian_dir): + print(f"Error: '{debian_dir}' is not a valid directory.") + sys.exit(1) + + process_debian_control(debian_dir, spec_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/packaging/deb-to-rpm/poetry.lock b/scripts/packaging/deb-to-rpm/poetry.lock new file mode 100644 index 000000000000..3c6699ee843d --- /dev/null +++ b/scripts/packaging/deb-to-rpm/poetry.lock @@ -0,0 +1,85 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "argparse" +version = "1.4.0" +description = "Python command-line parsing library" +optional = false +python-versions = "*" +files = [ + {file = "argparse-1.4.0-py2.py3-none-any.whl", hash = "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"}, + {file = "argparse-1.4.0.tar.gz", hash = "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4"}, +] + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "python-debian" +version = "0.1.49" +description = "Debian package related modules" +optional = false +python-versions = ">=3.5" +files = [ + {file = "python-debian-0.1.49.tar.gz", hash = "sha256:8cf677a30dbcb4be7a99536c17e11308a827a4d22028dc59a67f6c6dd3f0f58c"}, + {file = "python_debian-0.1.49-py3-none-any.whl", hash = "sha256:880f3bc52e31599f2a9b432bd7691844286825087fccdcf2f6ffd5cd79a26f9f"}, +] + +[package.dependencies] +chardet = "*" + +[[package]] +name = "rpm" +version = "0.2.0" +description = "Shim RPM module for use in virtualenvs." +optional = false +python-versions = ">=3.6" +files = [ + {file = "rpm-0.2.0-py3-none-any.whl", hash = "sha256:4050b6033f7403be0a34f42a742c49ba74f2b0c6129f0247115b6078b24ddd71"}, + {file = "rpm-0.2.0.tar.gz", hash = "sha256:b92285f65c9ddf77678cb3e51aa67827426408fac34cdd8d537d8c14e3eaffbf"}, +] + +[package.extras] +testing = ["tox"] + +[[package]] +name = "specfile" +version = "0.32.4" +description = "A library for parsing and manipulating RPM spec files." +optional = false +python-versions = ">=3.6" +files = [ + {file = "specfile-0.32.4-py3-none-any.whl", hash = "sha256:f8f893f5b857c9e259d55cb0d539dcbdd9bb8088c1cc0cbdc5a43538157b8452"}, + {file = "specfile-0.32.4.tar.gz", hash = "sha256:a0bb5ed0981aa22e89162c4ac82c1c0a5209f0395bb50ab4140c05bae02bfc20"}, +] + +[package.dependencies] +rpm = "*" +typing-extensions = "*" + +[package.extras] +testing = ["flexmock", "pytest"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9.18" +content-hash = "f6485310503d1ca4f0e725796ea36e981c23e0a16fc9d37fc4469201b40eb7fc" diff --git a/scripts/packaging/deb-to-rpm/pyproject.toml b/scripts/packaging/deb-to-rpm/pyproject.toml new file mode 100644 index 000000000000..1eddc697a32d --- /dev/null +++ b/scripts/packaging/deb-to-rpm/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "deb-to-rpm" +version = "0.1.0" +description = "" +authors = ["Pietro Abate "] +readme = "README.md" +package-mode = false + +[tool.poetry.dependencies] +python = "^3.9.18" +python-debian = "^0.1.49" +argparse = "^1.4.0" +specfile = "^0.32.3" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" -- GitLab From e6442350ec4a9da8c64844840a63c507feccced9 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Wed, 23 Oct 2024 15:30:06 +0200 Subject: [PATCH 2/4] packages: add scripts to build rpm files --- rpm-deps-build.Dockerfile | 72 ++++++++++++ scripts/ci/build-rpm-packages.sh | 53 +++++++++ scripts/ci/create_rpm_repo.sh | 169 +++++++++++++++++++++++++++ scripts/ci/prepare-rpm-repo.sh | 18 +++ scripts/packaging/build-rpm-local.sh | 131 +++++++++++++++++++++ 5 files changed, 443 insertions(+) create mode 100644 rpm-deps-build.Dockerfile create mode 100755 scripts/ci/build-rpm-packages.sh create mode 100755 scripts/ci/create_rpm_repo.sh create mode 100755 scripts/ci/prepare-rpm-repo.sh create mode 100755 scripts/packaging/build-rpm-local.sh diff --git a/rpm-deps-build.Dockerfile b/rpm-deps-build.Dockerfile new file mode 100644 index 000000000000..fb9040fb405c --- /dev/null +++ b/rpm-deps-build.Dockerfile @@ -0,0 +1,72 @@ +ARG IMAGE +# the image with proper version is set as ARG +#hadolint ignore=DL3006 +FROM ${IMAGE} + +ENV TZ=Etc/UTC +# Build blst used by ocaml-bls12-381 without ADX to support old CPU +# architectures. +# See https://gitlab.com/tezos/tezos/-/issues/1788 and +# https://gitlab.com/dannywillems/ocaml-bls12-381/-/merge_requests/135/ +ENV BLST_PORTABLE=true +ENV VENV_PATH=$HOME/venv + +#hadolint ignore=DL3041 +RUN dnf -y update &&\ + dnf install -y python3-pip rpmdevtools &&\ + dnf clean all &&\ + python3 -m venv "$VENV_PATH" &&\ + "$VENV_PATH/bin/pip" install -U pip setuptools &&\ + "$VENV_PATH/bin/pip" install poetry==1.8.3 + +WORKDIR /root/tezos +COPY ./scripts/version.sh ./scripts/version.sh +COPY ./scripts/pkg-common/install_opam.sh ./scripts/pkg-common/install_opam.sh +COPY scripts/ci/bin_packages_rpm_dependencies.sh \ + ./scripts/ci/bin_packages_rpm_dependencies.sh +RUN scripts/ci/bin_packages_rpm_dependencies.sh + +# we trust sw distributors +# We install sccache as a static binary because at the moment of writing +# the package sccache is not available on ubuntu jammy +#hadolint ignore=DL3008,DL3009 +RUN ARCH=$(uname -m) && \ + curl -L --output sccache.tgz "https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-$ARCH-unknown-linux-musl.tar.gz" && \ + tar zxvf sccache.tgz && \ + cp "sccache-v0.8.1-$ARCH-unknown-linux-musl/sccache" /usr/local/bin/sccache && \ + rm -Rf sccache* + +#hadolint ignore=SC2154 +RUN . ./scripts/version.sh && \ + curl -s https://sh.rustup.rs > rustup-init.sh && \ + chmod +x rustup-init.sh && \ + ./rustup-init.sh --profile minimal \ + --default-toolchain "$recommended_rust_version" -y + +RUN opam init --bare --disable-sandboxing + +# we do not need everything to run build-deps +# we copy the mininum amount of files to use +# the caching mechanism more efficiently +COPY --link scripts/install_build_deps.sh /root/tezos/scripts/ +COPY --link scripts/install_build_deps.rust.sh /root/tezos/scripts/ +COPY --link scripts/install_dal_trusted_setup.sh /root/tezos/scripts/ +COPY --link scripts/version.sh /root/tezos/scripts/ +COPY --link Makefile /root/tezos/ +COPY --link opam/virtual/octez-deps.opam.locked /root/tezos/opam/virtual/ +COPY --link opam /root/tezos/ + +WORKDIR /root/tezos + +# we download and copy the zcash params in a separate directory +# and before the make build-deps to create a second docker layer and +# optimize caching +RUN DAL_TRUSTED_SETUP="/root/tezos/dal-trusted-setup" \ + scripts/install_dal_trusted_setup.sh + +#hadolint ignore=SC2154, SC1091 +RUN . ./scripts/version.sh && \ + eval $(opam env) && \ + . "/root/.cargo/env" && \ + make build-deps && \ + mv dal-trusted-setup _opam/share/dal-trusted-setup diff --git a/scripts/ci/build-rpm-packages.sh b/scripts/ci/build-rpm-packages.sh new file mode 100755 index 000000000000..a075020d8c1f --- /dev/null +++ b/scripts/ci/build-rpm-packages.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# + +src_dir="$(pwd)" + +. scripts/version.sh +. scripts/ci/octez-release.sh + +BUILDDIR=$(pwd) +export BLST_PORTABLE=true +export PATH="/venv/bin/:$PATH" + +# fetch tags for releases +git fetch -q --tags + +# Prepare the building area: copying all files from +# the dependency image a staging area. This is necessary +# to build on arm64 where the BUILDDIR is in ram. +cp -a ./* /root/tezos/ +cp -a ./.git /root/tezos/ +cd /root/tezos/ || exit 1 + +. scripts/ci/octez-packages-version.sh + +case "$RELEASETYPE" in +ReleaseCandidate | TestReleaseCandidate | Release | TestRelease) + _VERSION=$VERSION + _CHANGELOG="New Release $VERSION / $CI_COMMIT_SHORT_SHA" + ;; +Master) + _VERSION=1:$(date +'%Y%m%d%H%M')+$CI_COMMIT_SHORT_SHA + _CHANGELOG="Packages for master $CI_COMMIT_SHORT_SHA" + ;; +SoftRelease) + _VERSION=1:$(date +'%Y%m%d%H%M')+${CI_COMMIT_TAG:-} + _CHANGELOG="Packages for tag ${CI_COMMIT_TAG:-}" + ;; +TestBranch) + _VERSION=1:$(date +'%Y%m%d%H%M')+$CI_COMMIT_SHORT_SHA + _CHANGELOG="Test package commit ${CI_COMMIT_REF_NAME:-}" + ;; +*) + echo "Cannot create package for this branch" + exit 1 + ;; +esac + +scripts/packaging/build-rpm-local.sh "$1" + +# Move the debian package to be packed as artifacts +mkdir -p "$BUILDDIR/packages/$DISTRIBUTION/$RELEASE" +cp -va "$HOME"/rpmbuild/RPMS/x86_64/*.rpm \ + "$BUILDDIR/packages/$DISTRIBUTION/$RELEASE" diff --git a/scripts/ci/create_rpm_repo.sh b/scripts/ci/create_rpm_repo.sh new file mode 100755 index 000000000000..6e3e65257eaa --- /dev/null +++ b/scripts/ci/create_rpm_repo.sh @@ -0,0 +1,169 @@ +#!/bin/sh + +set -eu + +# Create the APT repository for rockylinux packages and sign it using +# the private key available as ENV variable for production repository +# and using the file test_repo_private.key for test repositories. + +# uses : +# - scripts/packaging/Release.conf for release metadata +# - scripts/packaging/key.asc as the repository pub key + +# expected env vars +# - ARCHITECTURES +# - GPG_PASSPHRASE +# - GPG_KEY_ID +# - GPG_PRIVATE_KEY +# - GCP_LINUX_PACKAGES_BUCKET + +if [ $# -lt 2 ]; then + cat << EOF +Usage: $0 + +: The linux distribution, eg. rockylinux or ubuntu + +: The release of the Linux distribution, e.g. 'jammy', 'noble', '9.3'. +This argument can be repeated to build for multiple releases. + +Set the ARCHITECTURES env variable of packages built for +multiple architectures are available. Eg 'amd64 arm64' +EOF + exit 1 +fi + +ARCHITECTURES=${ARCHITECTURES:-"amd64"} + +#The prefix used for these packages in the repository. E.g. 'next' +PREFIX=${PREFIX:-""} + +# The linux distribution for which we are creating the rpm repository +# E.g. 'ubuntu' or 'rockylinux' +DISTRIBUTION=${1} +shift +# The release of the linux distribution for which +# we are creating the rpm repository +# E.g. 'jammy noble', '9.3' +RELEASES=$* + +# If it's a protected branch the value of $bucket will +# be set accordingly but the CI. +BUCKET="$GCP_LINUX_PACKAGES_BUCKET" + +oldPWD=$PWD + +. scripts/ci/octez-packages-version.sh + +case "$RELEASETYPE" in +ReleaseCandidate | TestReleaseCandidate) + TARGETDIR="public/$PREFIX/RC/$DISTRIBUTION" + ;; +Release | TestRelease) + TARGETDIR="public/$PREFIX/$DISTRIBUTION" + ;; +Master) + TARGETDIR="public/$PREFIX/master/$DISTRIBUTION" + ;; +SoftRelease) + TARGETDIR="public/$PREFIX/${CI_COMMIT_TAG}/$DISTRIBUTION" + ;; +TestBranch) + TARGETDIR="public/$PREFIX/$CI_COMMIT_REF_NAME/$DISTRIBUTION" + ;; +*) + echo "Cannot create a repository for this branch" + exit 1 + ;; +esac + +if [ "$CI_PROJECT_NAMESPACE" = "tezos" ] && [ "$CI_COMMIT_REF_PROTECTED" = "true" ]; then + # the keys used for official releases only + # These env vars are only available on tezos/tezos + # and in protected branches + # GPG_KEY_ID="5DC80C4ED0B7C4FE" + GPG_PRIVATE_KEY="$GPG_LINUX_PACKAGES_PRIVATE_KEY" + GPG_PASSPHRASE="$GPG_LINUX_PACKAGES_PASSPHRASE" + echo "$GPG_LINUX_PACKAGES_PUBLIC_KEY" > \ + ./scripts/packaging/package-signing-key-release.asc + GPG_PUBLIC_KEY="./scripts/packaging/package-signing-key-release.asc" +else + # This is strictly for testing + # We embed these keys here for testing only. + GPG_KEY_ID="CFC482F3CD08D36D" + GPG_PASSPHRASE="07cde771b39a4ed394864baa46126b" + GPG_PRIVATE_KEY=$(cat ./scripts/packaging/test_repo_private.key) + GPG_PUBLIC_KEY="./scripts/packaging/package-signing-key.asc" +fi + +export GPG_TTY=/dev/console +echo "$GPG_PRIVATE_KEY" | base64 --decode | gpg --batch --import -- +GPG_EMAIL=$(gpg --list-keys --with-colons "$GPG_KEY_ID" | grep uid | cut -d':' -f10) +rpm -v --import "$GPG_PUBLIC_KEY" + +mkdir -p "$TARGETDIR/dists" + +# Copying files +for release in $RELEASES; do + for architecture in $ARCHITECTURES; do # amd64, arm64 ... + echo "Setting up RPM repository for $DISTRIBUTION / $release / $architecture" + echo "targetdir: $TARGETDIR" + + # create the rpm repository root directory and copy the public key + cp "$GPG_PUBLIC_KEY" "$TARGETDIR/octez.asc" + + target="dists/${release}" + + mkdir -p "$TARGETDIR/${target}/" + + for file in packages/"${DISTRIBUTION}/${release}"/*.rpm; do + rpm --query --qf '%{NAME}-%{VERSION}-%{RELEASE}.rpm: %{SIGMD5}\n' "$file" + printf "Sha256: %s\n" "$(sha256sum "$file")\n" + + echo "$GPG_PASSPHRASE" | + gpg --batch --passphrase-fd 0 --pinentry-mode loopback \ + -u "$GPG_KEY_ID" --detach-sign --armor \ + "$file" || { + echo "Error signing $file" + exit 1 + } + rpm --define "_gpg_name $GPG_EMAIL" --resign "$file" || { + echo "Error resigning $file" + exit 1 + } + + rpm --query --qf '%{NAME}-%{VERSION}-%{RELEASE}.rpm: %{SIGMD5}\n' "$file" + printf "Sha256: %s\n" "$(sha256sum "$file")\n" + + rpm --checksig "$file" || { + echo "Signature check failed for $file" + exit 1 + } + cp "$file" "$TARGETDIR/${target}/" + echo "Signing and adding package $file to $TARGETDIR/${target}/" + echo "---------------------" + done + + echo "Create the Repository $TARGETDIR/${target}" + createrepo_c -v --update --retain-old-md 1 "$TARGETDIR/${target}" + + done + +done + +cd "$oldPWD" + +if [ "$CI_COMMIT_REF_PROTECTED" = "true" ]; then + echo "### Logging into protected repo ..." + echo "${GCP_PROTECTED_SERVICE_ACCOUNT}" | base64 -d > protected_sa.json + gcloud auth activate-service-account --key-file=protected_sa.json +else + echo "### Logging into standard repo ..." + # Nothing to do +fi + +GOOGLE_OAUTH_ACCESS_TOKEN=$(gcloud auth print-access-token) +export GOOGLE_OAUTH_ACCESS_TOKEN + +echo "Push to $BUCKET" + +gsutil -m cp -r public/* gs://"${BUCKET}" diff --git a/scripts/ci/prepare-rpm-repo.sh b/scripts/ci/prepare-rpm-repo.sh new file mode 100755 index 000000000000..94d75933675b --- /dev/null +++ b/scripts/ci/prepare-rpm-repo.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Install depedendencies for the apt_repo job + +dnf update + +tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM +[google-cloud-cli] +name=Google Cloud CLI +baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el9-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOM + +dnf install -y libxcrypt-compat.x86_64 google-cloud-cli \ + createrepo_c rpm-sign rpmdevtools diff --git a/scripts/packaging/build-rpm-local.sh b/scripts/packaging/build-rpm-local.sh new file mode 100755 index 000000000000..2085fe51fd2f --- /dev/null +++ b/scripts/packaging/build-rpm-local.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +set -eu + +script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" +src_dir="$(dirname "$(dirname "$script_dir")")" + +BUILD_DIR="$HOME/rpmbuild" +SOURCES_DIR="$BUILD_DIR/SOURCES" +SPECS_DIR="$BUILD_DIR/SPECS" +RPMS_DIR="$BUILD_DIR/RPMS" +BINARIES="$SPECS_DIR/binaries" + +# build spec files from debian control +cd "$src_dir/scripts/packaging/deb-to-rpm" || exit 1 +poetry lock && poetry install +cd - + +# shellcheck disable=SC1091 +. "$HOME/.cargo/env" +eval "$(opam env)" + +packages() { + # Build tezos as usual + make octez + + rm -f "${SPECS_DIR:?}/*.spec" + cd "$src_dir/scripts/packaging/deb-to-rpm" || exit 1 + poetry run ./deb-to-rpm.py \ + -d "$src_dir/scripts/packaging/octez/debian/" \ + -s "$SPECS_DIR" + cd - || exit 1 + + # Create RPM build directories + mkdir -p "$SOURCES_DIR" "$SPECS_DIR" "$RPMS_DIR" "$BINARIES" + + rm -f "${BINARIES:?}/*" + EXECUTABLES=$(cat script-inputs/*-executables) + for ex in $EXECUTABLES; do + if [ -f "$ex" ]; then + cp -f "$ex" "$BINARIES" + fi + done + + cd "$src_dir/scripts/packaging/octez" || exit 1 + cp -a manpages/ scripts/ "$SPECS_DIR/" + cd - || exit 1 +} + +zcash() { + + rm -Rf "${SPECS_DIR:?}/*.spec" + cd "$src_dir/scripts/packaging/deb-to-rpm" || exit 1 + poetry run ./deb-to-rpm.py \ + -d "$src_dir/scripts/packaging/octez-data/debian/" \ + -s "$SPECS_DIR" + cd - || exit 1 + + cd "$src_dir/scripts/packaging/octez-data" || exit 1 + cp -a "$OPAM_SWITCH_PREFIX/share/zcash-params/" \ + "$OPAM_SWITCH_PREFIX/share/dal-trusted-setup/" "$SPECS_DIR/" + cd - || exit 1 +} + +usage() { + echo "Usage: $0 (binaries|zcash) [--dev]" + exit 2 +} + +TARGET=all + +if [ -z "${CI:-}" ]; then + echo "Warning: You are compiling the debian packages locally." + echo + echo " This script should be only used for development." + echo " The version of the debian packages is set to be '0.0.1-1'" + echo " The version of the octez binaries depends on the git branch / tag" + echo +fi + +while [ $# -gt 0 ]; do + case ${1} in + "binaries") + TARGET=binaries + shift + ;; + "zcash") + TARGET=zcash + shift + ;; + "help") + usage + ;; + *) + echo "Unknown command or option: $1" + usage + ;; + esac +done + +case ${TARGET} in +"binaries") + echo "Building binary packages only" + packages + ;; +"zcash") + echo "Building data packages only" + zcash + ;; +"all") + echo "Building binary and data packages" + packages + zcash + ;; +esac + +for SPEC_FILE in "$SPECS_DIR"/*.spec; do + + # Build the RPM package + cd "$SPECS_DIR" || exit + rpmbuild -ba \ + --define '_source_filedir %{nil}' \ + --define "_binary_payload w2T16.xzdio" \ + "$SPEC_FILE" + +done + +cd "$src_dir" || exit +# Show the resulting RPM package +echo "Built RPM packages are located in $RPMS_DIR" +ls "$RPMS_DIR/x86_64/" -- GitLab From 8fa6ad822d05e6d5249abdff469c0924267abfe7 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Wed, 23 Oct 2024 15:31:34 +0200 Subject: [PATCH 3/4] packages: add CI pipelines for rpm packages --- .gitlab/ci/pipelines/before_merging.yml | 9 + .gitlab/ci/pipelines/rpm_repository_full.yml | 283 +++++++++++++++++ .../ci/pipelines/rpm_repository_partial.yml | 146 +++++++++ .../pipelines/rpm_repository_partial_auto.yml | 146 +++++++++ ci/bin/code_verification.ml | 15 +- ci/bin/common.ml | 2 + ci/bin/rpm_repository.ml | 300 ++++++++++++++++++ 7 files changed, 900 insertions(+), 1 deletion(-) create mode 100644 .gitlab/ci/pipelines/rpm_repository_full.yml create mode 100644 .gitlab/ci/pipelines/rpm_repository_partial.yml create mode 100644 .gitlab/ci/pipelines/rpm_repository_partial_auto.yml create mode 100644 ci/bin/rpm_repository.ml diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 0933db6b8bda..159717f5b824 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4570,6 +4570,15 @@ documentation:linkcheck: - make -C docs redirectcheck - make -C docs linkcheck +trigger:rpm_repository_partial: + stage: manual + rules: + - when: manual + allow_failure: true + needs: [] + trigger: + include: .gitlab/ci/pipelines/rpm_repository_partial.yml + trigger:debian_repository_partial: stage: manual rules: diff --git a/.gitlab/ci/pipelines/rpm_repository_full.yml b/.gitlab/ci/pipelines/rpm_repository_full.yml new file mode 100644 index 000000000000..c552b0d337ab --- /dev/null +++ b/.gitlab/ci/pipelines/rpm_repository_full.yml @@ -0,0 +1,283 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +workflow: + rules: + - if: $foo != "bar" || $foo == "bar" + when: always + +variables: + PIPELINE_TYPE: rpm_repository_full + +stages: +- images +- build +- publishing + +oc.docker-build-rockylinux-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - $TAGS + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + - gcp_arm64 + +oc.build-rockylinux: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh binaries + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + - gcp_arm64 + +oc.build-rockylinux-data: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh zcash + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + - gcp_arm64 + +apt_repo_rockylinux: + image: rockylinux:9.3 + stage: publishing + tags: + - gcp + needs: + - oc.build-rockylinux + - oc.build-rockylinux-data + dependencies: + - oc.build-rockylinux + - oc.build-rockylinux-data + timeout: 60 minutes + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-rpm-repo.sh + script: + - ./scripts/ci/create_rpm_repo.sh rockylinux 9.3 + variables: + ARCHITECTURES: amd64 arm64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + +oc.docker-build-fedora-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - $TAGS + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: fedora + parallel: + matrix: + - RELEASE: + - "93" + TAGS: + - gcp + - gcp_arm64 + +oc.build-fedora: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-fedora-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh binaries + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: fedora + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "93" + TAGS: + - gcp + - gcp_arm64 + +oc.build-fedora-data: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-fedora-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh zcash + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: fedora + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "93" + TAGS: + - gcp + - gcp_arm64 + +apt_repo_fedora: + image: fedora:39 + stage: publishing + tags: + - gcp + needs: + - oc.build-fedora + - oc.build-fedora-data + dependencies: + - oc.build-fedora + - oc.build-fedora-data + timeout: 60 minutes + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-rpm-repo.sh + script: + - ./scripts/ci/create_rpm_repo.sh fedora 93 + variables: + ARCHITECTURES: amd64 arm64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/.gitlab/ci/pipelines/rpm_repository_partial.yml b/.gitlab/ci/pipelines/rpm_repository_partial.yml new file mode 100644 index 000000000000..f5bf1d0ce192 --- /dev/null +++ b/.gitlab/ci/pipelines/rpm_repository_partial.yml @@ -0,0 +1,146 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +workflow: + rules: + - if: $foo != "bar" || $foo == "bar" + when: always + +variables: + PIPELINE_TYPE: rpm_repository_partial + +stages: +- images +- build +- publishing + +oc.docker-build-rockylinux-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - $TAGS + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +oc.build-rockylinux: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh binaries + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +oc.build-rockylinux-data: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh zcash + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +apt_repo_rockylinux: + image: rockylinux:9.3 + stage: publishing + tags: + - gcp + needs: + - oc.build-rockylinux + - oc.build-rockylinux-data + dependencies: + - oc.build-rockylinux + - oc.build-rockylinux-data + timeout: 60 minutes + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-rpm-repo.sh + script: + - ./scripts/ci/create_rpm_repo.sh rockylinux 9.3 + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml b/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml new file mode 100644 index 000000000000..6ed9f11d90fa --- /dev/null +++ b/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml @@ -0,0 +1,146 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +workflow: + rules: + - if: $foo != "bar" || $foo == "bar" + when: always + +variables: + PIPELINE_TYPE: rpm_repository_partial_auto + +stages: +- images +- build +- publishing + +oc.docker-build-rockylinux-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - $TAGS + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +oc.build-rockylinux: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh binaries + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +oc.build-rockylinux-data: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + stage: build + tags: + - $TAGS + needs: + - oc.docker-build-rockylinux-dependencies + dependencies: [] + timeout: 60 minutes + cache: + key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - . ./scripts/ci/sccache-start.sh + script: + - export CARGO_NET_OFFLINE=false + - ./scripts/ci/build-rpm-packages.sh zcash + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: rockylinux + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + SCCACHE_IDLE_TIMEOUT: "0" + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + retry: + max: 1 + when: + - stuck_or_timeout_failure + parallel: + matrix: + - RELEASE: + - "9.3" + TAGS: + - gcp + +apt_repo_rockylinux: + image: rockylinux:9.3 + stage: publishing + tags: + - gcp + needs: + - oc.build-rockylinux + - oc.build-rockylinux-data + dependencies: + - oc.build-rockylinux + - oc.build-rockylinux-data + timeout: 60 minutes + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-rpm-repo.sh + script: + - ./scripts/ci/create_rpm_repo.sh rockylinux 9.3 + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index ff1e6e3e10ea..6d78691241b0 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1736,6 +1736,17 @@ let jobs pipeline_type = ~stage:Stages.manual Debian_repository.child_pipeline_partial in + let job_rpm_repository_trigger_partial : tezos_job = + (* Same as [job_rpm_repository_trigger_auto] but manual, + so that one can trigger it without triggering the whole main pipeline. + See comment near the definition of [job_rpm_repository_trigger_auto]. *) + trigger_job + ~__POS__ + ~rules:(make_rules ~manual:Yes ()) + ~dependencies:(Dependent []) + ~stage:Stages.manual + Rpm_repository.child_pipeline_partial + in match pipeline_type with | Before_merging | Merge_train -> (* Note: manual jobs in stage [manual] (which is the final @@ -1814,7 +1825,9 @@ let jobs pipeline_type = @ [job_docker_verify_test_arm64; job_docker_verify_test_amd64] in if pipeline_type = Merge_train then jobs - else job_debian_repository_trigger_partial :: jobs + else + job_rpm_repository_trigger_partial + :: job_debian_repository_trigger_partial :: jobs (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] in diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 268f73d458be..65438b7ddeca 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -120,6 +120,8 @@ module Images_external = struct let fedora_39 = Image.mk_external ~image_path:"fedora:39" + let rockylinux_93 = Image.mk_external ~image_path:"rockylinux:9.3" + let opam_ubuntu_jammy = Image.mk_external ~image_path:"ocaml/opam:ubuntu-22.04" diff --git a/ci/bin/rpm_repository.ml b/ci/bin/rpm_repository.ml new file mode 100644 index 000000000000..6278f080525f --- /dev/null +++ b/ci/bin/rpm_repository.ml @@ -0,0 +1,300 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +(* This module defines the jobs of the [rockylinux_repository] child + pipeline. + + This pipeline builds the current and next Debian (and Ubuntu) + packages. *) + +open Gitlab_ci.Types +open Gitlab_ci.Util +open Tezos_ci +open Common + +let build_rockylinux_packages_image = + Image.mk_external + ~image_path:"$DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + +(** These are the set of Debian release-architecture combinations for + which we build deb packages in the job + [job_build_rockylinux_package]. A dependency image will be built once + for each combination of [RELEASE] and [TAGS]. + + If [release_pipeline] is false, we only tests a subset of the matrix, + one release, and one architecture. *) +let rockylinux_package_release_matrix = function + | Partial -> [[("RELEASE", ["9.3"]); ("TAGS", ["gcp"])]] + | Full | Release -> [[("RELEASE", ["9.3"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + +(** These are the set of Ubuntu release-architecture combinations for + which we build deb packages in the job + [job_build_fedora_package]. See {!rockylinux_package_release_matrix} + for more information. + + If [release_pipeline] is false, we only tests a subset of the matrix, + one release, and one architecture. *) +let fedora_package_release_matrix = function + | Partial -> [[("RELEASE", ["93"]); ("TAGS", ["gcp"])]] + | Full | Release -> [[("RELEASE", ["93"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + +let archs_variables pipeline = + let amd64 = List.map Tezos_ci.arch_to_string_alt [Amd64] in + let all = List.map Tezos_ci.arch_to_string_alt [Amd64; Arm64] in + match pipeline with + | Partial -> [("ARCHITECTURES", String.concat " " amd64)] + | Full | Release -> [("ARCHITECTURES", String.concat " " all)] + +(* Push .deb artifacts to storagecloud apt repository. *) +let make_job_apt_repo ?rules ~__POS__ ~name ?(stage = Stages.publishing) + ?(prefix = false) ?dependencies ~variables ~image script : tezos_job = + let variables = + variables + @ [("GNUPGHOME", "$CI_PROJECT_DIR/.gnupg")] + @ if prefix then [("PREFIX", "next")] else [] + in + job + ?rules + ?dependencies + ~__POS__ + ~stage + ~name + ~image + ~before_script: + (before_script ~source_version:true ["./scripts/ci/prepare-rpm-repo.sh"]) + ~variables + script + +(* The entire Debian packages pipeline. When [pipeline_type] is [Before_merging] + we test only on Debian stable. Returns a triplet, the first element is + the list of all jobs, the second is the job building fedora packages artifats + and the third rockylinux packages artifacts *) +let jobs pipeline_type = + let variables add = + ( "DEP_IMAGE", + "${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/build-$DISTRIBUTION-$RELEASE" + ) + (* this second variable is for a read only registry and we want it to be + tezos/tezos *) + :: ( "DEP_IMAGE_PROTECTED", + "${GCP_PROTECTED_REGISTRY}/tezos/tezos/build-$DISTRIBUTION-$RELEASE" ) + :: add + in + let make_job_docker_build_rockylinux_dependencies ~__POS__ ~name ~matrix + ~distribution = + job_docker_authenticated + ~__POS__ + ~name + ~stage:Stages.images + ~variables:(variables [("DISTRIBUTION", distribution)]) + ~parallel:(Matrix matrix) + ~tag:Dynamic + [ + "./scripts/ci/build-debian-packages-dependencies.sh \ + rpm-deps-build.Dockerfile"; + ] + in + let job_docker_build_rockylinux_dependencies : tezos_job = + make_job_docker_build_rockylinux_dependencies + ~__POS__ + ~name:"oc.docker-build-rockylinux-dependencies" + ~distribution:"rockylinux" + ~matrix:(rockylinux_package_release_matrix pipeline_type) + in + let job_docker_build_fedora_dependencies : tezos_job = + make_job_docker_build_rockylinux_dependencies + ~__POS__ + ~name:"oc.docker-build-fedora-dependencies" + ~distribution:"fedora" + ~matrix:(fedora_package_release_matrix pipeline_type) + in + let make_job_build_rockylinux_packages ~__POS__ ~name ~matrix ~distribution + ~script ~dependencies = + job + ~__POS__ + ~name + ~image:build_rockylinux_packages_image + ~stage:Stages.build + ~variables:(variables [("DISTRIBUTION", distribution)]) + ~parallel:(Matrix matrix) + ~dependencies + ~tag:Dynamic + ~retry:Gitlab_ci.Types.{max = 1; when_ = [Stuck_or_timeout_failure]} + ~artifacts:(artifacts ["packages/$DISTRIBUTION/$RELEASE"]) + [ + (* This is an hack to enable Cargo networking for jobs in child pipelines. + + There is an weird gotcha with how variables are passed to + child pipelines. Global variables of the parent pipeline + are passed to the child pipeline. Inside the child + pipeline, variables received from the parent pipeline take + precedence over job-level variables. It's bit strange. So + to override the default [CARGO_NET_OFFLINE=true], we cannot + just set it in the job-level variables of this job. + + [enable_sccache] adds the cache directive for [$CI_PROJECT_DIR/_sccache]. + + See + {{:https://docs.gitlab.com/ee/ci/variables/index.html#cicd-variable-precedence}here} + for more info. *) + "export CARGO_NET_OFFLINE=false"; + script; + ] + |> enable_sccache ~idle_timeout:"0" + in + + (* These jobs build the next packages in a matrix using the + build dependencies images *) + let job_build_rockylinux_package : tezos_job = + make_job_build_rockylinux_packages + ~__POS__ + ~name:"oc.build-rockylinux" + ~distribution:"rockylinux" + ~dependencies:(Dependent [Job job_docker_build_rockylinux_dependencies]) + ~script:"./scripts/ci/build-rpm-packages.sh binaries" + ~matrix:(rockylinux_package_release_matrix pipeline_type) + in + let job_build_fedora_package : tezos_job = + make_job_build_rockylinux_packages + ~__POS__ + ~name:"oc.build-fedora" + ~distribution:"fedora" + ~dependencies:(Dependent [Job job_docker_build_fedora_dependencies]) + ~script:"./scripts/ci/build-rpm-packages.sh binaries" + ~matrix:(fedora_package_release_matrix pipeline_type) + in + let job_build_rockylinux_package_data : tezos_job = + make_job_build_rockylinux_packages + ~__POS__ + ~name:"oc.build-rockylinux-data" + ~distribution:"rockylinux" + ~dependencies:(Dependent [Job job_docker_build_rockylinux_dependencies]) + ~script:"./scripts/ci/build-rpm-packages.sh zcash" + ~matrix:(rockylinux_package_release_matrix pipeline_type) + in + let job_build_fedora_package_data : tezos_job = + make_job_build_rockylinux_packages + ~__POS__ + ~name:"oc.build-fedora-data" + ~distribution:"fedora" + ~dependencies:(Dependent [Job job_docker_build_fedora_dependencies]) + ~script:"./scripts/ci/build-rpm-packages.sh zcash" + ~matrix:(fedora_package_release_matrix pipeline_type) + in + + (* These jobs create the apt repository for the next packages *) + let job_apt_repo_rockylinux = + make_job_apt_repo + ~__POS__ + ~name:"apt_repo_rockylinux" + ~prefix:true + ~dependencies: + (Dependent + [ + Artifacts job_build_rockylinux_package; + Artifacts job_build_rockylinux_package_data; + ]) + ~variables:(archs_variables pipeline_type) + ~image:Images.rockylinux_93 + ["./scripts/ci/create_rpm_repo.sh rockylinux 9.3"] + in + let job_apt_repo_fedora = + make_job_apt_repo + ~__POS__ + ~name:"apt_repo_fedora" + ~prefix:true + ~dependencies: + (Dependent + [ + Artifacts job_build_fedora_package; + Artifacts job_build_fedora_package_data; + ]) + ~variables:(archs_variables pipeline_type) + ~image:Images.fedora_39 + ["./scripts/ci/create_rpm_repo.sh fedora 93"] + in + (* These test the installability of the current packages *) + let _job_install_bin ~__POS__ ~name ~dependencies ~image ?(variables = []) + ?allow_failure script = + job + ?allow_failure + ~__POS__ + ~name + ~image + ~dependencies + ~variables + ~stage:Stages.publishing_tests + script + in + (* These test the upgrade of the current packages *) + let _job_upgrade_bin ~__POS__ ~name ~dependencies ~image ?allow_failure script + = + job + ?allow_failure + ~__POS__ + ~name + ~image + ~dependencies + ~stage:Stages.publishing_tests + script + in + let rockylinux_jobs = + [ + job_docker_build_rockylinux_dependencies; + job_build_rockylinux_package; + job_build_rockylinux_package_data; + job_apt_repo_rockylinux; + ] + in + let fedora_jobs = + [ + job_docker_build_fedora_dependencies; + job_build_fedora_package; + job_build_fedora_package_data; + job_apt_repo_fedora; + ] + in + match pipeline_type with + | Partial -> rockylinux_jobs + | Full -> rockylinux_jobs @ fedora_jobs + | Release -> rockylinux_jobs @ fedora_jobs + +let register ~auto ~description pipeline_type = + let pipeline_name = + match (pipeline_type, auto) with + | Partial, false -> "rpm_repository_partial" + | Partial, true -> "rpm_repository_partial_auto" + | Full, _ -> "rpm_repository_full" + | Release, _ -> "rpm_repository_release" + in + let jobs = jobs pipeline_type in + Pipeline.register_child pipeline_name ~description ~jobs + +let child_pipeline_partial = + register + ~description: + "A child pipeline of 'before_merging' (and thus 'merge_train') building \ + Debian stable .deb packages." + ~auto:false + Partial + +let child_pipeline_full = + register + ~description: + "A child pipeline of 'schedule_extended_test' testing the build of all \ + .deb packages." + ~auto:false + Full + +let child_pipeline_partial_auto = + register + ~description: + "A child pipeline of 'before_merging' (and thus 'merge_train') building \ + Rockylinux 9.3 .deb packages. Starts automatically on certain \ + conditions." + ~auto:true + Partial -- GitLab From 8d2e306d3c2f9f0f5a34083c4a902715c9e3807b Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Thu, 24 Oct 2024 15:58:13 +0200 Subject: [PATCH 4/4] XX --- .gitlab/ci/pipelines/rpm_repository_full.yml | 4 +- .../ci/pipelines/rpm_repository_partial.yml | 2 +- .../pipelines/rpm_repository_partial_auto.yml | 2 +- ci/bin/rpm_repository.ml | 5 +- scripts/packaging/build-rpm-local.sh | 11 ++- scripts/packaging/deb-to-rpm/deb-to-rpm.py | 71 ++++++++++++++++--- scripts/packaging/deb-to-rpm/poetry.lock | 12 ++-- scripts/packaging/octez/rpm/octez-node.post | 25 +++++++ 8 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 scripts/packaging/octez/rpm/octez-node.post diff --git a/.gitlab/ci/pipelines/rpm_repository_full.yml b/.gitlab/ci/pipelines/rpm_repository_full.yml index c552b0d337ab..7ca33eb4678b 100644 --- a/.gitlab/ci/pipelines/rpm_repository_full.yml +++ b/.gitlab/ci/pipelines/rpm_repository_full.yml @@ -24,7 +24,7 @@ oc.docker-build-rockylinux-dependencies: before_script: - ./scripts/ci/docker_initialize.sh script: - - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + - ./scripts/ci/build-packages-dependencies.sh rpm-deps-build.Dockerfile services: - docker:${DOCKER_VERSION}-dind variables: @@ -158,7 +158,7 @@ oc.docker-build-fedora-dependencies: before_script: - ./scripts/ci/docker_initialize.sh script: - - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + - ./scripts/ci/build-packages-dependencies.sh rpm-deps-build.Dockerfile services: - docker:${DOCKER_VERSION}-dind variables: diff --git a/.gitlab/ci/pipelines/rpm_repository_partial.yml b/.gitlab/ci/pipelines/rpm_repository_partial.yml index f5bf1d0ce192..405063c2500f 100644 --- a/.gitlab/ci/pipelines/rpm_repository_partial.yml +++ b/.gitlab/ci/pipelines/rpm_repository_partial.yml @@ -24,7 +24,7 @@ oc.docker-build-rockylinux-dependencies: before_script: - ./scripts/ci/docker_initialize.sh script: - - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + - ./scripts/ci/build-packages-dependencies.sh rpm-deps-build.Dockerfile services: - docker:${DOCKER_VERSION}-dind variables: diff --git a/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml b/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml index 6ed9f11d90fa..00bafc874ac7 100644 --- a/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml +++ b/.gitlab/ci/pipelines/rpm_repository_partial_auto.yml @@ -24,7 +24,7 @@ oc.docker-build-rockylinux-dependencies: before_script: - ./scripts/ci/docker_initialize.sh script: - - ./scripts/ci/build-debian-packages-dependencies.sh rpm-deps-build.Dockerfile + - ./scripts/ci/build-packages-dependencies.sh rpm-deps-build.Dockerfile services: - docker:${DOCKER_VERSION}-dind variables: diff --git a/ci/bin/rpm_repository.ml b/ci/bin/rpm_repository.ml index 6278f080525f..e3c19e8f0639 100644 --- a/ci/bin/rpm_repository.ml +++ b/ci/bin/rpm_repository.ml @@ -93,10 +93,7 @@ let jobs pipeline_type = ~variables:(variables [("DISTRIBUTION", distribution)]) ~parallel:(Matrix matrix) ~tag:Dynamic - [ - "./scripts/ci/build-debian-packages-dependencies.sh \ - rpm-deps-build.Dockerfile"; - ] + ["./scripts/ci/build-packages-dependencies.sh rpm-deps-build.Dockerfile"] in let job_docker_build_rockylinux_dependencies : tezos_job = make_job_docker_build_rockylinux_dependencies diff --git a/scripts/packaging/build-rpm-local.sh b/scripts/packaging/build-rpm-local.sh index 2085fe51fd2f..beab833802d1 100755 --- a/scripts/packaging/build-rpm-local.sh +++ b/scripts/packaging/build-rpm-local.sh @@ -28,6 +28,7 @@ packages() { cd "$src_dir/scripts/packaging/deb-to-rpm" || exit 1 poetry run ./deb-to-rpm.py \ -d "$src_dir/scripts/packaging/octez/debian/" \ + -r "$src_dir/scripts/packaging/octez/rpm/" \ -s "$SPECS_DIR" cd - || exit 1 @@ -44,6 +45,13 @@ packages() { cd "$src_dir/scripts/packaging/octez" || exit 1 cp -a manpages/ scripts/ "$SPECS_DIR/" + "$SPECS_DIR/"binaries/octez-node --help=groff > \ + "$SPECS_DIR/"manpages/octez-node.1 + "$SPECS_DIR/"binaries/octez-dal-node --help=groff > \ + "$SPECS_DIR/"manpages/octez-dal-node.1 + cp debian/*.service "$SPECS_DIR/" + cp debian/*.default "$SPECS_DIR/" + cp debian/*.manpages "$SPECS_DIR/" cd - || exit 1 } @@ -53,6 +61,7 @@ zcash() { cd "$src_dir/scripts/packaging/deb-to-rpm" || exit 1 poetry run ./deb-to-rpm.py \ -d "$src_dir/scripts/packaging/octez-data/debian/" \ + -r "$src_dir/scripts/packaging/octez/rpm/" \ -s "$SPECS_DIR" cd - || exit 1 @@ -70,7 +79,7 @@ usage() { TARGET=all if [ -z "${CI:-}" ]; then - echo "Warning: You are compiling the debian packages locally." + echo "Warning: You are compiling the rpm packages locally." echo echo " This script should be only used for development." echo " The version of the debian packages is set to be '0.0.1-1'" diff --git a/scripts/packaging/deb-to-rpm/deb-to-rpm.py b/scripts/packaging/deb-to-rpm/deb-to-rpm.py index f9e370af76cd..7be2b0f3f77a 100755 --- a/scripts/packaging/deb-to-rpm/deb-to-rpm.py +++ b/scripts/packaging/deb-to-rpm/deb-to-rpm.py @@ -6,7 +6,8 @@ from datetime import datetime from debian import deb822 from specfile import Specfile import argparse - +import glob +import shutil def generate_spec_file(file_path): content = """Name: test @@ -23,6 +24,9 @@ Enhances: a %description %install %files +%postun +%post +%preun """ # Create and write the content to the spec file @@ -111,7 +115,7 @@ def convert_deps(deps): # Return the cleaned and mapped dependencies as a string return " ".join(rpm_deps) -def process_debian_control(debian_dir, spec_dir): +def process_debian_control(debian_dir, rpm_dir, spec_dir): """Process the Debian control file and generate .spec files.""" control_file = os.path.join(debian_dir, 'control') @@ -196,6 +200,9 @@ def process_debian_control(debian_dir, spec_dir): sections.description = description files_section = [] install_section = [] + post_section = [] + preun_section = [] + postun_section = [] install_file = os.path.join( debian_dir, f"{package_name}.install" @@ -212,28 +219,67 @@ def process_debian_control(debian_dir, spec_dir): name = os.path.basename(src) # Add the installation commands to the %install section - install_section.append( - f"mkdir -p %{{buildroot}}{dest}") + install_section.append(f"mkdir -p %{{buildroot}}{dest}") install_section.append( - f"install -m 0755 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}{dest}") + f"install -D -m 0755 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}{dest}") # Add the destination to the %files section files_section.append(f"{dest}{name}") + for post_file in glob.glob(os.path.join(rpm_dir, f"{package_name}.post")): + if os.path.exists(post_file): + with open(post_file, 'r') as post_f: + post_section.append(post_f.read()) + # Handle .manpage files if they exist manpage_file = os.path.join( - debian_dir, f"{package_name}.manpage" + debian_dir, f"{package_name}.manpages" ) if os.path.exists(manpage_file): with open(manpage_file, 'r') as man_f: for man_line in man_f: - files_section.append(f"{man_line.strip()}") + src = man_line.strip() + name = os.path.basename(src) + install_section.append(f"install -D -m 644 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}%{{_mandir}}/man1/{name}") + install_section.append(f"gzip %{{buildroot}}%{{_mandir}}/man1/{name}") + files_section.append(f"%{{_mandir}}/man1/{name}.gz") + + # Handle .service files if they exist + for service_file in glob.glob(os.path.join(debian_dir, f"{package_name}*.service")): + if os.path.exists(service_file): + src = os.path.basename(service_file) + s = src.split(".") + if len(s) > 2: + service_name = f"{s[1]}.{s[2]}" + else: + service_name = f"{s[0]}.{s[1]}" + install_section.append( + f"install -D -m 644 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}/usr/lib/systemd/system/{service_name}") + post_section.append(f"%systemd_post {service_name}") + preun_section.append(f"%systemd_preun {service_name}") + postun_section.append(f"%systemd_postun_with_restart {service_name}") + files_section.append(f"/usr/lib/systemd/system/{service_name}") + + # Handle .default files if they exist + for default_file in glob.glob(os.path.join(debian_dir, f"{package_name}.*.default")): + if os.path.exists(default_file): + src = os.path.basename(default_file) + s = src.split(".") + if len(s) > 2: + default_name = f"{s[1]}.{s[2]}" + else: + default_name = f"{s[0]}.{s[1]}" + install_section.append( + f"install -D -m 644 $HOME/rpmbuild/SPECS/{src} %{{buildroot}}/etc/default/{default_name}") + files_section.append(f"/etc/default/{default_name}") # Add the install and files sections to the spec file sections.install = install_section sections.files = files_section - print(sections.install) + sections.post = post_section + sections.preun = preun_section + sections.postun = postun_section # Save the generated .spec file spec.save() @@ -252,6 +298,12 @@ def parse_args(): required=True, help="Directory containing Debian control, .install, and .manpage files.", ) + parser.add_argument( + '-r', + '--rpm-dir', + required=True, + help="Directory containing RPM specific post scripts", + ) parser.add_argument( '-s', '--spec-dir', @@ -266,13 +318,14 @@ def main(): args = parse_args() debian_dir = args.debian_dir + rpm_dir = args.rpm_dir spec_dir = args.spec_dir if not os.path.isdir(debian_dir): print(f"Error: '{debian_dir}' is not a valid directory.") sys.exit(1) - process_debian_control(debian_dir, spec_dir) + process_debian_control(debian_dir, rpm_dir, spec_dir) if __name__ == "__main__": diff --git a/scripts/packaging/deb-to-rpm/poetry.lock b/scripts/packaging/deb-to-rpm/poetry.lock index 3c6699ee843d..35b067323c0a 100644 --- a/scripts/packaging/deb-to-rpm/poetry.lock +++ b/scripts/packaging/deb-to-rpm/poetry.lock @@ -38,13 +38,13 @@ chardet = "*" [[package]] name = "rpm" -version = "0.2.0" +version = "0.3.0" description = "Shim RPM module for use in virtualenvs." optional = false python-versions = ">=3.6" files = [ - {file = "rpm-0.2.0-py3-none-any.whl", hash = "sha256:4050b6033f7403be0a34f42a742c49ba74f2b0c6129f0247115b6078b24ddd71"}, - {file = "rpm-0.2.0.tar.gz", hash = "sha256:b92285f65c9ddf77678cb3e51aa67827426408fac34cdd8d537d8c14e3eaffbf"}, + {file = "rpm-0.3.0-py3-none-any.whl", hash = "sha256:4f93082693fc82079d8ad080aebad1ec8aed1667c4f2081b3d5f2cd28dfb5e2f"}, + {file = "rpm-0.3.0.tar.gz", hash = "sha256:2c5415abb7d8a15428cb97a6f429c107c0619bfd4d0140c034af46e7c8d53b2e"}, ] [package.extras] @@ -52,13 +52,13 @@ testing = ["tox"] [[package]] name = "specfile" -version = "0.32.4" +version = "0.32.5" description = "A library for parsing and manipulating RPM spec files." optional = false python-versions = ">=3.6" files = [ - {file = "specfile-0.32.4-py3-none-any.whl", hash = "sha256:f8f893f5b857c9e259d55cb0d539dcbdd9bb8088c1cc0cbdc5a43538157b8452"}, - {file = "specfile-0.32.4.tar.gz", hash = "sha256:a0bb5ed0981aa22e89162c4ac82c1c0a5209f0395bb50ab4140c05bae02bfc20"}, + {file = "specfile-0.32.5-py3-none-any.whl", hash = "sha256:fdd02b6d205343e96041f5bec46336a982717bd4a90dba98ed4bac28c532eb72"}, + {file = "specfile-0.32.5.tar.gz", hash = "sha256:84d99782af4861ae83e0b5599b92a6f6a48958dafc64f5e5afeed994aa638493"}, ] [package.dependencies] diff --git a/scripts/packaging/octez/rpm/octez-node.post b/scripts/packaging/octez/rpm/octez-node.post new file mode 100644 index 000000000000..8c8a40d0058d --- /dev/null +++ b/scripts/packaging/octez/rpm/octez-node.post @@ -0,0 +1,25 @@ +TEZOS_HOME=/var/tezos + +# Work directory to store temporary files associated with this package +mkdir -p /run/octez-node + +DATADIR="$TEZOS_HOME/.tezos-node" + +# Check if the tezos user exists, if not, create it +if ! id "tezos" > /dev/null 2>&1; then + useradd --system --create-home \ + --home-dir "$TEZOS_HOME" --shell /bin/bash \ + --comment "admin user for octez" tezos +else + # Setup data directory in case the tezos user already exists + if [ ! -d "$TEZOS_HOME" ]; then + mkdir -p "$TEZOS_HOME" + chown tezos "$TEZOS_HOME" + fi +fi + +# Setup log directory +if [ ! -d /var/log/tezos ]; then + mkdir -p /var/log/tezos + chown tezos /var/log/tezos +fi -- GitLab