diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e92a8bb5770e37ebb5b8c05209157c6cf6a07140..3b073b59855ba29bf39fc4d239304752cf632e3b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,12 +6,19 @@ default: paths: - .apt before_script: + ## test docker-in-docker + - apt-get update -y + - apt-get install -y apt-transport-https ca-certificates curl software-properties-common + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + - add-apt-repository "deb [arch=armhf] https://download.docker.com/linux/ubuntu bionic stable" + - apt-get update -y + - apt-get install -y docker-ce + - docker info ## ## apt caching ## - rm -f /etc/apt/apt.conf.d/docker-clean - mkdir .apt && mkdir /var/cache/apt/archives && mount --bind .apt /var/cache/apt/archives/ - - apt-get update -y - apt-get install --no-install-recommends -y $(cat packages) variables: @@ -22,6 +29,10 @@ variables: IMG_DATE: date PROJECT_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_PATH} IS_GITLAB_CI_TRIGGERED: "yes" + DOCKER_TLS_CERTDIR: "" + +services: + - docker:dind workflow: rules: @@ -30,39 +41,21 @@ workflow: - when: always stages: - ## builds the early parts from pi-gen - - build-early-stages - ## builds the parts that are new from pier - - build-pier-stages + ## builds the image and stores it in the **must be mounted** directory /images + - build ## uploads the final images to GitLab - deploy -stages-012: - stage: build-early-stages - script: - - ./build.sh - - tar -czf ${PROJECT_PATH}/work/date-pier/work.tar.gz -C ${PROJECT_PATH}/work/date-pier/ stage2 +pierhost-build: + stage: build variables: - STAGE_LIST: "stage0 stage1 stage2" - rules: - - when: always - artifacts: - paths: - - work/date-pier/work.tar.gz - expire_in: 1 day - - -stages-67: - stage: build-pier-stages - variables: - STAGE_LIST: "stage6-core stage7-addons" + STAGE_LIST: "stage0 stage1 stage2 stage6-core stage7-addons" script: - - mkdir -p work/date-pier/ - - tar -C work/date-pier/ -xzf work/date-pier/work.tar.gz - - PREV_ROOTFS_DIR="${PROJECT_PATH}/work/date-pier/stage2/rootfs" ./build.sh + - ./build.sh + - ./store_build.sh -r -b "$CI_COMMIT_BRANCH" -c "$CI_COMMIT_SHORT_SHA" -s "deploy" -t "/images" artifacts: paths: - - deploy/ + - deploy/storage_location deploy: stage: deploy @@ -71,7 +64,7 @@ deploy: default: false script: - VERSION=$(cat version) - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file deploy/image_date-pier-with-services.zip ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/pier_images/${VERSION}/pierhost_image.zip' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file "$(cat "deploy/storage_location")/image_date-pier-with-services.zip" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/pier_images/${VERSION}/pierhost_image.zip' rules: - if: '$CI_COMMIT_BRANCH == "master"' when: always \ No newline at end of file diff --git a/stage6-core/04-docker-compose-essentials/files/docker-services-startup.service b/stage6-core/04-docker-compose-essentials/files/docker-services-startup.service index 69c8910a7e78cc3f25170445248bf2c0a85907e3..a0238a305cfea7e22150cd1bd9b05166869df7d8 100644 --- a/stage6-core/04-docker-compose-essentials/files/docker-services-startup.service +++ b/stage6-core/04-docker-compose-essentials/files/docker-services-startup.service @@ -1,6 +1,7 @@ [Unit] Description=Launches Dockerized Services -After=network-online.target local-fs.target websocketd.service hotspot.service +Wants=import_prefetch_images.service +After=network-online.target local-fs.target websocketd.service hotspot.service import_prefetch_images.service [Service] ExecStart=REPLACE_WITH_SCRIPTNAME diff --git a/stage7-addons/01-prepull-images/00-run.sh b/stage7-addons/01-prepull-images/00-run.sh new file mode 100755 index 0000000000000000000000000000000000000000..9f4cd2f1d23e539bca93d9ca181db270d28e204a --- /dev/null +++ b/stage7-addons/01-prepull-images/00-run.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e + +install -m 755 files/get_digest.py "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/get_digest.py" +install -m 755 files/import.sh "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/import.sh" +install -m 644 files/import_prefetch_images.service "${ROOTFS_DIR}/etc/systemd/system/import_prefetch_images.service" + +on_chroot << EOF + systemctl enable import_prefetch_images +EOF + +sed -i "s|REPLACE_WITH_SERVICES_DIR|/home/${FIRST_USER_NAME}/Services|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/import.sh" +sed -i "s|REPLACE_WITH_SERVICES_DIR|/home/${FIRST_USER_NAME}/Services|g" "${ROOTFS_DIR}/etc/systemd/system/import_prefetch_images.service" + +pushd ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services +mkdir prefetch +for image in $(cat pier-services/images) +do + digest=$(DOCKER_CLI_EXPERIMENTAL=enabled python3 get_digest.py $image) + docker pull $digest + # filename conversion + name=$(echo $image | sed 's-/-#-g') + docker save $digest | gzip > "prefetch/${name}.tar.gz" +done +popd diff --git a/stage7-addons/01-prepull-images/files/get_digest.py b/stage7-addons/01-prepull-images/files/get_digest.py new file mode 100644 index 0000000000000000000000000000000000000000..fefbc67b0fd56f8d20e4cedbb2b5c8e30372d053 --- /dev/null +++ b/stage7-addons/01-prepull-images/files/get_digest.py @@ -0,0 +1,40 @@ +import subprocess +import sys +import json + +assert(len(sys.argv) == 2) + +image = str(sys.argv[1]) + +result = subprocess.run(['docker', 'manifest', 'inspect', image], stdout=subprocess.PIPE) +string = result.stdout.decode('utf-8') +manifest = json.loads(string) +if manifest["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json": + # not multiarch + # we can simply pull the image + print(image) +elif manifest["mediaType"] == "application/vnd.docker.distribution.manifest.list.v2+json": + # multiarch + digest = "" + for part in manifest["manifests"]: + platform = part["platform"] + if platform["os"] == "linux" and platform["architecture"] == "arm": + if "variant" not in platform or platform["variant"] == "v7": + # correct version found + digest = part["digest"] + print(image + "@" + digest) + if digest == "": + # no arm v7 or generic arm found, search for v6 or v5 + for part in manifest["manifests"]: + platform = part["platform"] + if platform["os"] == "linux" and platform["architecture"] == "arm": + if platform["variant"] == "v6" or platform["variant"] == "v5": + # usable version found + digest = part["digest"] + print(image + "@" + digest) + if digest == "": + print("couldnt find correct manifest") + exit(2) +else: + print("couldnt determine image type") + exit(1) \ No newline at end of file diff --git a/stage7-addons/01-prepull-images/files/import.sh b/stage7-addons/01-prepull-images/files/import.sh new file mode 100644 index 0000000000000000000000000000000000000000..10d50e6a6473f0026f4ef774be5ad647c7ad71d2 --- /dev/null +++ b/stage7-addons/01-prepull-images/files/import.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +pushd REPLACE_WITH_SERVICES_DIR/prefetch +for file in * +do + docker load < "$file" > load_output + + # undo filename conversion + noext=$(basename "$file" .tar.gz) + tag=$(echo "$noext" | sed 's-#-/-g') + + # get image ID + fullsha=$(cat load_output | sed 's/^Loaded image ID: sha256://') + imageid=${fullsha:0:12} + + docker tag "$imageid" "$tag" +done +touch "done" +popd + +systemctl disable import_prefetch_images \ No newline at end of file diff --git a/stage7-addons/01-prepull-images/files/import_prefetch_images.service b/stage7-addons/01-prepull-images/files/import_prefetch_images.service new file mode 100644 index 0000000000000000000000000000000000000000..0fe917fab543af90c9a27ddaa2f60afcbb54ee00 --- /dev/null +++ b/stage7-addons/01-prepull-images/files/import_prefetch_images.service @@ -0,0 +1,12 @@ +[Unit] +Description=Loads the prefetched images for Pierhost services +Requires=docker.service +After=docker.service +ConditionPathExists=!REPLACE_WITH_SERVICES_DIR/prefetch/done + +[Service] +ExecStart=REPLACE_WITH_SERVICES_DIR/import.sh +Type=oneshot + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/store_build.sh b/store_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..486efe9856a2c42ea1885af42670ea5aab52f4c2 --- /dev/null +++ b/store_build.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +print_opts(){ + echo "" +} + +while getopts ":b:c:s:t:rh" opt; do + case $opt in + b) + branch=$OPTARG + ;; + c) + commit=$OPTARG + ;; + s) + source=$OPTARG + ;; + t) + target=$OPTARG + ;; + r) + report="yes" + ;; + h) + print_opts + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_opts + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_opts + exit 1 + ;; + esac +done + +if [ ! -v branch ] +then + echo "No branch specified, assuming 'nobranch'" + branch="nobranch" +fi + +if [ ! -v commit ] +then + echo "No commit specified, using current time" + commit=$(date +"%S") +fi + +if [ ! -v source ] || [ ! -d "$source" ] +then + echo "No source directory specified, or specified source is not an existing directory" >&2 + exit 2 +fi + +if [ ! -v target ] || [ ! -d "$target" ] +then + echo "No target directory specified, or specified target is not an existing directory" >&2 + exit 2 +fi + +branch=$(echo "$branch" | tr '\\:/<>\"|?*' "_") +commit=$(echo "$commit" | tr '\\:/<>\"|?*' "_") +timestamp=$(date +"%y-%m-%d_%H-%M") + +echo "Storing contents of ${source} in ${target}/${branch}/${timestamp}_${commit}" + +if [ -d "${target}/${branch}/${timestamp}_${commit}" ] +then + echo "Destination already existed, deleting" + rm -r "${target}/${branch}/${timestamp}_${commit}" +fi + +mkdir -p "${target}/${branch}" +cp -r "${source}" "${target}/${branch}/${timestamp}_${commit}" + +if [ $(ls -l "${target}/${branch}/" | grep -c ^d) -gt 3 ] +then + echo "More than 3 images in this branch, cleaning up" + pushd "${target}/${branch}/" > /dev/null + find * -maxdepth 0 | sort -r | tail -n +4 | xargs rm -r + popd > /dev/null +fi + +if [ -v report ] +then + echo "${target}/${branch}/${timestamp}_${commit}" > "${source}/storage_location" +fi