From d78be830625d8587f929a5f54f00ec4f986ebb88 Mon Sep 17 00:00:00 2001 From: Jon Doveston Date: Wed, 5 Feb 2025 15:01:05 +0000 Subject: [PATCH 1/3] Rename and tidy up some CI scripts --- .gitlab-ci.yml | 10 +++++----- scripts/ci/autodevops.sh | 0 .../ci/{feature_spec_setup => feature_spec_setup.sh} | 0 ..._spec_dependencies => install_spec_dependencies.sh} | 0 ...ntegration_spec_setup => integration_spec_setup.sh} | 0 scripts/ci/qa.sh | 0 scripts/ci/{run_specs => run_specs.sh} | 4 ++-- 7 files changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 scripts/ci/autodevops.sh rename scripts/ci/{feature_spec_setup => feature_spec_setup.sh} (100%) rename scripts/ci/{install_spec_dependencies => install_spec_dependencies.sh} (100%) rename scripts/ci/{integration_spec_setup => integration_spec_setup.sh} (100%) mode change 100644 => 100755 scripts/ci/qa.sh rename scripts/ci/{run_specs => run_specs.sh} (87%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b57b715cdb..0c303ff92a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,8 +135,8 @@ tag_auto_deploy: - if: '$PIPELINE_TYPE == "AUTO_DEPLOY_TRIGGER_PIPELINE"' .source_autodevops: - before_script: - - source scripts/ci/autodevops.sh + before_script: + - source scripts/ci/autodevops.sh update-trigger-branch: stage: prepare @@ -391,7 +391,7 @@ trigger-eks130: # --------------------------------------------------------------------------- -.specs: &specs +.specs: image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-20.12-postgresql-${PG_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.45-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36 stage: specs services: @@ -407,9 +407,9 @@ trigger-eks130: CHROME_VERSION: "123" extends: .source_autodevops script: - - ./scripts/ci/install_spec_dependencies + - ./scripts/ci/install_spec_dependencies.sh - set_context - - ./scripts/ci/run_specs + - ./scripts/ci/run_specs.sh artifacts: when: on_failure expire_in: 7d diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh old mode 100644 new mode 100755 diff --git a/scripts/ci/feature_spec_setup b/scripts/ci/feature_spec_setup.sh similarity index 100% rename from scripts/ci/feature_spec_setup rename to scripts/ci/feature_spec_setup.sh diff --git a/scripts/ci/install_spec_dependencies b/scripts/ci/install_spec_dependencies.sh similarity index 100% rename from scripts/ci/install_spec_dependencies rename to scripts/ci/install_spec_dependencies.sh diff --git a/scripts/ci/integration_spec_setup b/scripts/ci/integration_spec_setup.sh similarity index 100% rename from scripts/ci/integration_spec_setup rename to scripts/ci/integration_spec_setup.sh diff --git a/scripts/ci/qa.sh b/scripts/ci/qa.sh old mode 100644 new mode 100755 diff --git a/scripts/ci/run_specs b/scripts/ci/run_specs.sh similarity index 87% rename from scripts/ci/run_specs rename to scripts/ci/run_specs.sh index a1822a2312..f3c057f46c 100755 --- a/scripts/ci/run_specs +++ b/scripts/ci/run_specs.sh @@ -3,9 +3,9 @@ set -e if [[ -n "${VARIABLES_FILE}" ]]; then source "${VARIABLES_FILE}" - ./scripts/ci/feature_spec_setup + ./scripts/ci/feature_spec_setup.sh else - ./scripts/ci/integration_spec_setup + ./scripts/ci/integration_spec_setup.sh fi bundle config set --local path 'gems' -- GitLab From 3724424d6d3f4553af83102b9a13c2da0e70b77d Mon Sep 17 00:00:00 2001 From: Jon Doveston Date: Wed, 5 Feb 2025 15:17:23 +0000 Subject: [PATCH 2/3] Upgrade vcluster --- .gitlab-ci.yml | 1 + .gitlab/ci/review-apps.gitlab-ci.yml | 9 +++++---- scripts/ci/vcluster.sh | 21 ++++++++++++++++++--- scripts/ci/vcluster.template.yaml | 21 +++++++++++++++++++++ 4 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 scripts/ci/vcluster.template.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c303ff92a..a81a73993e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: DOCKER_VERSION: "27.1.1" HELM_VERSION: "3.10.3" KUBECTL_VERSION: "1.28.3" + VCLUSTER_VERSION: "0.22.3" STABLE_REPO_URL: "https://charts.helm.sh/stable" GOOGLE_APPLICATION_CREDENTIALS: ${CI_PROJECT_DIR}/.google_keyfile.json # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index cad1676c19..5a181d7189 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -1,12 +1,13 @@ .review_app_common: stage: review variables: - AGENT_NAME: "gke129-ci-cluster" # connect to 1.29 cluster until we have a dedicated cluster + AGENT_NAME: "gkevc-ci-cluster" environment: - name: gke129_vcluster/${VCLUSTER_NAME} + name: gkevc_review/${VCLUSTER_NAME} auto_stop_in: 1 hour before_script: - source scripts/ci/vcluster.sh + - vcluster_install allow_failure: true rules: - !reference [.rule:skip_if_no_cluster] @@ -46,7 +47,7 @@ review_vcluster_128: extends: .review_app_template variables: - VCLUSTER_K8S_VERSION: "1.28" + VCLUSTER_K8S_VERSION: "v1.28.0" VCLUSTER_NAME: vcluster-1-28-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA} environment: on_stop: stop_review_vcluster_128 @@ -60,7 +61,7 @@ stop_review_vcluster_128: review_vcluster_131: extends: .review_app_template variables: - VCLUSTER_K8S_VERSION: "1.31" + VCLUSTER_K8S_VERSION: "v1.31.0" VCLUSTER_NAME: vcluster-1-31-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA} environment: on_stop: stop_review_vcluster_131 diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index ef61484154..1f01b361b2 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -9,18 +9,33 @@ function cluster_connect() { fi } +function vcluster_install() { + if [ -z "${VCLUSTER_VERSION}" ] || [ "${VCLUSTER_VERSION,,}" == "default" ]; then + echo "No version specified, using default image version" + else + echo "Install vcluster version ${VCLUSTER_VERSION}" + curl -Lo /tmp/vcluster "https://github.com/loft-sh/vcluster/releases/download/v${VCLUSTER_VERSION}/vcluster-linux-amd64" + install -c -m 0755 /tmp/vcluster /usr/local/bin + fi + vcluster version +} + function vcluster_name() { printf ${VCLUSTER_NAME:0:52} } function vcluster_create() { + envsubst '$VCLUSTER_K8S_VERSION' < ./scripts/ci/vcluster.template.yaml > ./vcluster.yaml + cat vcluster.yaml + local vcluster_name=$(vcluster_name) vcluster create ${vcluster_name} \ --upgrade \ --namespace=${vcluster_name} \ - --kubernetes-version=${VCLUSTER_K8S_VERSION} \ --connect=false \ - --update-current=false + --values ./vcluster.yaml + + kubectl annotate namespace ${vcluster_name} janitor/ttl=2d } function vcluster_run() { @@ -44,7 +59,7 @@ function vcluster_helm_rollout_status() { } function vcluster_delete() { - vcluster delete $(vcluster_name) + vcluster delete $(vcluster_name) --delete-configmap --delete-namespace --ignore-not-found } function vcluster_info() { diff --git a/scripts/ci/vcluster.template.yaml b/scripts/ci/vcluster.template.yaml new file mode 100644 index 0000000000..d009b44aee --- /dev/null +++ b/scripts/ci/vcluster.template.yaml @@ -0,0 +1,21 @@ +controlPlane: + backingStore: + etcd: + deploy: + enabled: true + distro: + k8s: + enabled: true + version: "$VCLUSTER_K8S_VERSION" + statefulSet: + scheduling: + podManagementPolicy: OrderedReady +sync: + toHost: + ingresses: + enabled: true +integrations: + metricsServer: + enabled: true + nodes: true + pods: true -- GitLab From 8dff86178ffb43ac067223bb0b5b3becb68a2ecb Mon Sep 17 00:00:00 2001 From: Jon Doveston Date: Wed, 5 Feb 2025 15:29:35 +0000 Subject: [PATCH 3/3] Add vcluster QA jobs --- .gitlab-ci.yml | 13 ++ .../ci/environments/gkevc131.gitlab-ci.yml | 75 +++++++++++ .gitlab/ci/vcluster-review-apps.gitlab-ci.yml | 116 ++++++++++++++++++ scripts/ci/autodevops.sh | 20 ++- scripts/ci/vcluster.sh | 6 + scripts/ci/vcluster_deploy.sh | 28 +++++ 6 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 .gitlab/ci/environments/gkevc131.gitlab-ci.yml create mode 100644 .gitlab/ci/vcluster-review-apps.gitlab-ci.yml create mode 100755 scripts/ci/vcluster_deploy.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a81a73993e..b1e5f651b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,6 +92,10 @@ include: - local: '/.gitlab/ci/review-apps.gitlab-ci.yml' rules: - if: '$LIMIT_TO == "vcluster"' + - local: '/.gitlab/ci/vcluster-review-apps.gitlab-ci.yml' + - local: '/.gitlab/ci/environments/gkevc131.gitlab-ci.yml' + rules: + - if: '$LIMIT_TO == "gkevc131"' - local: '/.gitlab/ci/operator.gitlab-ci.yml' rules: - if: '$LIMIT_TO == "" || $LIMIT_TO == null' @@ -330,6 +334,15 @@ trigger-gke130a: rules: - !reference [.trigger_review_current, rules] +trigger-gkevc131: + extends: .trigger_review_job + needs: + - trigger_review_secondary + variables: + LIMIT_TO: gkevc131 + rules: + - !reference [.trigger_review_secondary, rules] + trigger-vcluster: extends: .trigger_review_job needs: diff --git a/.gitlab/ci/environments/gkevc131.gitlab-ci.yml b/.gitlab/ci/environments/gkevc131.gitlab-ci.yml new file mode 100644 index 0000000000..924afdb4a3 --- /dev/null +++ b/.gitlab/ci/environments/gkevc131.gitlab-ci.yml @@ -0,0 +1,75 @@ +.review_gkevc131_common_vars: &review_gkevc131_common_vars + VCLUSTER_K8S_VERSION: "v1.31.0" + VCLUSTER_NAME: "vc131-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}" + HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}-vc131" + # the VARIABLES_FILE value is from the CI_JOB_NAME of the review job + VARIABLES_FILE: "variables/review_gkevc131" + DOMAIN: "-${HOST_SUFFIX}.${KUBE_INGRESS_BASE_DOMAIN}" + +create_review_gkevc131: + stage: review + image: busybox + variables: + <<: *review_gkevc131_common_vars + GIT_STRATEGY: none + script: + - echo "Configuring gkevc_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN." + environment: &gkevc131_environment + name: &gkevc131_name gkevc_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA + url: "https://gitlab${DOMAIN}" + on_stop: stop_review_gkevc131 + auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN + action: start + +review_gkevc131: + variables: + <<: *review_gkevc131_common_vars + extends: .vcluster_review_template + environment: + <<: *gkevc131_environment + action: access + +stop_review_gkevc131: + variables: + <<: *review_gkevc131_common_vars + extends: .stop_vcluster_review_template + environment: + name: *gkevc131_name + action: stop + +review_specs_gkevc131: + extends: .vcluster_specs_template + variables: + <<: *review_gkevc131_common_vars + environment: + <<: *gkevc131_environment + action: access + needs: ['review_gkevc131'] + +qa_gkevc131: + extends: .qa_branch + variables: + <<: *review_gkevc131_common_vars + TEST_SUITE: $QA_SANITY_SUITE_OPTIONS + environment: + <<: *gkevc131_environment + action: access + needs: + - review_gkevc131 + - job: review_specs_gkevc131 + artifacts: false + +qa_gkevc131_full_suite_manual_trigger: + extends: .qa_full_suite_manual_trigger + script: echo 'The job will trigger Full E2E suite against GKE131 vcluster' + +qa_gkevc131_manual_full_suite: + extends: .qa_branch + parallel: 7 + variables: + <<: *review_gkevc131_common_vars + TEST_SUITE: $QA_FULL_SUITE_OPTIONS + environment: + <<: *gkevc131_environment + action: access + needs: ["qa_gkevc131_full_suite_manual_trigger"] diff --git a/.gitlab/ci/vcluster-review-apps.gitlab-ci.yml b/.gitlab/ci/vcluster-review-apps.gitlab-ci.yml new file mode 100644 index 0000000000..362c050d8b --- /dev/null +++ b/.gitlab/ci/vcluster-review-apps.gitlab-ci.yml @@ -0,0 +1,116 @@ +.vcluster_review_common_vars: &vcluster_review_common_vars + DNS_PROVIDER: "google" + AGENT_NAME: "gkevc-ci-cluster" + KUBE_INGRESS_BASE_DOMAIN: "cloud-native-vcluster.helm-charts.win" + KUBE_NAMESPACE: "default" + NAMESPACE: "${KUBE_NAMESPACE}" + HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" + VARIABLES_FILE: "variables/${CI_JOB_NAME}" + +.vcluster_review_template: + stage: review + retry: 1 + variables: + <<: *vcluster_review_common_vars + VARIABLES_FILE: "variables/${CI_JOB_NAME}" + ADD_NGINX_DNS_ANNOTATIONS: "yes" + needs: + - pipeline: $PARENT_PIPELINE_ID + job: pin_image_versions + artifacts: + when: always + paths: + - variables + allow_failure: true + before_script: + - source scripts/ci/autodevops.sh + - source scripts/ci/vcluster.sh + - vcluster_install + script: + # Set up varfile + - mkdir -p $(dirname "${VARIABLES_FILE}") + # Connect to host cluster + - set_context + - check_kube_domain + - kubectl version --output=json + # Create vcluster + - vcluster_create + - vcluster_run kubectl version --output=json + - vcluster_info + # Copy the TLS cert secret to vcluster + - vcluster_copy_secret ci-stack helm-charts-win-tls ${NAMESPACE} + # Deploy to vcluster + - vcluster_run ./scripts/ci/vcluster_deploy.sh + rules: + - !reference [.rule:skip_if_no_cluster] + - if: '$PIPELINE_TYPE =~ /DEFAULT_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /STABLE_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE == "QA_KNAPSACK_UPDATE_PIPELINE"' + +.stop_vcluster_review_template: + stage: review + variables: + <<: *vcluster_review_common_vars + GIT_CHECKOUT: "false" + allow_failure: true + before_script: + # If the branch has already been deleted, fall back to master. + - git checkout "${CI_COMMIT_REF_NAME}" || git checkout master + - source scripts/ci/autodevops.sh + - source scripts/ci/vcluster.sh + - vcluster_install + script: + # Connect to host cluster + - set_context + # Delete vcluster from host + - vcluster_delete + when: manual + allow_failure: true + rules: + - !reference [.review_template, rules] + +.vcluster_specs_template: + image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-20.12-postgresql-${PG_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.45-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36 + stage: specs + services: + - docker:${DOCKER_VERSION}-dind + variables: + <<: *vcluster_review_common_vars + DOCKER_HOST: tcp://docker:2375 + GITLAB_PASSWORD: ${ROOT_PASSWORD} + S3_CONFIG_PATH: /etc/gitlab/minio + RUBYGEMS_VERSION: "3.5" + GO_VERSION: "1.21" + RUST_VERSION: "1.73" + PG_VERSION: "14" + CHROME_VERSION: "123" + RSPEC_TAGS: type:feature + allow_failure: true + before_script: + - source scripts/ci/autodevops.sh + - source scripts/ci/vcluster.sh + - vcluster_install + script: + - ./scripts/ci/install_spec_dependencies.sh + - set_context + - vcluster_run kubectl version --output=json + - vcluster_run ./scripts/ci/run_specs.sh + artifacts: + when: on_failure + expire_in: 7d + paths: + - tmp/capybara + cache: + key: "debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-${CI_JOB_NAME}" + paths: + - gems + rules: + - if: '$PIPELINE_TYPE =~ /DEFAULT_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /STABLE_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + - if: '$PIPELINE_TYPE == "QA_KNAPSACK_UPDATE_PIPELINE"' + - !reference [.rule:skip_if_no_cluster] + - !reference [.specs, rules] diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh index 1ca18c2849..73607ddffd 100755 --- a/scripts/ci/autodevops.sh +++ b/scripts/ci/autodevops.sh @@ -64,10 +64,10 @@ function deploy() { #ROOT_PASSWORD=$(cat /dev/urandom | LC_TYPE=C tr -dc "[:alpha:]" | head -c 16) #echo "Generated root login: $ROOT_PASSWORD" - kubectl create secret generic "${RELEASE_NAME}-gitlab-initial-root-password" --from-literal=password=$ROOT_PASSWORD -o yaml --dry-run=client | kubectl replace --force -f - + kubectl create secret generic -n "${NAMESPACE}" "${RELEASE_NAME}-gitlab-initial-root-password" --from-literal=password=$ROOT_PASSWORD -o yaml --dry-run=client | kubectl replace --force -f - echo "${QA_EE_LICENSE}" > /tmp/license.gitlab - kubectl create secret generic "${RELEASE_NAME}-gitlab-license" --from-file=license=/tmp/license.gitlab -o yaml --dry-run=client | kubectl replace --force -f - + kubectl create secret generic -n "${NAMESPACE}" "${RELEASE_NAME}-gitlab-license" --from-file=license=/tmp/license.gitlab -o yaml --dry-run=client | kubectl replace --force -f - # YAML_FILE=""${KUBE_INGRESS_BASE_DOMAIN//\./-}.yaml" @@ -145,6 +145,21 @@ CIYAML cpu: 100m CIYAML + if [ -n "${ADD_NGINX_DNS_ANNOTATIONS}" ]; then + echo "ADD_NGINX_DNS_ANNOTATIONS detected" + DOMAIN="-$HOST_SUFFIX.$KUBE_INGRESS_BASE_DOMAIN" + # configure nginx and external-dns + cat << CIYAML > ci.nginx.yaml + nginx-ingress: + controller: + service: + annotations: + external-dns.alpha.kubernetes.io/ttl: "10" + external-dns.alpha.kubernetes.io/hostname: "kas${DOMAIN},minio${DOMAIN},registry${DOMAIN},gitlab${DOMAIN}" +CIYAML + NGINX_CONFIGURATION="-f ci.nginx.yaml" + fi + # PostgreSQL max_connection defaults to 100, which is apparently not enough to pass QA. cat << CIYAML > ci.psql.yaml postgresql: @@ -174,6 +189,7 @@ CIYAML $WAIT \ ${SENTRY_CONFIGURATION} \ ${MULTIARCH_CONFIGURATION} \ + ${NGINX_CONFIGURATION} \ -f ci.details.yaml \ -f ci.scale.yaml \ -f ci.psql.yaml \ diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 1f01b361b2..34e23ed85a 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -38,6 +38,12 @@ function vcluster_create() { kubectl annotate namespace ${vcluster_name} janitor/ttl=2d } +function vcluster_copy_secret() { + kubectl get secret -n $1 $2 -o yaml \ + | sed '/^ namespace: /d; /^ uid: /d; /^ resourceVersion: /d; /^ creationTimestamp: /d; /^ selfLink: /d; /^status:$/Q;' \ + | vcluster connect $(vcluster_name) -- kubectl apply -n $3 -f - +} + function vcluster_run() { vcluster connect $(vcluster_name) -- $@ } diff --git a/scripts/ci/vcluster_deploy.sh b/scripts/ci/vcluster_deploy.sh new file mode 100755 index 0000000000..31cec8c1d3 --- /dev/null +++ b/scripts/ci/vcluster_deploy.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +source scripts/ci/autodevops.sh + +echo "CI_ENVIRONMENT_SLUG=${CI_ENVIRONMENT_SLUG}" +echo "CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}" +echo "HOST_SUFFIX=${HOST_SUFFIX}" +echo "KUBE_INGRESS_BASE_DOMAIN=${KUBE_INGRESS_BASE_DOMAIN}" +echo "DOMAIN=${DOMAIN}" +echo "VARIABLES_FILE=${VARIABLES_FILE}" +echo "KUBE_NAMESPACE=${KUBE_NAMESPACE}" +echo "NAMESPACE=${NAMESPACE}" +echo "RELEASE_NAME=${RELEASE_NAME}" +echo "VCLUSTER_NAME=${VCLUSTER_NAME}" +echo "VCLUSTER_K8S_VERSION=${VCLUSTER_K8S_VERSION}" + +create_secret +deploy +wait_for_deploy +check_domain_ip + +# Generate variables file +echo "export GITLAB_URL=gitlab-${HOST_SUFFIX}.${KUBE_INGRESS_BASE_DOMAIN}" >> "${VARIABLES_FILE}" +echo "export GITLAB_ROOT_DOMAIN=${HOST_SUFFIX}.${KUBE_INGRESS_BASE_DOMAIN}" >> "${VARIABLES_FILE}" +echo "export REGISTRY_URL=registry-${HOST_SUFFIX}.${KUBE_INGRESS_BASE_DOMAIN}" >> "${VARIABLES_FILE}" +echo "export S3_ENDPOINT=https://minio-${HOST_SUFFIX}.${KUBE_INGRESS_BASE_DOMAIN}" >> "${VARIABLES_FILE}" +kubectl wait pods -n ${NAMESPACE} -l app=toolbox,release=${RELEASE_NAME} --for condition=Ready --timeout=60s +echo "export QA_GITLAB_REVISION=`kubectl exec -i $(kubectl get pods -lrelease=${RELEASE_NAME},app=toolbox -o custom-columns=":metadata.name") -c toolbox -- cat /srv/gitlab/REVISION`" >> "${VARIABLES_FILE}" -- GitLab