From 95a21a2337092f326f23a847ca6b8d96308c3f25 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 19:44:00 -0500 Subject: [PATCH 01/22] Add function to connect to vcluster Adds a function to connect to vcluster if "VCLUSTER_NAME" environment variable is set. --- scripts/ci/autodevops.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh index a0b0f1ff70..d13b67463a 100644 --- a/scripts/ci/autodevops.sh +++ b/scripts/ci/autodevops.sh @@ -259,6 +259,14 @@ function set_context() { fi } +function vcluster_connect() { + if [ -z ${VCLUSTER_NAME} ]; then + echo 'VCLUSTER_NAME not configured, skipping `vcluster connect`' + else + vcluster connect "${VCLUSTER_NAME}" + fi +} + function check_kube_domain() { if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then echo "ERROR: In order to deploy, KUBE_INGRESS_BASE_DOMAIN must be set as a variable at the group or project level, or manually added in .gitlab-cy.yml" -- GitLab From ff8874afed4bcab69b396694aad0b1beacc39d9f Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 19:44:56 -0500 Subject: [PATCH 02/22] Implement vcluster_connect function Implements the function to connect to vcluster in the related review jobs. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fe49ff8be..40697b49e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,7 @@ lint_package: script: - mkdir -p $(dirname "${VARIABLES_FILE}") - set_context + - vcluster_connect - check_kube_domain - kubectl version - ensure_namespace @@ -292,6 +293,7 @@ review_eks: - git checkout master - source scripts/ci/autodevops.sh - set_context + - vcluster_connect - delete - cleanup when: manual -- GitLab From 495d91076ffad69cdb8c01e47eb67fc88dbf9e4b Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 19:46:05 -0500 Subject: [PATCH 03/22] Add jobs to test against K8s 1.26 in vcluster Adds jobs that will test against the vcluster instance of Kubernetes 1.26, which must be created manually. --- .gitlab-ci.yml | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40697b49e7..ca15d5a65c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -272,6 +272,21 @@ review_gke125: auto_stop_in: 2 days resource_group: "gke125-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" +review_gke126: + variables: + DNS_PROVIDER: "google" + AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running + KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" + KUBE_NAMESPACE: "helm-charts-win" + VCLUSTER_NAME: vcluster-k8s-126 + extends: .review_template + environment: + name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_review_gke126 + auto_stop_in: 2 days + resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" + review_eks: variables: DNS_PROVIDER: "aws" @@ -325,6 +340,17 @@ stop_review_gke125: name: gke125_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG action: stop +stop_review_gke126: + variables: + AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running + KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" + KUBE_NAMESPACE: "helm-charts-win" + VCLUSTER_NAME: vcluster-k8s-126 + extends: .stop_review_template + environment: + name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + action: stop + stop_review_eks: extends: .stop_review_template environment: @@ -659,6 +685,24 @@ review_specs_gke125: - !reference [.specs, rules] needs: ['review_gke125'] +review_specs_gke126: + extends: .specs + variables: + VARIABLES_FILE: "variables/review_gke126" + RSPEC_TAGS: type:feature + AGENT_NAME: "gke125-ci-cluster" # cnonect to 1.25 cluster, which has vcluster running + KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" + KUBE_NAMESPACE: "helm-charts-win" + environment: + name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_review_gke126 + resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" + rules: + - !reference [.rule:skip_if_no_cluster] + - !reference [.specs, rules] + needs: ['review_gke126'] + review_specs_eks: extends: .specs variables: @@ -904,6 +948,19 @@ qa_gke125: - job: review_specs_gke125 artifacts: false +qa_gke126: + extends: .qa_branch + variables: + VARIABLES_FILE: "variables/review_gke126" + TEST_SUITE: $QA_SANITY_SUITE_OPTIONS + environment: + name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + on_stop: stop_review_gke126 + needs: + - review_gke126 + - job: review_specs_gke126 + artifacts: false + qa_eks: extends: .qa_branch variables: @@ -1007,6 +1064,21 @@ qa_gke125_manual_full_suite: on_stop: stop_review_gke125 needs: [ "qa_gke125_full_suite_manual_trigger"] +qa_gke126_full_suite_manual_trigger: + extends: .qa_full_suite_manual_trigger + script: echo 'The job will trigger Full E2E suite against GKE126' + +qa_gke126_manual_full_suite: + extends: .qa_branch + parallel: 7 + variables: + VARIABLES_FILE: "variables/review_gke126" + TEST_SUITE: $QA_FULL_SUITE_OPTIONS + environment: + name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + on_stop: stop_review_gke126 + needs: [ "qa_gke126_full_suite_manual_trigger"] + qa_eks_full_suite_manual_trigger: extends: .qa_full_suite_manual_trigger script: echo 'The job will trigger Full E2E suite against EKS' -- GitLab From 3227526a5b44d27d25b8ae169a47e52617b7b338 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 11:36:50 -0500 Subject: [PATCH 04/22] Rename environment to use GKE 1.25 agent Renames the environments for the 1.26-related jobs to use the GKE 1.25 agent connection. --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca15d5a65c..174180ab03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -281,7 +281,7 @@ review_gke126: VCLUSTER_NAME: vcluster-k8s-126 extends: .review_template environment: - name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN on_stop: stop_review_gke126 auto_stop_in: 2 days @@ -348,7 +348,7 @@ stop_review_gke126: VCLUSTER_NAME: vcluster-k8s-126 extends: .stop_review_template environment: - name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG action: stop stop_review_eks: @@ -694,7 +694,7 @@ review_specs_gke126: KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" KUBE_NAMESPACE: "helm-charts-win" environment: - name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN on_stop: stop_review_gke126 resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" @@ -954,7 +954,7 @@ qa_gke126: VARIABLES_FILE: "variables/review_gke126" TEST_SUITE: $QA_SANITY_SUITE_OPTIONS environment: - name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG on_stop: stop_review_gke126 needs: - review_gke126 @@ -1075,7 +1075,7 @@ qa_gke126_manual_full_suite: VARIABLES_FILE: "variables/review_gke126" TEST_SUITE: $QA_FULL_SUITE_OPTIONS environment: - name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG + name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG on_stop: stop_review_gke126 needs: [ "qa_gke126_full_suite_manual_trigger"] -- GitLab From df29c5124c5ebeed7fa84d289a7eb014025e5fbe Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 11:42:38 -0500 Subject: [PATCH 05/22] Add kubeval job for 1.26 Adds a job to run kubeval against 1.26.7. --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 174180ab03..f614d14c9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,6 +208,17 @@ lint_package: --set global.hpa.apiVersion=autoscaling/v2 --set global.batch.cronJob.apiVersion=batch/v1 +"Validate 1.26.7": + extends: .kubeval + variables: + KUBE_VERSION: "1.26.7" + KUBEVAL_SCHEMA_LOCATION: "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/" + HELM_EXTRAS: >- + --set global.ingress.apiVersion=networking.k8s.io/v1 + --set global.pdb.apiVersion=policy/v1 + --set global.hpa.apiVersion=autoscaling/v2 + --set global.batch.cronJob.apiVersion=batch/v1 + .review_template: stage: review variables: -- GitLab From 214acfe49c9d59ad20085233aababc3d7f41e6e0 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 13:29:47 -0500 Subject: [PATCH 06/22] Rename vcluster instances Renames the vcluster instances, as the 'vcluster-' prefix ends up being redundant in the name of the namespace. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f614d14c9b..41ba98b65e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -289,7 +289,7 @@ review_gke126: AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" KUBE_NAMESPACE: "helm-charts-win" - VCLUSTER_NAME: vcluster-k8s-126 + VCLUSTER_NAME: k8s-126 extends: .review_template environment: name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG @@ -356,7 +356,7 @@ stop_review_gke126: AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" KUBE_NAMESPACE: "helm-charts-win" - VCLUSTER_NAME: vcluster-k8s-126 + VCLUSTER_NAME: k8s-126 extends: .stop_review_template environment: name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG -- GitLab From 40ccc8b64e42a593776d0f159ccff09463226c9f Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 13:30:27 -0500 Subject: [PATCH 07/22] [tmp] Use build base image with vcluster Temporarily uses a build base image that contains vcluster. --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41ba98b65e..6ddca9c067 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,9 @@ # Note: Auto CI does not work with multiple buildpacks yet default: - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base-helm-3.7 + # image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base-helm-3.7 + # TODO: restore image above once it contains `vcluster` + image: registry.gitlab.com/mnielsen/tmp-registry/gitlab-charts-build-base:vcluster variables: AUTO_DEPLOY_TAG_REGEX: '^[0-9]+\.[0-9]+\.[0-9]+\+[a-z0-9]{7,}$' -- GitLab From 7e81c8bb6dfee16b59e939192488f5fe0f8982f1 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 14:45:09 -0500 Subject: [PATCH 08/22] Specify namespace for Secret generation Specifies the namespace argument for the `kubectl` commands that create the license and root password Secrets. We got away with not specifying these before because `set_context()` would connect to a specific namespace, but with vcluster we're inside a different context. --- scripts/ci/autodevops.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh index d13b67463a..303c4ab703 100644 --- a/scripts/ci/autodevops.sh +++ b/scripts/ci/autodevops.sh @@ -85,10 +85,10 @@ function deploy() { #ROOT_PASSWORD=$(cat /dev/urandom | LC_TYPE=C tr -dc "[:alpha:]" | head -c 16) #echo "Generated root login: $ROOT_PASSWORD" - kubectl create secret generic "${RELEASE_NAME}-gitlab-initial-root-password" --from-literal=password=$ROOT_PASSWORD -o yaml --dry-run=client | kubectl replace --force -f - + kubectl -n ${NAMESPACE} create secret generic "${RELEASE_NAME}-gitlab-initial-root-password" --from-literal=password=$ROOT_PASSWORD -o yaml --dry-run=client | kubectl -n ${NAMESPACE} replace --force -f - echo "${QA_EE_LICENSE}" > /tmp/license.gitlab - kubectl create secret generic "${RELEASE_NAME}-gitlab-license" --from-file=license=/tmp/license.gitlab -o yaml --dry-run=client | kubectl replace --force -f - + kubectl -n ${NAMESPACE} create secret generic "${RELEASE_NAME}-gitlab-license" --from-file=license=/tmp/license.gitlab -o yaml --dry-run=client | kubectl -n ${NAMESPACE} replace --force -f - # YAML_FILE=""${KUBE_INGRESS_BASE_DOMAIN//\./-}.yaml" -- GitLab From f226b15deb4cd83361734f44a11effe035b91c3a Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 15:19:47 -0500 Subject: [PATCH 09/22] Set namespace in vcluster context Specs don't specify the namespace, so we need to set the namespace manually before proceeding. --- scripts/ci/autodevops.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh index 303c4ab703..968d6a4213 100644 --- a/scripts/ci/autodevops.sh +++ b/scripts/ci/autodevops.sh @@ -264,6 +264,11 @@ function vcluster_connect() { echo 'VCLUSTER_NAME not configured, skipping `vcluster connect`' else vcluster connect "${VCLUSTER_NAME}" + + # Ensure that ${NAMESPACE} exists in the vcluster context, too. + ensure_namespace + + kubectl config set-context --current --namespace=${NAMESPACE} fi } -- GitLab From d40e93f31d985fcb0183fca27be845196464a3e1 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 10 Aug 2023 15:45:35 -0500 Subject: [PATCH 10/22] Install vcluster and connect in specs job For GKE 1.26 specs job, install vcluster and connect to the related instance. --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ddca9c067..96320da74f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -644,8 +644,11 @@ review-docs-cleanup: CHROME_VERSION: "109" extends: .source_autodevops script: + - curl -L -o /usr/local/bin/vcluster "https://github.com/loft-sh/vcluster/releases/download/v0.15.5/vcluster-linux-amd64" + - chmod +x /usr/local/bin/vcluster - ./scripts/ci/install_spec_dependencies - set_context + - vcluster_connect - ./scripts/ci/run_specs artifacts: when: on_failure @@ -706,6 +709,7 @@ review_specs_gke126: AGENT_NAME: "gke125-ci-cluster" # cnonect to 1.25 cluster, which has vcluster running KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" KUBE_NAMESPACE: "helm-charts-win" + VCLUSTER_NAME: k8s-126 environment: name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN -- GitLab From d00ba6f49ed1f67a8d0f1a8898b12d6394904cc6 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 18:24:29 -0500 Subject: [PATCH 11/22] Run a separate vcluster instance per review app This makes a few changes: * Pulls vcluster-related jobs into a separate CI file * Creates a separate script for vcluster-related commands * Runs a vcluster instance for each review app * Only deploys a simple chart without any connection to external-dns or cert-manager * Uses parallel:matrix syntax in CI to keep configuration DRY --- .gitlab-ci.yml | 79 +---------------------- .gitlab/ci/review-apps.gitlab-ci.yml | 33 ++++++++++ scripts/ci/autodevops.sh | 13 ---- scripts/ci/vcluster.sh | 94 ++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 91 deletions(-) create mode 100644 .gitlab/ci/review-apps.gitlab-ci.yml create mode 100755 scripts/ci/vcluster.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96320da74f..183b946350 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,6 +75,7 @@ stages: include: - local: '/.gitlab/ci/rules.gitlab-ci.yml' + - local: '/.gitlab/ci/review-apps.gitlab-ci.yml' - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml - template: Jobs/Secret-Detection.latest.gitlab-ci.yml - template: Jobs/SAST.latest.gitlab-ci.yml @@ -231,7 +232,6 @@ lint_package: script: - mkdir -p $(dirname "${VARIABLES_FILE}") - set_context - - vcluster_connect - check_kube_domain - kubectl version - ensure_namespace @@ -285,21 +285,6 @@ review_gke125: auto_stop_in: 2 days resource_group: "gke125-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" -review_gke126: - variables: - DNS_PROVIDER: "google" - AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running - KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" - KUBE_NAMESPACE: "helm-charts-win" - VCLUSTER_NAME: k8s-126 - extends: .review_template - environment: - name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG - url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN - on_stop: stop_review_gke126 - auto_stop_in: 2 days - resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" - review_eks: variables: DNS_PROVIDER: "aws" @@ -321,7 +306,6 @@ review_eks: - git checkout master - source scripts/ci/autodevops.sh - set_context - - vcluster_connect - delete - cleanup when: manual @@ -353,17 +337,6 @@ stop_review_gke125: name: gke125_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG action: stop -stop_review_gke126: - variables: - AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster, which has vcluster running - KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" - KUBE_NAMESPACE: "helm-charts-win" - VCLUSTER_NAME: k8s-126 - extends: .stop_review_template - environment: - name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG - action: stop - stop_review_eks: extends: .stop_review_template environment: @@ -644,11 +617,8 @@ review-docs-cleanup: CHROME_VERSION: "109" extends: .source_autodevops script: - - curl -L -o /usr/local/bin/vcluster "https://github.com/loft-sh/vcluster/releases/download/v0.15.5/vcluster-linux-amd64" - - chmod +x /usr/local/bin/vcluster - ./scripts/ci/install_spec_dependencies - set_context - - vcluster_connect - ./scripts/ci/run_specs artifacts: when: on_failure @@ -701,25 +671,6 @@ review_specs_gke125: - !reference [.specs, rules] needs: ['review_gke125'] -review_specs_gke126: - extends: .specs - variables: - VARIABLES_FILE: "variables/review_gke126" - RSPEC_TAGS: type:feature - AGENT_NAME: "gke125-ci-cluster" # cnonect to 1.25 cluster, which has vcluster running - KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v126.helm-charts.win" - KUBE_NAMESPACE: "helm-charts-win" - VCLUSTER_NAME: k8s-126 - environment: - name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG - url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN - on_stop: stop_review_gke126 - resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}" - rules: - - !reference [.rule:skip_if_no_cluster] - - !reference [.specs, rules] - needs: ['review_gke126'] - review_specs_eks: extends: .specs variables: @@ -965,19 +916,6 @@ qa_gke125: - job: review_specs_gke125 artifacts: false -qa_gke126: - extends: .qa_branch - variables: - VARIABLES_FILE: "variables/review_gke126" - TEST_SUITE: $QA_SANITY_SUITE_OPTIONS - environment: - name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG - on_stop: stop_review_gke126 - needs: - - review_gke126 - - job: review_specs_gke126 - artifacts: false - qa_eks: extends: .qa_branch variables: @@ -1081,21 +1019,6 @@ qa_gke125_manual_full_suite: on_stop: stop_review_gke125 needs: [ "qa_gke125_full_suite_manual_trigger"] -qa_gke126_full_suite_manual_trigger: - extends: .qa_full_suite_manual_trigger - script: echo 'The job will trigger Full E2E suite against GKE126' - -qa_gke126_manual_full_suite: - extends: .qa_branch - parallel: 7 - variables: - VARIABLES_FILE: "variables/review_gke126" - TEST_SUITE: $QA_FULL_SUITE_OPTIONS - environment: - name: gke125_vcluster_126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG - on_stop: stop_review_gke126 - needs: [ "qa_gke126_full_suite_manual_trigger"] - qa_eks_full_suite_manual_trigger: extends: .qa_full_suite_manual_trigger script: echo 'The job will trigger Full E2E suite against EKS' diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml new file mode 100644 index 0000000000..5fc2916987 --- /dev/null +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -0,0 +1,33 @@ +review_apps: + stage: review + variables: + AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster until we have a dedicated cluster + before_script: + - source scripts/ci/vcluster.sh + script: + - cluster_connect + - kubectl version + - vcluster_create + - vcluster_connect + - kubectl version + parallel: + matrix: + - VCLUSTER_K8S_VERSION: + - "1.26" + rules: + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + +stop_review_apps: + extends: review_apps + dependencies: [] + script: + - cluster_connect + - vcluster_delete + rules: + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + when: delayed + start_in: 1 hour + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + when: delayed + start_in: 1 hour diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh index 968d6a4213..77605c4a33 100644 --- a/scripts/ci/autodevops.sh +++ b/scripts/ci/autodevops.sh @@ -259,19 +259,6 @@ function set_context() { fi } -function vcluster_connect() { - if [ -z ${VCLUSTER_NAME} ]; then - echo 'VCLUSTER_NAME not configured, skipping `vcluster connect`' - else - vcluster connect "${VCLUSTER_NAME}" - - # Ensure that ${NAMESPACE} exists in the vcluster context, too. - ensure_namespace - - kubectl config set-context --current --namespace=${NAMESPACE} - fi -} - function check_kube_domain() { if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then echo "ERROR: In order to deploy, KUBE_INGRESS_BASE_DOMAIN must be set as a variable at the group or project level, or manually added in .gitlab-cy.yml" diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh new file mode 100755 index 0000000000..aacf732ff7 --- /dev/null +++ b/scripts/ci/vcluster.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +function cluster_connect() { + if [ -z ${AGENT_NAME+x} ] || [ -z ${AGENT_PROJECT_PATH+x} ]; then + echo "No AGENT_NAME or AGENT_PROJECT_PATH set, using the default" + else + kubectl config get-contexts + kubectl config use-context ${AGENT_PROJECT_PATH}:${AGENT_NAME} + fi +} + +function vcluster_name() { + echo -n "vcluster-k8s-${VCLUSTER_K8S_VERSION}-${CI_COMMIT_REF_SLUG}" +} + +function vcluster_create() { + name=$(vcluster_name) + vcluster create ${name} \ + --upgrade \ + --namespace=${name} \ + --kubernetes-version=${VCLUSTER_K8S_VERSION} \ + --connect=false \ + --update-current=false +} + +function vcluster_connect() { + name=$(vcluster_name) + vcluster connect ${name} +} + +function vcluster_deploy() { + helm dependency update + + cat << CIVALUES > ci.yaml + global: + hosts: + https: false + image: + pullPolicy: Always + ingress: + configureCertmanager: false + tls: + enabled: false + appConfig: + initialDefaults: + signupEnabled: false + gitlab: + webservice: + minReplicas: 1 # 2 + maxReplicas: 3 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1500M # 2.5G + sidekiq: + minReplicas: 1 # 1 + maxReplicas: 2 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1000M # 2G + gitlab-shell: + minReplicas: 1 # 2 + maxReplicas: 2 # 10 + toolbox: + enabled: true + gitlab-runner: + certsSecretName: gitlab-wildcard-tls-chain + certmanager: + install: false + nginx-ingress: + controller: + replicaCount: 1 # 2 + redis: + resources: + requests: + cpu: 100m + minio: + resources: + requests: + cpu: 100m +CIVALUES + + helm upgrade --install \ + gitlab \ + --wait --timeout 600s \ + -f ci.yaml \ + . +} + +function vcluster_delete() { + name=$(vcluster_name) + vcluster delete ${name} +} -- GitLab From aeef6c5d3b253f12cf1fb456eef85a0bfcb5662a Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 19:12:21 -0500 Subject: [PATCH 12/22] Replace parallel:matrix setup parallel:matrix doesn't seem to work well with environments. --- .gitlab/ci/review-apps.gitlab-ci.yml | 47 ++++++++++++++++++---------- scripts/ci/vcluster.sh | 2 +- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 5fc2916987..0653d4e6e6 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -1,33 +1,46 @@ -review_apps: +.review_app_common: stage: review variables: AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster until we have a dedicated cluster + environment: + name: gke125_vcluster/${VCLUSTER_K8S_VERSION}/$CI_COMMIT_REF_SLUG + auto_stop_in: 1 hour before_script: - source scripts/ci/vcluster.sh + rules: + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + +.review_app_template: + extends: .review_app_common script: - cluster_connect - kubectl version - vcluster_create - vcluster_connect - kubectl version - parallel: - matrix: - - VCLUSTER_K8S_VERSION: - - "1.26" - rules: - - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' - - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' -stop_review_apps: - extends: review_apps +.stop_review_app_template: + extends: .review_app_common dependencies: [] script: - cluster_connect - vcluster_delete - rules: - - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' - when: delayed - start_in: 1 hour - - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' - when: delayed - start_in: 1 hour + environment: + action: stop + +# Below, create two jobs for each Kubernetes version: +# - one to create the environment +# - one to stop the environment + +review_gke_k8s_126: + extends: .review_app_template + variables: + VCLUSTER_K8S_VERSION: "1.26" + environment: + on_stop: stop_review_gke_k8s_126 + +stop_review_gke_k8s_126: + extends: .stop_review_app_template + variables: + VCLUSTER_K8S_VERSION: "1.26" diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index aacf732ff7..68dcbcc39c 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -10,7 +10,7 @@ function cluster_connect() { } function vcluster_name() { - echo -n "vcluster-k8s-${VCLUSTER_K8S_VERSION}-${CI_COMMIT_REF_SLUG}" + echo -n "vcluster-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}" } function vcluster_create() { -- GitLab From cf971a4a360bc6e2a062818d1ac1da632700bdf5 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 19:35:03 -0500 Subject: [PATCH 13/22] Refactor vcluster name for RFC compatibility --- .gitlab/ci/review-apps.gitlab-ci.yml | 6 +++--- scripts/ci/vcluster.sh | 15 ++++----------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 0653d4e6e6..b697b4b1d5 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -3,7 +3,7 @@ variables: AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster until we have a dedicated cluster environment: - name: gke125_vcluster/${VCLUSTER_K8S_VERSION}/$CI_COMMIT_REF_SLUG + name: gke125_vcluster/${VCLUSTER_NAME} auto_stop_in: 1 hour before_script: - source scripts/ci/vcluster.sh @@ -37,10 +37,10 @@ review_gke_k8s_126: extends: .review_app_template variables: VCLUSTER_K8S_VERSION: "1.26" + VCLUSTER_NAME: vcluster-1-26-${CI_COMMIT_REF_SLUG} environment: on_stop: stop_review_gke_k8s_126 stop_review_gke_k8s_126: extends: .stop_review_app_template - variables: - VCLUSTER_K8S_VERSION: "1.26" + variables: !reference [review_gke_k8s_126, variables] diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 68dcbcc39c..679994b87b 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -9,23 +9,17 @@ function cluster_connect() { fi } -function vcluster_name() { - echo -n "vcluster-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}" -} - function vcluster_create() { - name=$(vcluster_name) - vcluster create ${name} \ + vcluster create ${VCLUSTER_NAME} \ --upgrade \ - --namespace=${name} \ + --namespace=${VCLUSTER_NAME} \ --kubernetes-version=${VCLUSTER_K8S_VERSION} \ --connect=false \ --update-current=false } function vcluster_connect() { - name=$(vcluster_name) - vcluster connect ${name} + vcluster connect ${VCLUSTER_NAME} } function vcluster_deploy() { @@ -89,6 +83,5 @@ CIVALUES } function vcluster_delete() { - name=$(vcluster_name) - vcluster delete ${name} + vcluster delete ${VCLUSTER_NAME} } -- GitLab From 3ab560f20d5f2afb138b7d6a6a124a0687cd7e8b Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 19:49:48 -0500 Subject: [PATCH 14/22] Ensure stop environment is manual --- .gitlab/ci/review-apps.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index b697b4b1d5..28bd064b4d 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -28,6 +28,7 @@ - vcluster_delete environment: action: stop + when: manual # Below, create two jobs for each Kubernetes version: # - one to create the environment -- GitLab From e337208a4060f8848d41314c544ee25787e5ca64 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 19:50:13 -0500 Subject: [PATCH 15/22] Change NGINX Service type to NodePort To avoid creating a loadbalancer. --- scripts/ci/vcluster.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 679994b87b..fb091d933a 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -65,6 +65,8 @@ function vcluster_deploy() { nginx-ingress: controller: replicaCount: 1 # 2 + service: + type: NodePort # to avoid creating a LoadBalancer redis: resources: requests: -- GitLab From 9e5cc806500a64e101be1cac01b7d477dbd0bfc8 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 19:52:09 -0500 Subject: [PATCH 16/22] Expose vcluster to prevent blocking command Without exposing vcluster via loadbalancer, it port-forwards the connection which creates a blocking command. --- scripts/ci/vcluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index fb091d933a..1af9d78ec1 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -11,6 +11,7 @@ function cluster_connect() { function vcluster_create() { vcluster create ${VCLUSTER_NAME} \ + --expose \ --upgrade \ --namespace=${VCLUSTER_NAME} \ --kubernetes-version=${VCLUSTER_K8S_VERSION} \ -- GitLab From 98c0e474ed320dc462c2fa9f04fb8f77f57b75b0 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 20:12:00 -0500 Subject: [PATCH 17/22] Disable vcluster expose, run commands directly Runs commands passed to `vcluster connect` instead of having to expose the cluster via a LoadBalancer. --- .gitlab/ci/review-apps.gitlab-ci.yml | 4 +- scripts/ci/vcluster.sh | 61 ++-------------------------- scripts/ci/vcluster_helm_values.yaml | 49 ++++++++++++++++++++++ 3 files changed, 55 insertions(+), 59 deletions(-) create mode 100644 scripts/ci/vcluster_helm_values.yaml diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 28bd064b4d..82185ba259 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -17,8 +17,8 @@ - cluster_connect - kubectl version - vcluster_create - - vcluster_connect - - kubectl version + - vcluster_run kubectl version + - vcluster_deploy .stop_review_app_template: extends: .review_app_common diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 1af9d78ec1..8738c17f0f 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -11,7 +11,6 @@ function cluster_connect() { function vcluster_create() { vcluster create ${VCLUSTER_NAME} \ - --expose \ --upgrade \ --namespace=${VCLUSTER_NAME} \ --kubernetes-version=${VCLUSTER_K8S_VERSION} \ @@ -19,69 +18,17 @@ function vcluster_create() { --update-current=false } -function vcluster_connect() { - vcluster connect ${VCLUSTER_NAME} +function vcluster_run() { + vcluster connect ${VCLUSTER_NAME} -- $@ } function vcluster_deploy() { helm dependency update - cat << CIVALUES > ci.yaml - global: - hosts: - https: false - image: - pullPolicy: Always - ingress: - configureCertmanager: false - tls: - enabled: false - appConfig: - initialDefaults: - signupEnabled: false - gitlab: - webservice: - minReplicas: 1 # 2 - maxReplicas: 3 # 10 - resources: - requests: - cpu: 500m # 900m - memory: 1500M # 2.5G - sidekiq: - minReplicas: 1 # 1 - maxReplicas: 2 # 10 - resources: - requests: - cpu: 500m # 900m - memory: 1000M # 2G - gitlab-shell: - minReplicas: 1 # 2 - maxReplicas: 2 # 10 - toolbox: - enabled: true - gitlab-runner: - certsSecretName: gitlab-wildcard-tls-chain - certmanager: - install: false - nginx-ingress: - controller: - replicaCount: 1 # 2 - service: - type: NodePort # to avoid creating a LoadBalancer - redis: - resources: - requests: - cpu: 100m - minio: - resources: - requests: - cpu: 100m -CIVALUES - - helm upgrade --install \ + vcluster_run helm upgrade --install \ gitlab \ --wait --timeout 600s \ - -f ci.yaml \ + -f ./scripts/ci/vcluster_helm_values.yaml \ . } diff --git a/scripts/ci/vcluster_helm_values.yaml b/scripts/ci/vcluster_helm_values.yaml new file mode 100644 index 0000000000..1768538d88 --- /dev/null +++ b/scripts/ci/vcluster_helm_values.yaml @@ -0,0 +1,49 @@ +global: + hosts: + https: false + image: + pullPolicy: Always + ingress: + configureCertmanager: false + tls: + enabled: false + appConfig: + initialDefaults: + signupEnabled: false +gitlab: + webservice: + minReplicas: 1 # 2 + maxReplicas: 3 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1500M # 2.5G + sidekiq: + minReplicas: 1 # 1 + maxReplicas: 2 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1000M # 2G + gitlab-shell: + minReplicas: 1 # 2 + maxReplicas: 2 # 10 + toolbox: + enabled: true +gitlab-runner: + certsSecretName: gitlab-wildcard-tls-chain +certmanager: + install: false +nginx-ingress: + controller: + replicaCount: 1 # 2 + service: + type: NodePort # to avoid creating a LoadBalancer +redis: + resources: + requests: + cpu: 100m +minio: + resources: + requests: + cpu: 100m -- GitLab From 7a1f735242f55a379eeb9abbb86a9c790825ce7c Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 21:00:21 -0500 Subject: [PATCH 18/22] Test the Gitlab homepage --- .gitlab/ci/review-apps.gitlab-ci.yml | 4 +++- scripts/ci/vcluster.sh | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 82185ba259..21e0758d4e 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -18,7 +18,9 @@ - kubectl version - vcluster_create - vcluster_run kubectl version - - vcluster_deploy + - vcluster_helm_deploy + - vcluster_helm_rollout_status + - vcluster_confirm_homepage .stop_review_app_template: extends: .review_app_common diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 8738c17f0f..1b1fa42dc6 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -22,7 +22,7 @@ function vcluster_run() { vcluster connect ${VCLUSTER_NAME} -- $@ } -function vcluster_deploy() { +function vcluster_helm_deploy() { helm dependency update vcluster_run helm upgrade --install \ @@ -32,6 +32,18 @@ function vcluster_deploy() { . } +function vcluster_helm_rollout_status() { + kubectl rollout status deployment -lrelease=gitlab --timeout=60s +} + +function vcluster_confirm_homepage() { + vcluster_run nohup kubectl port-forward service/gitlab-nginx-ingress-controller 8080:http & + + curl -IL --fail \ + http://gitlab.example.com:8080 \ + --resolve gitlab.example.com:8080:127.0.0.1 +} + function vcluster_delete() { vcluster delete ${VCLUSTER_NAME} } -- GitLab From d4f359b114b2b25dc92f40de5ede5f5f8138fc6d Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 21:00:45 -0500 Subject: [PATCH 19/22] Print out guidance to connect to vcluster --- .gitlab/ci/review-apps.gitlab-ci.yml | 1 + scripts/ci/vcluster.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 21e0758d4e..e90f0e5759 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -21,6 +21,7 @@ - vcluster_helm_deploy - vcluster_helm_rollout_status - vcluster_confirm_homepage + - vcluster_info .stop_review_app_template: extends: .review_app_common diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 1b1fa42dc6..7c2586a21a 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -47,3 +47,10 @@ function vcluster_confirm_homepage() { function vcluster_delete() { vcluster delete ${VCLUSTER_NAME} } + +function vcluster_info() { + echo "To connect to the virtual cluster:" + echo "1. Connect to host cluster via kubectl: ${AGENT_NAME}" + echo "2. Connect to virtual cluster: vcluster connect ${VCLUSTER_NAME}" + echo "3. Open a separate terminal window and run your kubectl and helm commands." +} -- GitLab From 200e13f2bc61bb1c8dd1cbd97a8d368f21152df8 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 21:01:15 -0500 Subject: [PATCH 20/22] Disable gitlab-runner chart gitlab-runner won't be able to connect to the instance so it will prevent the `helm install` from returning successfully. --- scripts/ci/vcluster_helm_values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/vcluster_helm_values.yaml b/scripts/ci/vcluster_helm_values.yaml index 1768538d88..5663ec9276 100644 --- a/scripts/ci/vcluster_helm_values.yaml +++ b/scripts/ci/vcluster_helm_values.yaml @@ -31,7 +31,7 @@ gitlab: toolbox: enabled: true gitlab-runner: - certsSecretName: gitlab-wildcard-tls-chain + install: false certmanager: install: false nginx-ingress: -- GitLab From 9a158e22256e9aa38acadffa3c6f24b6cbad6fc6 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 21:09:59 -0500 Subject: [PATCH 21/22] Temporarily disable helm rollout status Shorthand -l not available in this version of kubectl --- .gitlab/ci/review-apps.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index e90f0e5759..5d875692a1 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -19,7 +19,7 @@ - vcluster_create - vcluster_run kubectl version - vcluster_helm_deploy - - vcluster_helm_rollout_status + # - vcluster_helm_rollout_status - vcluster_confirm_homepage - vcluster_info -- GitLab From 3144a82af3ea8bb39802c2a2559ef5e01626dd19 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 7 Sep 2023 21:46:31 -0500 Subject: [PATCH 22/22] Use rollout status as check instead of homepage The homepage check required kubectl port-forward, and even `nohup` with `&` didn't get it working correctly in CI even though it worked in a local terminal. Using kubectl for the rollout status of Webservice is still a helpful check because the readiness probes are configured, and the rollout status won't return successfully until all containers in the Pod(s) are Ready. --- .gitlab/ci/review-apps.gitlab-ci.yml | 3 +-- scripts/ci/vcluster.sh | 10 +--------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 5d875692a1..62948e5cd3 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -19,8 +19,7 @@ - vcluster_create - vcluster_run kubectl version - vcluster_helm_deploy - # - vcluster_helm_rollout_status - - vcluster_confirm_homepage + - vcluster_helm_rollout_status - vcluster_info .stop_review_app_template: diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh index 7c2586a21a..853365b3c5 100755 --- a/scripts/ci/vcluster.sh +++ b/scripts/ci/vcluster.sh @@ -33,15 +33,7 @@ function vcluster_helm_deploy() { } function vcluster_helm_rollout_status() { - kubectl rollout status deployment -lrelease=gitlab --timeout=60s -} - -function vcluster_confirm_homepage() { - vcluster_run nohup kubectl port-forward service/gitlab-nginx-ingress-controller 8080:http & - - curl -IL --fail \ - http://gitlab.example.com:8080 \ - --resolve gitlab.example.com:8080:127.0.0.1 + vcluster_run kubectl rollout status deployment/gitlab-webservice-default --timeout=300s } function vcluster_delete() { -- GitLab