From aabd06a1d644d70f3b86ea22b4e2627ad9ab9eb0 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Fri, 8 Sep 2023 15:32:02 -0500 Subject: [PATCH 1/8] Add script for vcluster-related functions Adds a script with vcluster-related functions to be called in CI, along with a dedicated Helm values file to keep the script simple. --- scripts/ci/vcluster.sh | 48 +++++++++++++++++++++++++++ scripts/ci/vcluster_helm_values.yaml | 49 ++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100755 scripts/ci/vcluster.sh create mode 100644 scripts/ci/vcluster_helm_values.yaml diff --git a/scripts/ci/vcluster.sh b/scripts/ci/vcluster.sh new file mode 100755 index 0000000000..853365b3c5 --- /dev/null +++ b/scripts/ci/vcluster.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +function cluster_connect() { + if [ -z ${AGENT_NAME+x} ] || [ -z ${AGENT_PROJECT_PATH+x} ]; then + echo "No AGENT_NAME or AGENT_PROJECT_PATH set, using the default" + else + kubectl config get-contexts + kubectl config use-context ${AGENT_PROJECT_PATH}:${AGENT_NAME} + fi +} + +function vcluster_create() { + vcluster create ${VCLUSTER_NAME} \ + --upgrade \ + --namespace=${VCLUSTER_NAME} \ + --kubernetes-version=${VCLUSTER_K8S_VERSION} \ + --connect=false \ + --update-current=false +} + +function vcluster_run() { + vcluster connect ${VCLUSTER_NAME} -- $@ +} + +function vcluster_helm_deploy() { + helm dependency update + + vcluster_run helm upgrade --install \ + gitlab \ + --wait --timeout 600s \ + -f ./scripts/ci/vcluster_helm_values.yaml \ + . +} + +function vcluster_helm_rollout_status() { + vcluster_run kubectl rollout status deployment/gitlab-webservice-default --timeout=300s +} + +function vcluster_delete() { + vcluster delete ${VCLUSTER_NAME} +} + +function vcluster_info() { + echo "To connect to the virtual cluster:" + echo "1. Connect to host cluster via kubectl: ${AGENT_NAME}" + echo "2. Connect to virtual cluster: vcluster connect ${VCLUSTER_NAME}" + echo "3. Open a separate terminal window and run your kubectl and helm commands." +} diff --git a/scripts/ci/vcluster_helm_values.yaml b/scripts/ci/vcluster_helm_values.yaml new file mode 100644 index 0000000000..5663ec9276 --- /dev/null +++ b/scripts/ci/vcluster_helm_values.yaml @@ -0,0 +1,49 @@ +global: + hosts: + https: false + image: + pullPolicy: Always + ingress: + configureCertmanager: false + tls: + enabled: false + appConfig: + initialDefaults: + signupEnabled: false +gitlab: + webservice: + minReplicas: 1 # 2 + maxReplicas: 3 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1500M # 2.5G + sidekiq: + minReplicas: 1 # 1 + maxReplicas: 2 # 10 + resources: + requests: + cpu: 500m # 900m + memory: 1000M # 2G + gitlab-shell: + minReplicas: 1 # 2 + maxReplicas: 2 # 10 + toolbox: + enabled: true +gitlab-runner: + install: false +certmanager: + install: false +nginx-ingress: + controller: + replicaCount: 1 # 2 + service: + type: NodePort # to avoid creating a LoadBalancer +redis: + resources: + requests: + cpu: 100m +minio: + resources: + requests: + cpu: 100m -- GitLab From fd44a2639a11780a147b7367a333e229f4219991 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Fri, 8 Sep 2023 15:34:19 -0500 Subject: [PATCH 2/8] Add CI template with vcluster jobs Adds a dedicated CI file for vcluster-related jobs. --- .gitlab-ci.yml | 1 + .gitlab/ci/review-apps.gitlab-ci.yml | 49 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 .gitlab/ci/review-apps.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c2802c01a..6451bd340a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,6 +82,7 @@ stages: include: - local: '/.gitlab/ci/rules.gitlab-ci.yml' + - local: '/.gitlab/ci/review-apps.gitlab-ci.yml' - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml - template: Jobs/Secret-Detection.latest.gitlab-ci.yml - template: Jobs/SAST.latest.gitlab-ci.yml diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml new file mode 100644 index 0000000000..62948e5cd3 --- /dev/null +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -0,0 +1,49 @@ +.review_app_common: + stage: review + variables: + AGENT_NAME: "gke125-ci-cluster" # connect to 1.25 cluster until we have a dedicated cluster + environment: + name: gke125_vcluster/${VCLUSTER_NAME} + auto_stop_in: 1 hour + before_script: + - source scripts/ci/vcluster.sh + rules: + - if: '$PIPELINE_TYPE =~ /MR_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/' + +.review_app_template: + extends: .review_app_common + script: + - cluster_connect + - kubectl version + - vcluster_create + - vcluster_run kubectl version + - vcluster_helm_deploy + - vcluster_helm_rollout_status + - vcluster_info + +.stop_review_app_template: + extends: .review_app_common + dependencies: [] + script: + - cluster_connect + - vcluster_delete + environment: + action: stop + when: manual + +# Below, create two jobs for each Kubernetes version: +# - one to create the environment +# - one to stop the environment + +review_gke_k8s_126: + extends: .review_app_template + variables: + VCLUSTER_K8S_VERSION: "1.26" + VCLUSTER_NAME: vcluster-1-26-${CI_COMMIT_REF_SLUG} + environment: + on_stop: stop_review_gke_k8s_126 + +stop_review_gke_k8s_126: + extends: .stop_review_app_template + variables: !reference [review_gke_k8s_126, variables] -- GitLab From f813bd7e7084e7d97d3f4e0b52609630e4d7585d Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Fri, 8 Sep 2023 15:35:07 -0500 Subject: [PATCH 3/8] Add kubeconform job for 1.26 Adds a kubeconform job to test 1.26.8. --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6451bd340a..41d0d02373 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,6 +233,22 @@ lint_package: cronJob: apiVersion: batch/v1 +"Validate 1.26.8": + extends: .kubeconform + variables: + KUBE_VERSION: "1.26.8" + HELM_SETTINGS: | + global: + ingress: + apiVersion: networking.k8s.io/v1 + pdb: + apiVersion: policy/v1 + hpa: + apiVersion: autoscaling/v2 + batch: + cronJob: + apiVersion: batch/v1 + .review_template: stage: review variables: -- GitLab From f45922d99cf7f8852c1242035c48f9f1edeaa317 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Fri, 8 Sep 2023 15:35:43 -0500 Subject: [PATCH 4/8] To remove: override default image for one with vcluster Overrides the default image to include one that has vcluster included. We can revert this commit once the base image has vcluster built in. --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41d0d02373..de762e9152 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,9 @@ # Note: Auto CI does not work with multiple buildpacks yet default: - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base-helm-3.7 + # image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base-helm-3.7 + # TODO: restore image above once it contains `vcluster` + image: registry.gitlab.com/mnielsen/tmp-registry/gitlab-charts-build-base:vcluster variables: AUTO_DEPLOY_TAG_REGEX: '^[0-9]+\.[0-9]+\.[0-9]+\+[a-z0-9]{7,}$' -- GitLab From 17a551e350da2888ed46a08d14623aab565698c8 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Fri, 8 Sep 2023 16:33:57 -0500 Subject: [PATCH 5/8] Add jobs to test against Kubernetes 1.27 --- .gitlab-ci.yml | 16 ++++++++++++++++ .gitlab/ci/review-apps.gitlab-ci.yml | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de762e9152..7ca847b638 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -251,6 +251,22 @@ lint_package: cronJob: apiVersion: batch/v1 +"Validate 1.27.5": + extends: .kubeconform + variables: + KUBE_VERSION: "1.27.5" + HELM_SETTINGS: | + global: + ingress: + apiVersion: networking.k8s.io/v1 + pdb: + apiVersion: policy/v1 + hpa: + apiVersion: autoscaling/v2 + batch: + cronJob: + apiVersion: batch/v1 + .review_template: stage: review variables: diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 62948e5cd3..996662fefb 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -36,6 +36,8 @@ # - one to create the environment # - one to stop the environment +# Kubernetes 1.26 + review_gke_k8s_126: extends: .review_app_template variables: @@ -47,3 +49,17 @@ review_gke_k8s_126: stop_review_gke_k8s_126: extends: .stop_review_app_template variables: !reference [review_gke_k8s_126, variables] + +# Kubernetes 1.27 + +review_gke_k8s_127: + extends: .review_app_template + variables: + VCLUSTER_K8S_VERSION: "1.27" + VCLUSTER_NAME: vcluster-1-27-${CI_COMMIT_REF_SLUG} + environment: + on_stop: stop_review_gke_k8s_127 + +stop_review_gke_k8s_127: + extends: .stop_review_app_template + variables: !reference [review_gke_k8s_127, variables] -- GitLab From 37823a1d9c7efd1627e33bca59fd45b553e4cefc Mon Sep 17 00:00:00 2001 From: Clemens Beck Date: Thu, 14 Sep 2023 15:10:41 +0000 Subject: [PATCH 6/8] DRY up HELM_SETTINGS configs --- .gitlab-ci.yml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ca847b638..5447023eb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -239,33 +239,13 @@ lint_package: extends: .kubeconform variables: KUBE_VERSION: "1.26.8" - HELM_SETTINGS: | - global: - ingress: - apiVersion: networking.k8s.io/v1 - pdb: - apiVersion: policy/v1 - hpa: - apiVersion: autoscaling/v2 - batch: - cronJob: - apiVersion: batch/v1 + HELM_SETTINGS: !reference ["Validate 1.25.7", variables, HELM_SETTINGS] "Validate 1.27.5": extends: .kubeconform variables: KUBE_VERSION: "1.27.5" - HELM_SETTINGS: | - global: - ingress: - apiVersion: networking.k8s.io/v1 - pdb: - apiVersion: policy/v1 - hpa: - apiVersion: autoscaling/v2 - batch: - cronJob: - apiVersion: batch/v1 + HELM_SETTINGS: !reference ["Validate 1.25.7", variables, HELM_SETTINGS] .review_template: stage: review -- GitLab From 13466d68f9d53588642fd79641e726deb5eed805 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 14 Sep 2023 10:21:53 -0500 Subject: [PATCH 7/8] Rename vcluster review app jobs Renames the vcluster review app jobs, replacing 'gke_k8s' with 'vcluster' because 'gke' and 'k8s' are a bit irrelevant/redundant. --- .gitlab/ci/review-apps.gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 996662fefb..60ac65e2cc 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -38,28 +38,28 @@ # Kubernetes 1.26 -review_gke_k8s_126: +review_vcluster_126: extends: .review_app_template variables: VCLUSTER_K8S_VERSION: "1.26" VCLUSTER_NAME: vcluster-1-26-${CI_COMMIT_REF_SLUG} environment: - on_stop: stop_review_gke_k8s_126 + on_stop: stop_review_vcluster_126 -stop_review_gke_k8s_126: +stop_review_vcluster_126: extends: .stop_review_app_template - variables: !reference [review_gke_k8s_126, variables] + variables: !reference [review_vcluster_126, variables] # Kubernetes 1.27 -review_gke_k8s_127: +review_vcluster_127: extends: .review_app_template variables: VCLUSTER_K8S_VERSION: "1.27" VCLUSTER_NAME: vcluster-1-27-${CI_COMMIT_REF_SLUG} environment: - on_stop: stop_review_gke_k8s_127 + on_stop: stop_review_vcluster_127 -stop_review_gke_k8s_127: +stop_review_vcluster_127: extends: .stop_review_app_template - variables: !reference [review_gke_k8s_127, variables] + variables: !reference [review_vcluster_127, variables] -- GitLab From a40ff4ed3c127af719c655d81b37b923a316f860 Mon Sep 17 00:00:00 2001 From: Clemens Beck Date: Tue, 19 Sep 2023 09:24:27 +0200 Subject: [PATCH 8/8] Add sections to vcluster review logs --- .gitlab/ci/review-apps.gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/ci/review-apps.gitlab-ci.yml b/.gitlab/ci/review-apps.gitlab-ci.yml index 60ac65e2cc..d3f80b7082 100644 --- a/.gitlab/ci/review-apps.gitlab-ci.yml +++ b/.gitlab/ci/review-apps.gitlab-ci.yml @@ -14,12 +14,18 @@ .review_app_template: extends: .review_app_common script: + - echo -e "\e[0Ksection_start:`date +%s`:connect_vcluster\r\e[0KConnecting to review vcluster" - cluster_connect - kubectl version - vcluster_create - vcluster_run kubectl version + - echo -e "\e[0Ksection_end:`date +%s`:connect_vcluster\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:deploy_app\r\e[0KDeploy GitLab to vcluster" - vcluster_helm_deploy + - echo -e "\e[0Ksection_end:`date +%s`:deploy_app\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:wait_rollout\r\e[0KWait for rollout" - vcluster_helm_rollout_status + - echo -e "\e[0Ksection_end:`date +%s`:wait_rollout\r\e[0K" - vcluster_info .stop_review_app_template: -- GitLab