From 7a23a1bddd1b34bed2a8ab50ac8e7d6053c44929 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:19:53 -0500 Subject: [PATCH 01/12] Add skeleton stage/job for k3s testing --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fe49ff8be..d897cdf8df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,7 @@ variables: STRICT_VERSIONS: "true" stages: + - k3s - prepare - test - preflight @@ -93,6 +94,25 @@ iac-sast: needs: [] before_script: [] +test: + stage: k3s + image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/3.7.1-kube-1.20.11-alpine-3.14 + parallel: + matrix: + - K3S_VERSION: + - v1.24.14-k3s1 + - v1.25.10-k3s1 + - v1.26.5-k3s1 + services: + - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/$K3S_VERSION + alias: k3s + script: + - apk add curl + - curl k3s:8081 > k3s.yaml + - export KUBECONFIG=$(pwd)/k3s.yaml + - kubectl version + - kubectl cluster-info + tag_auto_deploy: stage: staging-release script: -- GitLab From fe69836345d58f64f7590ca32dd7597f8af4f7fe Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:26:12 -0500 Subject: [PATCH 02/12] Apply updates from example reference Pulls in changes to this section from https://gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/-/blob/master/.gitlab-ci.yml?ref_type=heads --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d897cdf8df..df63b144ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,13 +103,17 @@ test: - v1.24.14-k3s1 - v1.25.10-k3s1 - v1.26.5-k3s1 + - v1.27.1-k3s1 + variables: + BUILD_IMAGE_NAME: "$CI_REGISTRY_IMAGE/build/$CI_COMMIT_REF_SLUG:$K3S_VERSION" services: - - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/$K3S_VERSION + - name: $BUILD_IMAGE_NAME alias: k3s script: + - apk upgrade - apk add curl - - curl k3s:8081 > k3s.yaml - - export KUBECONFIG=$(pwd)/k3s.yaml + - curl -f k3s:8081?service=k3s > kubeconfig + - export KUBECONFIG=kubeconfig - kubectl version - kubectl cluster-info -- GitLab From d4c58e28867ca18aed05f0dd9b3ff3878be72981 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:28:04 -0500 Subject: [PATCH 03/12] Revert change to image path calculation As we're in a separate project, we need to hard-code the project path instead of using the variable. --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df63b144ca..253f6b0980 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,10 +104,8 @@ test: - v1.25.10-k3s1 - v1.26.5-k3s1 - v1.27.1-k3s1 - variables: - BUILD_IMAGE_NAME: "$CI_REGISTRY_IMAGE/build/$CI_COMMIT_REF_SLUG:$K3S_VERSION" services: - - name: $BUILD_IMAGE_NAME + - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/$K3S_VERSION alias: k3s script: - apk upgrade -- GitLab From ed81d7a73b4216ea636e0f4f06a97f2ed1021fb4 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:35:34 -0500 Subject: [PATCH 04/12] Add commands to deploy Helm chart --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 253f6b0980..a8cd554f32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,6 +96,7 @@ iac-sast: test: stage: k3s + extends: .source_autodevops image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/3.7.1-kube-1.20.11-alpine-3.14 parallel: matrix: @@ -114,6 +115,10 @@ test: - export KUBECONFIG=kubeconfig - kubectl version - kubectl cluster-info + - ensure_namespace + - create_secret + - deploy + - wait_for_deploy tag_auto_deploy: stage: staging-release -- GitLab From d9f7ba5702cecc8f0581781fda67db275a856c94 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:40:47 -0500 Subject: [PATCH 05/12] Install bash before sourcing autodevops.sh --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8cd554f32..56a443a0d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,6 @@ iac-sast: test: stage: k3s - extends: .source_autodevops image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/3.7.1-kube-1.20.11-alpine-3.14 parallel: matrix: @@ -108,13 +107,15 @@ test: services: - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/$K3S_VERSION alias: k3s - script: + before_script: - apk upgrade - - apk add curl + - apk add bash curl + - source scripts/ci/autodevops.sh - curl -f k3s:8081?service=k3s > kubeconfig - export KUBECONFIG=kubeconfig - kubectl version - kubectl cluster-info + script: - ensure_namespace - create_secret - deploy -- GitLab From 6d11c429d93190aa2e87cb4861b27af1e79cfd3c Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:53:57 -0500 Subject: [PATCH 06/12] Use bash for scripts --- .gitlab-ci.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56a443a0d6..ba3d922fcd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,16 +110,19 @@ test: before_script: - apk upgrade - apk add bash curl - - source scripts/ci/autodevops.sh - - curl -f k3s:8081?service=k3s > kubeconfig - - export KUBECONFIG=kubeconfig - - kubectl version - - kubectl cluster-info script: - - ensure_namespace - - create_secret - - deploy - - wait_for_deploy + - /bin/bash + - -c + - | + source scripts/ci/autodevops.sh + curl -f k3s:8081?service=k3s > kubeconfig + export KUBECONFIG=kubeconfig + kubectl version + kubectl cluster-info + ensure_namespace + create_secret + deploy + wait_for_deploy tag_auto_deploy: stage: staging-release -- GitLab From 33ca55c7c945fe435204055a71e9f582faf6c07c Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 16:57:26 -0500 Subject: [PATCH 07/12] Put commands in their own script --- .gitlab-ci.yml | 13 +------------ scripts/test_in_k3s.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) create mode 100755 scripts/test_in_k3s.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba3d922fcd..c77d9b725a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,18 +111,7 @@ test: - apk upgrade - apk add bash curl script: - - /bin/bash - - -c - - | - source scripts/ci/autodevops.sh - curl -f k3s:8081?service=k3s > kubeconfig - export KUBECONFIG=kubeconfig - kubectl version - kubectl cluster-info - ensure_namespace - create_secret - deploy - wait_for_deploy + - /bin/bash ./scripts/test_in_k3s.sh tag_auto_deploy: stage: staging-release diff --git a/scripts/test_in_k3s.sh b/scripts/test_in_k3s.sh new file mode 100755 index 0000000000..470ed2e47c --- /dev/null +++ b/scripts/test_in_k3s.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +curl -f k3s:8081?service=k3s > kubeconfig +export KUBECONFIG=kubeconfig + +kubectl version +kubectl cluster-info + +source scripts/ci/autodevops.sh +ensure_namespace +create_secret +deploy +wait_for_deploy -- GitLab From f6a164d2075a6630fbad8babe3e78994329b36a4 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 17:27:39 -0500 Subject: [PATCH 08/12] No longer source autodevops.sh --- .gitlab-ci.yml | 13 +++++++++++-- scripts/test_in_k3s.sh | 13 ------------- 2 files changed, 11 insertions(+), 15 deletions(-) delete mode 100755 scripts/test_in_k3s.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c77d9b725a..745daab7dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -109,9 +109,18 @@ test: alias: k3s before_script: - apk upgrade - - apk add bash curl + - apk add curl + - curl -f k3s:8081?service=k3s > kubeconfig + - export KUBECONFIG=kubeconfig + - kubectl version + - kubectl cluster-info script: - - /bin/bash ./scripts/test_in_k3s.sh + - > + helm install gitlab . + -f ./examples/kind/values-base.yaml + -f ./examples/kind/values-ssl.yaml + --set global.hosts.domain=$(hostname -i).nip.io + - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=120s tag_auto_deploy: stage: staging-release diff --git a/scripts/test_in_k3s.sh b/scripts/test_in_k3s.sh deleted file mode 100755 index 470ed2e47c..0000000000 --- a/scripts/test_in_k3s.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -curl -f k3s:8081?service=k3s > kubeconfig -export KUBECONFIG=kubeconfig - -kubectl version -kubectl cluster-info - -source scripts/ci/autodevops.sh -ensure_namespace -create_secret -deploy -wait_for_deploy -- GitLab From 3ecbe35ef39d6bd2655c18a7dcc83c294949619e Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 17:28:55 -0500 Subject: [PATCH 09/12] Run helm dependency update --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 745daab7dc..af0c45a70f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,11 +115,12 @@ test: - kubectl version - kubectl cluster-info script: + - helm dependency update - > helm install gitlab . - -f ./examples/kind/values-base.yaml - -f ./examples/kind/values-ssl.yaml - --set global.hosts.domain=$(hostname -i).nip.io + -f ./examples/kind/values-base.yaml + -f ./examples/kind/values-ssl.yaml + --set global.hosts.domain=$(hostname -i).nip.io - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=120s tag_auto_deploy: -- GitLab From 5cf12a391cd48639a0d1c7a5aa52b828fdf5a447 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 17:37:16 -0500 Subject: [PATCH 10/12] Extend timeout, kubectl get pods --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af0c45a70f..92d10ded19 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,7 +121,9 @@ test: -f ./examples/kind/values-base.yaml -f ./examples/kind/values-ssl.yaml --set global.hosts.domain=$(hostname -i).nip.io - - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=120s + - sleep 10 + - kubectl get pods + - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=180s tag_auto_deploy: stage: staging-release -- GitLab From 020c25438e5c6ece10bef2ca3589b99f324111a1 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 17:42:40 -0500 Subject: [PATCH 11/12] Set timeout to 10 minutes --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92d10ded19..66e307ff93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,7 +123,7 @@ test: --set global.hosts.domain=$(hostname -i).nip.io - sleep 10 - kubectl get pods - - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=180s + - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=600s tag_auto_deploy: stage: staging-release -- GitLab From a1c60ef5def6c6e201195fa7cfae0e2b32fda757 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Wed, 9 Aug 2023 17:52:32 -0500 Subject: [PATCH 12/12] Calculate time to wait for Webservice --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66e307ff93..ab20edcc9f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,7 +123,7 @@ test: --set global.hosts.domain=$(hostname -i).nip.io - sleep 10 - kubectl get pods - - kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=600s + - time kubectl wait pods -l app=webservice,release=gitlab --for condition=Ready --timeout=600s tag_auto_deploy: stage: staging-release -- GitLab