From f6188508887977229e714e5beaac4c2c522b1fe3 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 12:20:20 -0700
Subject: [PATCH 01/15] Add multiarch tuning variable

---
 scripts/ci/arm_nodeselectors.yaml | 14 ++++++++++++++
 scripts/ci/autodevops.sh          |  6 +++++-
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 scripts/ci/arm_nodeselectors.yaml

diff --git a/scripts/ci/arm_nodeselectors.yaml b/scripts/ci/arm_nodeselectors.yaml
new file mode 100644
index 0000000000..e88e65b465
--- /dev/null
+++ b/scripts/ci/arm_nodeselectors.yaml
@@ -0,0 +1,14 @@
+---
+
+global:
+  nodeSelector:
+    kubernetes.io/arch: arm64
+
+minio:
+  nodeSelector:
+    kubernetes.io/arch: amd64
+
+nginx-ingress:
+  nodeSelector:
+    kubernetes.io/arch: amd64
+
diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh
index a50bde0b96..cef67a73cf 100644
--- a/scripts/ci/autodevops.sh
+++ b/scripts/ci/autodevops.sh
@@ -154,10 +154,14 @@ CIYAML
 
     SENTRY_CONFIGURATION="-f ci.sentry.yaml"
   fi
-
+  MULTIARCH_CONFIGURATION=""
+  if [ "${DEPLOY_MULTIARCH}" == "true" ]; then
+    MULTIARCH_CONFIGURATION="-f scripts/ci/arm_nodeselectors.yaml"
+  fi
   helm upgrade --install \
     $WAIT \
     ${SENTRY_CONFIGURATION} \
+    ${MULTIARCH_CONFIGURATION} \
     -f ci.details.yaml \
     -f ci.scale.yaml \
     -f ci.psql.yaml \
-- 
GitLab


From 8c7c0b3afc98f25c581d519c01b3d81f9892dc14 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 13:57:15 -0700
Subject: [PATCH 02/15] Add multiarch testing jobs

---
 .gitlab-ci.yml | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 40d6952030..6d1217ec7a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -332,6 +332,14 @@ pin_image_versions:
   KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129.helm-charts.win"
   KUBE_NAMESPACE: "helm-charts-win"
 
+.review_gke129a_common_vars: &review_gke129a_common_vars
+  DNS_PROVIDER: "google"
+  AGENT_NAME: "gke129-ci-cluster"
+  KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129a.helm-charts.win"
+  KUBE_NAMESPACE: "helm-charts-win"
+  DEPLOY_MULTIARCH: "true"
+  
+
 .review_eks_common_vars: &review_eks_common_vars
   DNS_PROVIDER: "aws"
   AGENT_NAME: "eks129-ci-cluster"
@@ -405,6 +413,32 @@ review_gke129:
     auto_stop_in: "${REVIEW_APPS_AUTO_STOP_IN}"
     action: access
 
+create_review_gke129a:
+  stage: review
+  image: busybox
+  variables:
+    <<: *review_gke129a_common_vars
+    GIT_STRATEGY: none
+  script:
+    - echo "Configuring gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
+  environment:
+    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_review_gke129a
+    auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN
+    action: start
+
+review_gke129a:
+  variables:
+    <<: *review_gke129a_common_vars
+  extends: .review_template
+  environment:
+    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_review_gke129a
+    auto_stop_in: "${REVIEW_APPS_AUTO_STOP_IN}"
+    action: access
+
 # See comments in create_review_gke127.
 create_review_eks:
   stage: review
@@ -465,6 +499,14 @@ stop_review_gke129:
     name: gke129_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     action: stop
 
+stop_review_gke129a:
+  variables:
+    <<:  *review_gke129a_common_vars
+  extends: .stop_review_template
+  environment:
+    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    action: stop
+
 stop_review_eks:
   variables:
     <<: *review_eks_common_vars
@@ -656,6 +698,22 @@ review_specs_gke129:
     - !reference [.specs, rules]
   needs: ['review_gke129']
 
+review_specs_gke129a:
+  extends: .specs
+  variables:
+    <<: *review_gke129a_common_vars
+    VARIABLES_FILE: "variables/review_gke129a"
+    RSPEC_TAGS: type:feature
+  environment:
+    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_review_gke129a
+    action: access
+  rules:
+    - !reference [.rule:skip_if_no_cluster]
+    - !reference [.specs, rules]
+  needs: ['review_gke129a']
+
 review_specs_eks:
   extends: .specs
   variables:
-- 
GitLab


From 2cf327b6be6705f2f906795797efa657d0385431 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 14:20:11 -0700
Subject: [PATCH 03/15] Split single- and multiarch deployments in the same
 cluster

---
 scripts/ci/autodevops.sh | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh
index cef67a73cf..fc7d773237 100644
--- a/scripts/ci/autodevops.sh
+++ b/scripts/ci/autodevops.sh
@@ -8,15 +8,26 @@ export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
 
 # Derive the Helm RELEASE argument from CI_ENVIRONMENT_SLUG
 if [[ $CI_ENVIRONMENT_SLUG =~ ^[^-]+-review ]]; then
+  # if multiarch deployment is on - we will be deploying *two*
+  # charts - one for "amd64" and second for "arm64" thus the need
+  # to avoid name collision:
+  if [ "${DEPLOY_MULTIARCH}" == "tue" ]; then
+    RELEASE_NAME="rvw-a-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}"
+  else
+    RELEASE_NAME=rvw-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}
+  fi
   # if a "review", use $REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
-  RELEASE_NAME=rvw-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}
   # Trim release name to leave room for prefixes/suffixes
   RELEASE_NAME=${RELEASE_NAME:0:30}
   # Trim any hyphens in the suffix
   RELEASE_NAME=${RELEASE_NAME%-}
 else
   # otherwise, use CI_ENVIRONMENT_SLUG
-  RELEASE_NAME=$CI_ENVIRONMENT_SLUG
+  if [ "${DEPLOY_MULTIARCH}" == "tue" ]; then
+    RELEASE_NAME="a-${CI_ENVIRONMENT_SLUG}"
+  else
+    RELEASE_NAME=$CI_ENVIRONMENT_SLUG
+  fi
 fi
 export RELEASE_NAME
 
-- 
GitLab


From 8be64b4713685ec51f762d97cacd753c128b9561 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 14:42:48 -0700
Subject: [PATCH 04/15] Modify environment name to work proper with KAS

---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d1217ec7a..ec48b61a99 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -422,7 +422,7 @@ create_review_gke129a:
   script:
     - echo "Configuring gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
   environment:
-    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
     on_stop: stop_review_gke129a
     auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN
@@ -433,7 +433,7 @@ review_gke129a:
     <<: *review_gke129a_common_vars
   extends: .review_template
   environment:
-    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
     on_stop: stop_review_gke129a
     auto_stop_in: "${REVIEW_APPS_AUTO_STOP_IN}"
@@ -504,7 +504,7 @@ stop_review_gke129a:
     <<:  *review_gke129a_common_vars
   extends: .stop_review_template
   environment:
-    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     action: stop
 
 stop_review_eks:
@@ -705,7 +705,7 @@ review_specs_gke129a:
     VARIABLES_FILE: "variables/review_gke129a"
     RSPEC_TAGS: type:feature
   environment:
-    name: gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
     on_stop: stop_review_gke129a
     action: access
-- 
GitLab


From 167d810b7dc2f38d784bab5f0a60c38e59656162 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 15:37:46 -0700
Subject: [PATCH 05/15] Add more visibility to new changes

---
 .gitlab-ci.yml           | 2 +-
 scripts/ci/autodevops.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ec48b61a99..0f40d123d8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -420,7 +420,7 @@ create_review_gke129a:
     <<: *review_gke129a_common_vars
     GIT_STRATEGY: none
   script:
-    - echo "Configuring gke129a_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
+    - echo "Configuring gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
   environment:
     name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh
index fc7d773237..ad1798f2d6 100644
--- a/scripts/ci/autodevops.sh
+++ b/scripts/ci/autodevops.sh
@@ -54,6 +54,7 @@ function previousDeployFailed() {
 }
 
 function deploy() {
+  echo "DEPLOY_MULTIARCH: $DEPLOY_MULTIARCH"
   # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
   if [ "$RELEASE_NAME" != "production" ] && previousDeployFailed ; then
     echo "Deployment in bad state, cleaning up $RELEASE_NAME"
-- 
GitLab


From cda75224359c46371e18b4dcd9c7715d1f0e775b Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 16 Sep 2024 15:58:34 -0700
Subject: [PATCH 06/15] Fix typo in conditional

---
 scripts/ci/autodevops.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/autodevops.sh b/scripts/ci/autodevops.sh
index ad1798f2d6..804203ea06 100644
--- a/scripts/ci/autodevops.sh
+++ b/scripts/ci/autodevops.sh
@@ -11,7 +11,7 @@ if [[ $CI_ENVIRONMENT_SLUG =~ ^[^-]+-review ]]; then
   # if multiarch deployment is on - we will be deploying *two*
   # charts - one for "amd64" and second for "arm64" thus the need
   # to avoid name collision:
-  if [ "${DEPLOY_MULTIARCH}" == "tue" ]; then
+  if [ "${DEPLOY_MULTIARCH}" == "true" ]; then
     RELEASE_NAME="rvw-a-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}"
   else
     RELEASE_NAME=rvw-${REVIEW_REF_PREFIX}${CI_COMMIT_SHORT_SHA}
@@ -23,7 +23,7 @@ if [[ $CI_ENVIRONMENT_SLUG =~ ^[^-]+-review ]]; then
   RELEASE_NAME=${RELEASE_NAME%-}
 else
   # otherwise, use CI_ENVIRONMENT_SLUG
-  if [ "${DEPLOY_MULTIARCH}" == "tue" ]; then
+  if [ "${DEPLOY_MULTIARCH}" == "true" ]; then
     RELEASE_NAME="a-${CI_ENVIRONMENT_SLUG}"
   else
     RELEASE_NAME=$CI_ENVIRONMENT_SLUG
-- 
GitLab


From 9c6162fcb175fdf680617b25bcda0dbbbf9cfccc Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Tue, 17 Sep 2024 08:53:25 -0700
Subject: [PATCH 07/15] Optimize and minimize changes required

---
 .gitlab-ci.yml | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f40d123d8..e6a1088944 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -335,7 +335,7 @@ pin_image_versions:
 .review_gke129a_common_vars: &review_gke129a_common_vars
   DNS_PROVIDER: "google"
   AGENT_NAME: "gke129-ci-cluster"
-  KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129a.helm-charts.win"
+  KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129.helm-charts.win"
   KUBE_NAMESPACE: "helm-charts-win"
   DEPLOY_MULTIARCH: "true"
   
@@ -421,8 +421,8 @@ create_review_gke129a:
     GIT_STRATEGY: none
   script:
     - echo "Configuring gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
-  environment:
-    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+  environment: &gke129a_environment
+    name: &gke129a_name gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
     on_stop: stop_review_gke129a
     auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN
@@ -433,10 +433,7 @@ review_gke129a:
     <<: *review_gke129a_common_vars
   extends: .review_template
   environment:
-    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
-    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke129a
-    auto_stop_in: "${REVIEW_APPS_AUTO_STOP_IN}"
+    <<: *gke129a_environment
     action: access
 
 # See comments in create_review_gke127.
@@ -504,7 +501,8 @@ stop_review_gke129a:
     <<:  *review_gke129a_common_vars
   extends: .stop_review_template
   environment:
-    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    # name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: *gke129a_name
     action: stop
 
 stop_review_eks:
@@ -705,9 +703,7 @@ review_specs_gke129a:
     VARIABLES_FILE: "variables/review_gke129a"
     RSPEC_TAGS: type:feature
   environment:
-    name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
-    url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke129a
+    <<: *gke129a_environment
     action: access
   rules:
     - !reference [.rule:skip_if_no_cluster]
-- 
GitLab


From 82e1302c774f4801354e6122f6045f3d6a166982 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Tue, 17 Sep 2024 09:38:57 -0700
Subject: [PATCH 08/15] Force arch on required charts

---
 scripts/ci/arm_nodeselectors.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/ci/arm_nodeselectors.yaml b/scripts/ci/arm_nodeselectors.yaml
index e88e65b465..56bb42094d 100644
--- a/scripts/ci/arm_nodeselectors.yaml
+++ b/scripts/ci/arm_nodeselectors.yaml
@@ -8,6 +8,14 @@ minio:
   nodeSelector:
     kubernetes.io/arch: amd64
 
+gitlab-runner:
+  nodeSelector:
+    kubernetes.io/arch: arm64
+
+postgresql:
+  nodeSelector:
+    kubernetes.io/arch: arm64
+
 nginx-ingress:
   nodeSelector:
     kubernetes.io/arch: amd64
-- 
GitLab


From 0cf427874ed101b70919f3bcf8106f75317f64a9 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Tue, 17 Sep 2024 11:09:55 -0700
Subject: [PATCH 09/15] Fix platform selection for required charts

---
 scripts/ci/arm_nodeselectors.yaml | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/arm_nodeselectors.yaml b/scripts/ci/arm_nodeselectors.yaml
index 56bb42094d..9092e339ac 100644
--- a/scripts/ci/arm_nodeselectors.yaml
+++ b/scripts/ci/arm_nodeselectors.yaml
@@ -13,8 +13,25 @@ gitlab-runner:
     kubernetes.io/arch: arm64
 
 postgresql:
-  nodeSelector:
-    kubernetes.io/arch: arm64
+  primary:
+    nodeSelector:
+      kubernetes.io/arch: arm64
+  readReplicas:
+    nodeSelector:
+      kubernetes.io/arch: arm64
+  backup.cronjob.labels:
+    nodeSelector:
+      kubernetes.io/arch: arm64
+
+
+redis:
+  master:
+    nodeSelector:
+      kubernetes.io/arch: arm64
+  replica:
+    nodeSelector:
+      kubernetes.io/arch: arm64
+
 
 nginx-ingress:
   nodeSelector:
-- 
GitLab


From 74b29b4790cf9980626d9c089e13ba6f44174c30 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Tue, 17 Sep 2024 15:07:49 -0700
Subject: [PATCH 10/15] Remove redis arm64

---
 scripts/ci/arm_nodeselectors.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/ci/arm_nodeselectors.yaml b/scripts/ci/arm_nodeselectors.yaml
index 9092e339ac..9ab65e6313 100644
--- a/scripts/ci/arm_nodeselectors.yaml
+++ b/scripts/ci/arm_nodeselectors.yaml
@@ -23,14 +23,14 @@ postgresql:
     nodeSelector:
       kubernetes.io/arch: arm64
 
-
-redis:
-  master:
-    nodeSelector:
-      kubernetes.io/arch: arm64
-  replica:
-    nodeSelector:
-      kubernetes.io/arch: arm64
+# Redis fails with arm64
+# redis:
+#   master:
+#     nodeSelector:
+#       kubernetes.io/arch: arm64
+#   replica:
+#     nodeSelector:
+#       kubernetes.io/arch: arm64
 
 
 nginx-ingress:
-- 
GitLab


From e62e8078f6e75ae1dd3558c9f6c91fa98c4fc790 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Tue, 17 Sep 2024 15:45:48 -0700
Subject: [PATCH 11/15] Refactor entire platform into it's own file

---
 .gitlab-ci.yml                         | 55 +----------------
 .gitlab/ci/environments/gke129-arm.yml | 82 ++++++++++++++++++++++++++
 2 files changed, 83 insertions(+), 54 deletions(-)
 create mode 100644 .gitlab/ci/environments/gke129-arm.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e6a1088944..e2c03db4d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -88,6 +88,7 @@ include:
   - local: '/.gitlab/ci/rules.gitlab-ci.yml'
   - local: '/.gitlab/ci/review-apps.gitlab-ci.yml'
   - local: '/.gitlab/ci/operator.gitlab-ci.yml'
+  - local: '/.gitlab/ci/environments/gke129-arm.yml'
   - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml
   - template: Jobs/Secret-Detection.latest.gitlab-ci.yml
   - template: Jobs/SAST.latest.gitlab-ci.yml
@@ -332,14 +333,6 @@ pin_image_versions:
   KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129.helm-charts.win"
   KUBE_NAMESPACE: "helm-charts-win"
 
-.review_gke129a_common_vars: &review_gke129a_common_vars
-  DNS_PROVIDER: "google"
-  AGENT_NAME: "gke129-ci-cluster"
-  KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129.helm-charts.win"
-  KUBE_NAMESPACE: "helm-charts-win"
-  DEPLOY_MULTIARCH: "true"
-  
-
 .review_eks_common_vars: &review_eks_common_vars
   DNS_PROVIDER: "aws"
   AGENT_NAME: "eks129-ci-cluster"
@@ -413,29 +406,6 @@ review_gke129:
     auto_stop_in: "${REVIEW_APPS_AUTO_STOP_IN}"
     action: access
 
-create_review_gke129a:
-  stage: review
-  image: busybox
-  variables:
-    <<: *review_gke129a_common_vars
-    GIT_STRATEGY: none
-  script:
-    - echo "Configuring gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
-  environment: &gke129a_environment
-    name: &gke129a_name gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
-    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke129a
-    auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN
-    action: start
-
-review_gke129a:
-  variables:
-    <<: *review_gke129a_common_vars
-  extends: .review_template
-  environment:
-    <<: *gke129a_environment
-    action: access
-
 # See comments in create_review_gke127.
 create_review_eks:
   stage: review
@@ -496,15 +466,6 @@ stop_review_gke129:
     name: gke129_review/$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     action: stop
 
-stop_review_gke129a:
-  variables:
-    <<:  *review_gke129a_common_vars
-  extends: .stop_review_template
-  environment:
-    # name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
-    name: *gke129a_name
-    action: stop
-
 stop_review_eks:
   variables:
     <<: *review_eks_common_vars
@@ -696,20 +657,6 @@ review_specs_gke129:
     - !reference [.specs, rules]
   needs: ['review_gke129']
 
-review_specs_gke129a:
-  extends: .specs
-  variables:
-    <<: *review_gke129a_common_vars
-    VARIABLES_FILE: "variables/review_gke129a"
-    RSPEC_TAGS: type:feature
-  environment:
-    <<: *gke129a_environment
-    action: access
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['review_gke129a']
-
 review_specs_eks:
   extends: .specs
   variables:
diff --git a/.gitlab/ci/environments/gke129-arm.yml b/.gitlab/ci/environments/gke129-arm.yml
new file mode 100644
index 0000000000..885a6b7d83
--- /dev/null
+++ b/.gitlab/ci/environments/gke129-arm.yml
@@ -0,0 +1,82 @@
+
+.review_gke129a_common_vars: &review_gke129a_common_vars
+  DNS_PROVIDER: "google"
+  AGENT_NAME: "gke129-ci-cluster"
+  KUBE_INGRESS_BASE_DOMAIN: "cloud-native-v129.helm-charts.win"
+  KUBE_NAMESPACE: "helm-charts-win"
+  DEPLOY_MULTIARCH: "true"
+
+create_review_gke129a:
+  stage: review
+  image: busybox
+  variables:
+    <<: *review_gke129a_common_vars
+    GIT_STRATEGY: none
+  script:
+    - echo "Configuring gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA to automatically stop in $REVIEW_APPS_AUTO_STOP_IN."
+  environment: &gke129a_environment
+    name: &gke129a_name gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_review_gke129a
+    auto_stop_in: $REVIEW_APPS_AUTO_STOP_IN
+    action: start
+
+review_gke129a:
+  variables:
+    <<: *review_gke129a_common_vars
+  extends: .review_template
+  environment:
+    <<: *gke129a_environment
+    action: access
+
+stop_review_gke129a:
+  variables:
+    <<:  *review_gke129a_common_vars
+  extends: .stop_review_template
+  environment:
+    # name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
+    name: *gke129a_name
+    action: stop
+
+review_specs_gke129a:
+  extends: .specs
+  variables:
+    <<: *review_gke129a_common_vars
+    VARIABLES_FILE: "variables/review_gke129a"
+    RSPEC_TAGS: type:feature
+  environment:
+    <<: *gke129a_environment
+    action: access
+  rules:
+    - !reference [.rule:skip_if_no_cluster]
+    - !reference [.specs, rules]
+  needs: ['review_gke129a']
+
+qa_gke129a:
+  extends: .qa_branch
+  variables:
+    VARIABLES_FILE: "variables/review_gke129a"
+    TEST_SUITE: $QA_SANITY_SUITE_OPTIONS
+  environment:
+    <<: *gke129a_environment
+    action: access
+  needs:
+    - review_gke129a
+    - job: review_specs_gke129a
+      artifacts: false
+
+qa_gke129a_full_suite_manual_trigger:
+  extends: .qa_full_suite_manual_trigger
+  script: echo 'The job will trigger Full E2E suite against GKE129'
+
+qa_gke129a_manual_full_suite:
+  extends: .qa_branch
+  parallel: 7
+  variables:
+    VARIABLES_FILE: "variables/review_gke129a"
+    TEST_SUITE: $QA_FULL_SUITE_OPTIONS
+  environment:
+    <<: *gke129a_environment
+    action: access
+  needs: [ "qa_gke129a_full_suite_manual_trigger"]
+
-- 
GitLab


From b5e56a9fb347b1836fc4396c8779216e4f488440 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Fri, 27 Sep 2024 14:34:25 -0700
Subject: [PATCH 12/15] Run arm64 tests only on branches where we do build
 arm64 images

---
 .gitlab-ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e2c03db4d0..876d9b63cf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,6 +89,11 @@ include:
   - local: '/.gitlab/ci/review-apps.gitlab-ci.yml'
   - local: '/.gitlab/ci/operator.gitlab-ci.yml'
   - local: '/.gitlab/ci/environments/gke129-arm.yml'
+    rules:
+      # CANONICAL_DEFAULT_BRANCH_PIPELINE
+      - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/charts/gitlab" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      # CANONICAL_MR_PIPELINE
+      - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/charts/gitlab" && $CI_PIPELINE_SOURCE == "merge_request_event"'
   - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml
   - template: Jobs/Secret-Detection.latest.gitlab-ci.yml
   - template: Jobs/SAST.latest.gitlab-ci.yml
-- 
GitLab


From 01ae9e1226d77953722bf7b6b7d20511b595a092 Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Thu, 3 Oct 2024 11:13:43 -0700
Subject: [PATCH 13/15] Limit scope to MRs and Main branch, excluding stable
 etc.

---
 .gitlab/ci/environments/gke129-arm.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitlab/ci/environments/gke129-arm.yml b/.gitlab/ci/environments/gke129-arm.yml
index 885a6b7d83..317a49adf7 100644
--- a/.gitlab/ci/environments/gke129-arm.yml
+++ b/.gitlab/ci/environments/gke129-arm.yml
@@ -28,6 +28,12 @@ review_gke129a:
   environment:
     <<: *gke129a_environment
     action: access
+  rules:
+    # Initially we only execute for MRs and Default branch as the only ones
+    # with multiarch images
+    - if: '$PIPELINE_TYPE != "CANONICAL_MR_PIPELINE" && $PIPELINE_TYPE != "CANONICAL_DEFAULT_BRANCH_PIPELINE"'
+      when: never
+    - !reference [.review_template, rules]
 
 stop_review_gke129a:
   variables:
-- 
GitLab


From 82e4bfcd3df492b32bea761a904a3eb4379f4fee Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Thu, 3 Oct 2024 14:46:16 -0700
Subject: [PATCH 14/15] Allow arm64 testing failures as a first iteration

---
 .gitlab/ci/environments/gke129-arm.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.gitlab/ci/environments/gke129-arm.yml b/.gitlab/ci/environments/gke129-arm.yml
index 317a49adf7..6ec39ca321 100644
--- a/.gitlab/ci/environments/gke129-arm.yml
+++ b/.gitlab/ci/environments/gke129-arm.yml
@@ -28,6 +28,8 @@ review_gke129a:
   environment:
     <<: *gke129a_environment
     action: access
+  # initially allow arm64 failures 
+  allow_failure: true
   rules:
     # Initially we only execute for MRs and Default branch as the only ones
     # with multiarch images
@@ -39,6 +41,8 @@ stop_review_gke129a:
   variables:
     <<:  *review_gke129a_common_vars
   extends: .stop_review_template
+  # initially allow arm64 failures 
+  allow_failure: true
   environment:
     # name: gke129_review/a-$REVIEW_REF_PREFIX$CI_COMMIT_SHORT_SHA
     name: *gke129a_name
@@ -50,6 +54,8 @@ review_specs_gke129a:
     <<: *review_gke129a_common_vars
     VARIABLES_FILE: "variables/review_gke129a"
     RSPEC_TAGS: type:feature
+  # initially allow arm64 failures 
+  allow_failure: true
   environment:
     <<: *gke129a_environment
     action: access
@@ -63,6 +69,8 @@ qa_gke129a:
   variables:
     VARIABLES_FILE: "variables/review_gke129a"
     TEST_SUITE: $QA_SANITY_SUITE_OPTIONS
+  # initially allow arm64 failures 
+  allow_failure: true
   environment:
     <<: *gke129a_environment
     action: access
@@ -81,6 +89,8 @@ qa_gke129a_manual_full_suite:
   variables:
     VARIABLES_FILE: "variables/review_gke129a"
     TEST_SUITE: $QA_FULL_SUITE_OPTIONS
+  # initially allow arm64 failures 
+  allow_failure: true
   environment:
     <<: *gke129a_environment
     action: access
-- 
GitLab


From 9178c6ed5c25635951ee83b788ca9d642b7cdaee Mon Sep 17 00:00:00 2001
From: Dmitry Makovey <dmakovey@gitlab.com>
Date: Mon, 21 Oct 2024 08:16:08 -0700
Subject: [PATCH 15/15] Rename file as per @jondoveston request

---
 .gitlab-ci.yml                                                  | 2 +-
 .../environments/{gke129-arm.yml => gke129-arm.gitlab-ci.yml}   | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .gitlab/ci/environments/{gke129-arm.yml => gke129-arm.gitlab-ci.yml} (100%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 876d9b63cf..bcc4bff273 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -88,7 +88,7 @@ include:
   - local: '/.gitlab/ci/rules.gitlab-ci.yml'
   - local: '/.gitlab/ci/review-apps.gitlab-ci.yml'
   - local: '/.gitlab/ci/operator.gitlab-ci.yml'
-  - local: '/.gitlab/ci/environments/gke129-arm.yml'
+  - local: '/.gitlab/ci/environments/gke129-arm.gitlab-ci.yml'
     rules:
       # CANONICAL_DEFAULT_BRANCH_PIPELINE
       - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/charts/gitlab" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
diff --git a/.gitlab/ci/environments/gke129-arm.yml b/.gitlab/ci/environments/gke129-arm.gitlab-ci.yml
similarity index 100%
rename from .gitlab/ci/environments/gke129-arm.yml
rename to .gitlab/ci/environments/gke129-arm.gitlab-ci.yml
-- 
GitLab