From 64252771aae2dabd3354d41e14bda2c01ae8421e Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Wed, 3 Sep 2025 07:25:56 +0530 Subject: [PATCH] chore: add topology service TLS secret and cell configuration to the migrations and sidekiq deployment template This is needed so when the DB migration job runs, it has: - Cell settings are configured so it knows that the sequence needs to be altered. - TLS secrets so the communication with the topology service can be done See the investigation thread for more detail: https://gitlab.com/gitlab-com/gl-infra/tenant-scale/cells-infrastructure/team/-/issues/479#note_2721187221 We are also adding it to the sidekiq deployment as we would be connecting to the TS eventually from there for [claiming](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/topology_service/#claim-service) Part of: https://gitlab.com/gitlab-com/gl-infra/tenant-scale/cells-infrastructure/team/-/issues/479 --- .../charts/migrations/templates/_jobspec.yaml | 2 + .../migrations/templates/configmap.yaml | 2 + .../charts/sidekiq/templates/configmap.yaml | 1 + .../charts/sidekiq/templates/deployment.yaml | 2 + .../charts/toolbox/templates/configmap.yaml | 10 +--- .../charts/toolbox/templates/deployment.yaml | 21 +------- .../charts/webservice/templates/configmap.yml | 10 +--- .../webservice/templates/deployment.yaml | 21 +------- charts/gitlab/templates/_gitlab.yaml.tpl | 8 ++-- spec/configuration/cell_spec.rb | 22 +++++---- templates/_helpers.tpl | 48 +++++++++++++++++++ 11 files changed, 79 insertions(+), 68 deletions(-) diff --git a/charts/gitlab/charts/migrations/templates/_jobspec.yaml b/charts/gitlab/charts/migrations/templates/_jobspec.yaml index cccea6ee0d..654d2f0125 100644 --- a/charts/gitlab/charts/migrations/templates/_jobspec.yaml +++ b/charts/gitlab/charts/migrations/templates/_jobspec.yaml @@ -109,6 +109,7 @@ spec: - name: migrations-secrets mountPath: /srv/gitlab/config/secrets.yml subPath: rails-secrets/secrets.yml + {{- include "gitlab.topologyService.volumeMounts" (dict "context" $ "secretsVolumeName" "migrations-secrets") | nindent 12 }} - name: migrations-secrets mountPath: /srv/gitlab/config/initial_root_password subPath: migrations/initial_root_password @@ -160,6 +161,7 @@ spec: {{- range $.Values.local.psql }} {{- include "gitlab.psql.secret" . | nindent 10 }} {{- end }} + {{- include "gitlab.topologyService.mountSecrets" $ | nindent 10 }} {{- if include "gitlab.geo.secondary" $ }} - secret: name: {{ template "gitlab.geo.psql.password.secret" . }} diff --git a/charts/gitlab/charts/migrations/templates/configmap.yaml b/charts/gitlab/charts/migrations/templates/configmap.yaml index f34c114cbb..79db88d069 100644 --- a/charts/gitlab/charts/migrations/templates/configmap.yaml +++ b/charts/gitlab/charts/migrations/templates/configmap.yaml @@ -54,10 +54,12 @@ data: {{- include "gitlab.appConfig.repositories" . | nindent 6 }} {{- include "gitlab.appConfig.sidekiq.configuration" $ | nindent 6 }} {{- include "gitlab.appConfig.oidcProvider.configuration" $ | nindent 6 }} + {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.geo.config" $ | nindent 6 }} configure: | {{- include "gitlab.scripts.configure.secrets" (dict "required" "rails-secrets migrations gitaly") | nindent 4 }} {{- include "gitlab.psql.ssl.initScript" . | nindent 4 }} {{- include "gitlab.geo.psql.ssl.initScript" . | nindent 4 }} + {{- include "gitlab.topologyService.configureScript" $ | nindent 4 }} # Leave this here - This line denotes end of block to the parser. {{- end }} diff --git a/charts/gitlab/charts/sidekiq/templates/configmap.yaml b/charts/gitlab/charts/sidekiq/templates/configmap.yaml index 74e470708d..2a22befe4b 100644 --- a/charts/gitlab/charts/sidekiq/templates/configmap.yaml +++ b/charts/gitlab/charts/sidekiq/templates/configmap.yaml @@ -161,5 +161,6 @@ data: {{- end }} {{- include "gitlab.psql.ssl.initScript" . | nindent 4 }} {{- include "gitlab.geo.psql.ssl.initScript" . | nindent 4 }} + {{- include "gitlab.topologyService.configureScript" $ | nindent 4 }} # Leave this here - This line denotes end of block to the parser. {{- end }} diff --git a/charts/gitlab/charts/sidekiq/templates/deployment.yaml b/charts/gitlab/charts/sidekiq/templates/deployment.yaml index 225bb7634d..91dc3a4d0c 100644 --- a/charts/gitlab/charts/sidekiq/templates/deployment.yaml +++ b/charts/gitlab/charts/sidekiq/templates/deployment.yaml @@ -177,6 +177,7 @@ spec: mountPath: /srv/gitlab/config/secrets.yml subPath: rails-secrets/secrets.yml readOnly: true + {{- include "gitlab.topologyService.volumeMounts" (dict "context" $ "secretsVolumeName" "sidekiq-secrets") | nindent 10 }} resources: {{- toYaml $.Values.init.resources | nindent 12 }} {{- include "gitlab.image.pullSecrets" $imageCfg | indent 6 }} @@ -320,6 +321,7 @@ spec: - key: "tls.key" path: "sidekiq-metrics/sidekiq-metrics.key" {{- end }} + {{- include "gitlab.topologyService.mountSecrets" $ | nindent 10 }} {{- include "gitlab.gitaly.clientSecrets" $ | nindent 10 }} {{- if $.Values.global.clickhouse.enabled }} {{- include "gitlab.clickhouse.main.secrets" $ | nindent 10 }} diff --git a/charts/gitlab/charts/toolbox/templates/configmap.yaml b/charts/gitlab/charts/toolbox/templates/configmap.yaml index 158bdfd8b9..48ed392def 100644 --- a/charts/gitlab/charts/toolbox/templates/configmap.yaml +++ b/charts/gitlab/charts/toolbox/templates/configmap.yaml @@ -156,15 +156,7 @@ data: mv "/${secret_dir}/objectstorage/.s3cfg" "/${secret_dir}/.s3cfg" fi {{- end }} - {{- if $.Values.global.appConfig.cell.enabled }} - if [ -d /init-config/topology-service ]; then - mkdir -p /init-secrets/topology-service - {{- if $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - cp -v -L /init-config/topology-service/tls.key /init-secrets/topology-service/tls.key - cp -v -L /init-config/topology-service/tls.crt /init-secrets/topology-service/tls.crt - {{- end }} - fi - {{- end }} + {{- include "gitlab.topologyService.configureScript" $ | nindent 4 }} configure-gsutil: | if [ -e "$GOOGLE_APPLICATION_CREDENTIALS" ]; then # The following script is used to configure gsutil when creating backups diff --git a/charts/gitlab/charts/toolbox/templates/deployment.yaml b/charts/gitlab/charts/toolbox/templates/deployment.yaml index 27444c8cbc..9f0316ddbc 100644 --- a/charts/gitlab/charts/toolbox/templates/deployment.yaml +++ b/charts/gitlab/charts/toolbox/templates/deployment.yaml @@ -156,16 +156,7 @@ spec: - name: toolbox-secrets mountPath: /srv/gitlab/config/secrets.yml subPath: rails-secrets/secrets.yml - {{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - - name: toolbox-secrets - mountPath: /srv/gitlab/config/topology-service/tls.crt - subPath: topology-service/tls.crt - readOnly: true - - name: toolbox-secrets - mountPath: /srv/gitlab/config/topology-service/tls.key - subPath: topology-service/tls.key - readOnly: true - {{- end }} + {{- include "gitlab.topologyService.volumeMounts" (dict "context" $ "secretsVolumeName" "toolbox-secrets") | nindent 12 }} - name: toolbox-tmp mountPath: '/srv/gitlab/tmp' {{- if and .Values.persistence.enabled .Values.persistence.subPath }} @@ -220,15 +211,7 @@ spec: {{- range $.Values.local.psql }} {{- include "gitlab.psql.secret" . | nindent 10 }} {{- end }} - {{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - - secret: - name: {{ template "topology-service.tls.secret" $ }} - items: - - key: "tls.crt" - path: "topology-service/tls.crt" - - key: "tls.key" - path: "topology-service/tls.key" - {{- end }} + {{- include "gitlab.topologyService.mountSecrets" $ | nindent 10 }} {{- if include "gitlab.geo.secondary" $ }} - secret: name: {{ template "gitlab.geo.psql.password.secret" . }} diff --git a/charts/gitlab/charts/webservice/templates/configmap.yml b/charts/gitlab/charts/webservice/templates/configmap.yml index 136f6e7c49..5ef7566926 100644 --- a/charts/gitlab/charts/webservice/templates/configmap.yml +++ b/charts/gitlab/charts/webservice/templates/configmap.yml @@ -183,15 +183,7 @@ data: cp -v -L /init-config/webservice-metrics/webservice-metrics.crt /init-secrets/webservice-metrics/webservice-metrics.crt cp -v -L /init-config/webservice-metrics/webservice-metrics.key /init-secrets/webservice-metrics/webservice-metrics.key {{- end }} - {{- if $.Values.global.appConfig.cell.enabled }} - if [ -d /init-config/topology-service ]; then - mkdir -p /init-secrets/topology-service - {{- if $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - cp -v -L /init-config/topology-service/tls.key /init-secrets/topology-service/tls.key - cp -v -L /init-config/topology-service/tls.crt /init-secrets/topology-service/tls.crt - {{- end }} - fi - {{- end }} + {{- include "gitlab.topologyService.configureScript" $ | nindent 4 }} {{- include "webservice.datamodel.prepare" $ -}} {{/* BEGIN range deployments */}} diff --git a/charts/gitlab/charts/webservice/templates/deployment.yaml b/charts/gitlab/charts/webservice/templates/deployment.yaml index 90e99ed393..5db51df8cf 100644 --- a/charts/gitlab/charts/webservice/templates/deployment.yaml +++ b/charts/gitlab/charts/webservice/templates/deployment.yaml @@ -282,16 +282,7 @@ spec: mountPath: /srv/gitlab/config/puma.key subPath: puma/puma.key {{- end }} - {{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - - name: webservice-secrets - mountPath: /srv/gitlab/config/topology-service/tls.crt - subPath: topology-service/tls.crt - readOnly: true - - name: webservice-secrets - mountPath: /srv/gitlab/config/topology-service/tls.key - subPath: topology-service/tls.key - readOnly: true - {{- end }} + {{- include "gitlab.topologyService.volumeMounts" (dict "context" $ "secretsVolumeName" "webservice-secrets") | nindent 12 }} - name: webservice-config mountPath: '/etc/krb5.conf' subPath: krb5.conf @@ -495,15 +486,7 @@ spec: - key: "tls.key" path: "gitlab-workhorse/tls.key" {{- end }} - {{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} - - secret: - name: {{ template "topology-service.tls.secret" $ }} - items: - - key: "tls.crt" - path: "topology-service/tls.crt" - - key: "tls.key" - path: "topology-service/tls.key" - {{- end }} + {{- include "gitlab.topologyService.mountSecrets" $ | nindent 10 }} {{- if eq (include "webservice-metrics.tls.enabled" $) "true" }} - secret: name: {{ template "webservice-metrics.tls.secret" $ }} diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index 75b188909e..bbf92026d1 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -64,12 +64,12 @@ cell: skip_sequence_alteration: {{ eq .database.skipSequenceAlteration true }} topology_service_client: address: {{ .topologyServiceClient.address | quote }} - {{- if .topologyServiceClient.tls.enabled }} + tls: + enabled: {{ .topologyServiceClient.tls.enabled }} + {{- if .topologyServiceClient.tls.enabled }} private_key_file: "/srv/gitlab/config/topology-service/tls.key" certificate_file: "/srv/gitlab/config/topology-service/tls.crt" - tls: - enabled: true - {{- end }} + {{- end }} {{- end -}} {{- end -}} {{- end -}} diff --git a/spec/configuration/cell_spec.rb b/spec/configuration/cell_spec.rb index e1ffa80d80..4da410b378 100644 --- a/spec/configuration/cell_spec.rb +++ b/spec/configuration/cell_spec.rb @@ -6,7 +6,7 @@ require 'helm_template_helper' require 'yaml' describe 'cells configuration' do - let(:charts) { %w[webservice sidekiq toolbox] } + let(:charts) { %w[migrations webservice sidekiq toolbox] } let(:default_values) do HelmTemplate.defaults end @@ -35,7 +35,10 @@ describe 'cells configuration' do 'skipSequenceAlteration' => false }, 'topologyServiceClient' => { - 'address' => 'topology-service.gitlab.example.com:443' + 'address' => 'topology-service.gitlab.example.com:443', + 'tls' => { + 'enabled' => false + } } } } @@ -48,14 +51,17 @@ describe 'cells configuration' do it 'generates cell configuration in the gitlab.yml file' do expected_values = { - "enabled" => true, - "id" => 1, - "database" => + 'enabled' => true, + 'id' => 1, + 'database' => { - "skip_sequence_alteration" => false + 'skip_sequence_alteration' => false }, - "topology_service_client" => { - "address" => "topology-service.gitlab.example.com:443" + 'topology_service_client' => { + 'address' => 'topology-service.gitlab.example.com:443', + 'tls' => { + 'enabled' => false + } } } diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index d70b28960b..54ca27c7eb 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -700,3 +700,51 @@ Return the Topology Service TLS Secret name {{- define "topology-service.tls.secret" -}} {{- default (printf "%s-topology-service-tls" .Release.Name) $.Values.global.appConfig.cell.topologyServiceClient.tls.secret | quote -}} {{- end -}} + +{{/* +Mount topology service TLS secrets in projected volume sources +Usage: {{ include "gitlab.topologyService.mountSecrets" $ | nindent 10 }} +*/}} +{{- define "gitlab.topologyService.mountSecrets" -}} +{{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} +- secret: + name: {{ template "topology-service.tls.secret" $ }} + items: + - key: "tls.crt" + path: "topology-service/tls.crt" + - key: "tls.key" + path: "topology-service/tls.key" +{{- end }} +{{- end -}} + +{{/* +Volume mounts for topology service TLS files +Usage: {{ include "gitlab.topologyService.volumeMounts" (dict "context" $ "secretsVolumeName" "webservice-secrets") | nindent 12 }} +*/}} +{{- define "gitlab.topologyService.volumeMounts" -}} +{{- $context := .context -}} +{{- if and $context.Values.global.appConfig.cell.enabled $context.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} +- name: {{ .secretsVolumeName }} + mountPath: /srv/gitlab/config/topology-service/tls.crt + subPath: topology-service/tls.crt + readOnly: true +- name: {{ .secretsVolumeName }} + mountPath: /srv/gitlab/config/topology-service/tls.key + subPath: topology-service/tls.key + readOnly: true +{{- end }} +{{- end -}} + +{{/* +Configure script for topology service TLS secrets +Usage: {{ include "gitlab.topologyService.configureScript" $ | nindent 4 }} +*/}} +{{- define "gitlab.topologyService.configureScript" -}} +{{- if and $.Values.global.appConfig.cell.enabled $.Values.global.appConfig.cell.topologyServiceClient.tls.enabled }} + if [ -d /init-config/topology-service ]; then + mkdir -p /init-secrets/topology-service + cp -v -L /init-config/topology-service/tls.key /init-secrets/topology-service/tls.key + cp -v -L /init-config/topology-service/tls.crt /init-secrets/topology-service/tls.crt + fi +{{- end }} +{{- end -}} -- GitLab