From 2235f20a3eca094bf7bbaff1be5c9d6c4f19ccac Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Fri, 12 Sep 2025 16:53:10 -0300 Subject: [PATCH 1/8] Adds database traffic capture config Related https://gitlab.com/gitlab-org/gitlab/-/merge_requests/202140 --- .../charts/sidekiq/templates/configmap.yaml | 1 + .../charts/toolbox/templates/configmap.yaml | 1 + .../charts/webservice/templates/configmap.yml | 1 + charts/gitlab/templates/_gitlab.yaml.tpl | 13 ++++ .../database_traffic_capture_spec.rb | 70 +++++++++++++++++++ values.yaml | 14 +++- 6 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 spec/configuration/database_traffic_capture_spec.rb diff --git a/charts/gitlab/charts/sidekiq/templates/configmap.yaml b/charts/gitlab/charts/sidekiq/templates/configmap.yaml index f66058e37f..3f1ae43873 100644 --- a/charts/gitlab/charts/sidekiq/templates/configmap.yaml +++ b/charts/gitlab/charts/sidekiq/templates/configmap.yaml @@ -126,6 +126,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} + {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/charts/toolbox/templates/configmap.yaml b/charts/gitlab/charts/toolbox/templates/configmap.yaml index 492b3e978a..647d0731b1 100644 --- a/charts/gitlab/charts/toolbox/templates/configmap.yaml +++ b/charts/gitlab/charts/toolbox/templates/configmap.yaml @@ -106,6 +106,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} + {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/charts/webservice/templates/configmap.yml b/charts/gitlab/charts/webservice/templates/configmap.yml index fb269bd8e8..5420d56235 100644 --- a/charts/gitlab/charts/webservice/templates/configmap.yml +++ b/charts/gitlab/charts/webservice/templates/configmap.yml @@ -138,6 +138,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} + {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index 85eb13ba4c..67b1bb2bd1 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -62,6 +62,19 @@ workspaces: {{- end -}} {{- end -}} +{{- define "gitlab.appConfig.database_traffic_capture" -}} +{{- with .Values.global.appConfig.databaseTrafficCapture -}} +database_traffic_capture: + config: + storage: + connector: + provider: {{ .config.storage.connector.provider | quote }} + project_id: {{ .config.storage.connector.projectId | quote }} + credentials: {{ .config.storage.connector.credentials | quote }} + bucket: {{ .config.storage.connector.bucket | quote }} +{{- end -}} +{{- end -}} + {{- define "gitlab.appConfig.cell" -}} {{- if eq .Values.global.appConfig.cell.enabled true -}} {{- with .Values.global.appConfig.cell -}} diff --git a/spec/configuration/database_traffic_capture_spec.rb b/spec/configuration/database_traffic_capture_spec.rb new file mode 100644 index 0000000000..3163334c96 --- /dev/null +++ b/spec/configuration/database_traffic_capture_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'hash_deep_merge' +require 'helm_template_helper' +require 'yaml' + +describe 'database traffic capture configuration' do + let(:charts) { %w[webservice sidekiq toolbox] } + let(:default_values) { HelmTemplate.defaults } + + context 'when no configuration is set' do + let(:helm_template) { HelmTemplate.new(default_values) } + + it 'generates no database traffic capture configuration in the gitlab.yml file' do + charts.each do |chart| + expect(gitlab_yml_database_traffic_capture(chart)).to eq(nil) + end + end + end + + context 'when custom configuration is set' do + let(:helm_template) { HelmTemplate.new(cell_values.deep_merge!(default_values)) } + let(:cell_values) do + { + 'global' => { + 'appConfig' => { + 'databaseTrafficCapture' => { + 'config' => { + 'storage' => { + 'connector' => { + 'provider' => 'provider-name', + 'projectId' => 'project-id', + 'credentials' => 'path/to/credentials.json', + 'bucket' => 'bucket-name', + } + } + } + } + } + } + } + end + + it 'generates cell configuration in the gitlab.yml file' do + expected_values = { + 'config' => { + 'storage' => { + 'connector' => { + 'provider' => 'provider-name', + 'project_id' => 'project-id', + 'credentials' => 'path/to/credentials.json', + 'bucket' => 'bucket-name', + } + } + } + } + + charts.each do |chart| + expect(gitlab_yml_database_traffic_capture(chart)).to eq(expected_values) + end + end + end + + def gitlab_yml_database_traffic_capture(chart) + YAML.safe_load( + helm_template.resources_by_kind('ConfigMap')["ConfigMap/test-#{chart}"]['data']['gitlab.yml.erb'] + )['production']['database_traffic_capture'] + end +end diff --git a/values.yaml b/values.yaml index dd5b09649a..01ddf49abb 100644 --- a/values.yaml +++ b/values.yaml @@ -513,6 +513,16 @@ global: workspaces: {} + ## Configure Database Traffic Capture. This is a Gitlab internal-use experiment. + database_traffic_capture: + config: + storage: + connector: + provider: "" + project_id: "" + credentials: "" + bucket: "" + ## Configure GitLab Cells. Cells is a GitLab internal-use experiment. ## https://docs.gitlab.com/administration/cells/ cell: @@ -686,9 +696,9 @@ global: notificationSecret: {} # secret: # key: - + ## Container Registry database configuration - database: + database: password: {} # user: registry # name: registry -- GitLab From e8708bdac6e072bea62b95a665db6cbc859d482c Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Fri, 12 Sep 2025 17:54:52 -0300 Subject: [PATCH 2/8] Fix Rubocop offenses --- spec/configuration/database_traffic_capture_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/configuration/database_traffic_capture_spec.rb b/spec/configuration/database_traffic_capture_spec.rb index 3163334c96..853f1d2dfb 100644 --- a/spec/configuration/database_traffic_capture_spec.rb +++ b/spec/configuration/database_traffic_capture_spec.rb @@ -14,7 +14,7 @@ describe 'database traffic capture configuration' do it 'generates no database traffic capture configuration in the gitlab.yml file' do charts.each do |chart| - expect(gitlab_yml_database_traffic_capture(chart)).to eq(nil) + expect(gitlab_yml_database_traffic_capture(chart)).to be_nil end end end @@ -32,7 +32,7 @@ describe 'database traffic capture configuration' do 'provider' => 'provider-name', 'projectId' => 'project-id', 'credentials' => 'path/to/credentials.json', - 'bucket' => 'bucket-name', + 'bucket' => 'bucket-name' } } } @@ -50,7 +50,7 @@ describe 'database traffic capture configuration' do 'provider' => 'provider-name', 'project_id' => 'project-id', 'credentials' => 'path/to/credentials.json', - 'bucket' => 'bucket-name', + 'bucket' => 'bucket-name' } } } -- GitLab From 9bdcd192c070a99c6a9d8d3a8a30556c7073cddc Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Mon, 15 Sep 2025 17:18:26 -0300 Subject: [PATCH 3/8] Undo rubymine formatting --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 01ddf49abb..7bd6e5784b 100644 --- a/values.yaml +++ b/values.yaml @@ -510,7 +510,7 @@ global: # externalUrl: # internalUrl: # clientTimeoutSeconds: - + workspaces: {} ## Configure Database Traffic Capture. This is a Gitlab internal-use experiment. -- GitLab From 2496fef24ff0542e274b780958a47a90dee71b07 Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Mon, 15 Sep 2025 17:38:34 -0300 Subject: [PATCH 4/8] Update test names --- spec/configuration/database_traffic_capture_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/configuration/database_traffic_capture_spec.rb b/spec/configuration/database_traffic_capture_spec.rb index 853f1d2dfb..9c898af4fb 100644 --- a/spec/configuration/database_traffic_capture_spec.rb +++ b/spec/configuration/database_traffic_capture_spec.rb @@ -20,8 +20,8 @@ describe 'database traffic capture configuration' do end context 'when custom configuration is set' do - let(:helm_template) { HelmTemplate.new(cell_values.deep_merge!(default_values)) } - let(:cell_values) do + let(:helm_template) { HelmTemplate.new(database_traffic_capture_values.deep_merge!(default_values)) } + let(:database_traffic_capture_values) do { 'global' => { 'appConfig' => { @@ -42,7 +42,7 @@ describe 'database traffic capture configuration' do } end - it 'generates cell configuration in the gitlab.yml file' do + it 'generates database traffic capture configuration in the gitlab.yml file' do expected_values = { 'config' => { 'storage' => { -- GitLab From 04b7877dee3c4cc4562869fd08a62ed6cfb7c4a7 Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Thu, 18 Sep 2025 10:49:49 -0300 Subject: [PATCH 5/8] Update values to camelCase --- charts/gitlab/templates/_gitlab.yaml.tpl | 2 ++ values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index 67b1bb2bd1..c67822b922 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -64,6 +64,7 @@ workspaces: {{- define "gitlab.appConfig.database_traffic_capture" -}} {{- with .Values.global.appConfig.databaseTrafficCapture -}} +{{- if and .config.storage.connector.provider (ne .config.storage.connector.provider "") -}} database_traffic_capture: config: storage: @@ -74,6 +75,7 @@ database_traffic_capture: bucket: {{ .config.storage.connector.bucket | quote }} {{- end -}} {{- end -}} +{{- end -}} {{- define "gitlab.appConfig.cell" -}} {{- if eq .Values.global.appConfig.cell.enabled true -}} diff --git a/values.yaml b/values.yaml index 7bd6e5784b..fa230a2fc1 100644 --- a/values.yaml +++ b/values.yaml @@ -514,12 +514,12 @@ global: workspaces: {} ## Configure Database Traffic Capture. This is a Gitlab internal-use experiment. - database_traffic_capture: + databaseTrafficCapture: config: storage: connector: provider: "" - project_id: "" + projectId: "" credentials: "" bucket: "" -- GitLab From 4bdcaba1afde3a1b0f1e5df792303c2bb2c12e90 Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Fri, 19 Sep 2025 15:41:05 -0300 Subject: [PATCH 6/8] Update all function names to camelCase --- charts/gitlab/charts/sidekiq/templates/configmap.yaml | 2 +- charts/gitlab/charts/toolbox/templates/configmap.yaml | 2 +- charts/gitlab/charts/webservice/templates/configmap.yml | 2 +- charts/gitlab/templates/_gitlab.yaml.tpl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/gitlab/charts/sidekiq/templates/configmap.yaml b/charts/gitlab/charts/sidekiq/templates/configmap.yaml index 3f1ae43873..b5b8ac21dc 100644 --- a/charts/gitlab/charts/sidekiq/templates/configmap.yaml +++ b/charts/gitlab/charts/sidekiq/templates/configmap.yaml @@ -126,7 +126,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} - {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} + {{- include "gitlab.appConfig.databaseTrafficCapture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/charts/toolbox/templates/configmap.yaml b/charts/gitlab/charts/toolbox/templates/configmap.yaml index 647d0731b1..76c9b70c3c 100644 --- a/charts/gitlab/charts/toolbox/templates/configmap.yaml +++ b/charts/gitlab/charts/toolbox/templates/configmap.yaml @@ -106,7 +106,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} - {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} + {{- include "gitlab.appConfig.databaseTrafficCapture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/charts/webservice/templates/configmap.yml b/charts/gitlab/charts/webservice/templates/configmap.yml index 5420d56235..aa89f757d7 100644 --- a/charts/gitlab/charts/webservice/templates/configmap.yml +++ b/charts/gitlab/charts/webservice/templates/configmap.yml @@ -138,7 +138,7 @@ data: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) {{- include "gitlab.appConfig.kas" . | nindent 6 }} {{- include "gitlab.appConfig.workspaces" . | nindent 6 }} - {{- include "gitlab.appConfig.database_traffic_capture" . | nindent 6 }} + {{- include "gitlab.appConfig.databaseTrafficCapture" . | nindent 6 }} {{- include "gitlab.appConfig.cell" . | nindent 6 }} {{- include "gitlab.appConfig.suggested_reviewers" . | nindent 6 }} {{- include "gitlab.appConfig.zoekt" . | nindent 6 }} diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index c67822b922..4cbb2764b2 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -62,7 +62,7 @@ workspaces: {{- end -}} {{- end -}} -{{- define "gitlab.appConfig.database_traffic_capture" -}} +{{- define "gitlab.appConfig.databaseTrafficCapture" -}} {{- with .Values.global.appConfig.databaseTrafficCapture -}} {{- if and .config.storage.connector.provider (ne .config.storage.connector.provider "") -}} database_traffic_capture: -- GitLab From f89385b5a7fcf1e2739ec3daf273c07a1b072056 Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Thu, 25 Sep 2025 16:33:45 -0300 Subject: [PATCH 7/8] Properly safeguards the db traffic config --- charts/gitlab/templates/_gitlab.yaml.tpl | 15 ++--- .../database_traffic_capture_spec.rb | 61 +++++++++++++++++++ 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index 4cbb2764b2..3ab87da16f 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -63,17 +63,18 @@ workspaces: {{- end -}} {{- define "gitlab.appConfig.databaseTrafficCapture" -}} -{{- with .Values.global.appConfig.databaseTrafficCapture -}} -{{- if and .config.storage.connector.provider (ne .config.storage.connector.provider "") -}} +{{- $databaseTrafficCapture := .Values.global.appConfig.databaseTrafficCapture | default dict -}} +{{- $connector := dig "config" "storage" "connector" dict $databaseTrafficCapture -}} +{{- $provider := $connector.provider | default "" -}} +{{- if and $provider (ne $provider "") -}} database_traffic_capture: config: storage: connector: - provider: {{ .config.storage.connector.provider | quote }} - project_id: {{ .config.storage.connector.projectId | quote }} - credentials: {{ .config.storage.connector.credentials | quote }} - bucket: {{ .config.storage.connector.bucket | quote }} -{{- end -}} + provider: {{ $provider | quote }} + project_id: {{ $connector.projectId | default "" | quote }} + credentials: {{ $connector.credentials | default "" | quote }} + bucket: {{ $connector.bucket | default "" | quote }} {{- end -}} {{- end -}} diff --git a/spec/configuration/database_traffic_capture_spec.rb b/spec/configuration/database_traffic_capture_spec.rb index 9c898af4fb..61018a0906 100644 --- a/spec/configuration/database_traffic_capture_spec.rb +++ b/spec/configuration/database_traffic_capture_spec.rb @@ -19,6 +19,67 @@ describe 'database traffic capture configuration' do end end + context 'when partial configuration is set' do + let(:helm_template) { HelmTemplate.new(database_traffic_capture_values.deep_merge!(default_values)) } + let(:database_traffic_capture_values) do + { + 'global' => { + 'appConfig' => { + 'databaseTrafficCapture' => { + 'config' => {} + } + } + } + } + end + + it 'generates no database traffic capture configuration in the gitlab.yml file' do + charts.each do |chart| + expect(gitlab_yml_database_traffic_capture(chart)).to be_nil + end + end + end + + context 'when partial configuration with the provider name is set' do + let(:helm_template) { HelmTemplate.new(database_traffic_capture_values.deep_merge!(default_values)) } + let(:database_traffic_capture_values) do + { + 'global' => { + 'appConfig' => { + 'databaseTrafficCapture' => { + 'config' => { + 'storage' => { + 'connector' => { + 'provider' => 'provider' + } + } + } + } + } + } + } + end + + it 'generates database traffic capture configuration in the gitlab.yml file with defaults' do + expected_values = { + 'config' => { + 'storage' => { + 'connector' => { + 'provider' => 'provider', + 'project_id' => '', + 'credentials' => '', + 'bucket' => '' + } + } + } + } + + charts.each do |chart| + expect(gitlab_yml_database_traffic_capture(chart)).to eq(expected_values) + end + end + end + context 'when custom configuration is set' do let(:helm_template) { HelmTemplate.new(database_traffic_capture_values.deep_merge!(default_values)) } let(:database_traffic_capture_values) do -- GitLab From 0220f07c61a72186d7ab3fe2f28dd0faedf38e64 Mon Sep 17 00:00:00 2001 From: Leonardo Rosa Date: Mon, 29 Sep 2025 16:21:46 -0300 Subject: [PATCH 8/8] Apply reviewer suggestions --- charts/gitlab/templates/_gitlab.yaml.tpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/gitlab/templates/_gitlab.yaml.tpl b/charts/gitlab/templates/_gitlab.yaml.tpl index 3ab87da16f..15b3fab136 100644 --- a/charts/gitlab/templates/_gitlab.yaml.tpl +++ b/charts/gitlab/templates/_gitlab.yaml.tpl @@ -63,18 +63,18 @@ workspaces: {{- end -}} {{- define "gitlab.appConfig.databaseTrafficCapture" -}} -{{- $databaseTrafficCapture := .Values.global.appConfig.databaseTrafficCapture | default dict -}} -{{- $connector := dig "config" "storage" "connector" dict $databaseTrafficCapture -}} -{{- $provider := $connector.provider | default "" -}} -{{- if and $provider (ne $provider "") -}} +{{- with .Values.global.appConfig.databaseTrafficCapture -}} +{{- $connector := dig "config" "storage" "connector" (dict "provider" "") . -}} +{{- if ne $connector.provider "" -}} database_traffic_capture: config: storage: connector: - provider: {{ $provider | quote }} - project_id: {{ $connector.projectId | default "" | quote }} - credentials: {{ $connector.credentials | default "" | quote }} - bucket: {{ $connector.bucket | default "" | quote }} + provider: {{ $connector.provider | quote }} + project_id: {{ $connector.projectId | quote }} + credentials: {{ $connector.credentials | quote }} + bucket: {{ $connector.bucket | quote }} +{{- end -}} {{- end -}} {{- end -}} -- GitLab