From 318b7db75211a31c4265601d653505e960304792 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 09:28:07 +0000 Subject: [PATCH 01/26] Add feature flag configuration With this Feature flag it should be possible to fetch secrets from several AWS Services in GitLab CI Changelog: added EE: true --- .../feature_flags/development/aws_secret_manager.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 config/feature_flags/development/aws_secret_manager.yml diff --git a/config/feature_flags/development/aws_secret_manager.yml b/config/feature_flags/development/aws_secret_manager.yml new file mode 100644 index 00000000000000..c328bce43279ab --- /dev/null +++ b/config/feature_flags/development/aws_secret_manager.yml @@ -0,0 +1,10 @@ +# config/feature_flags/development/aws_secret_manager.yml + +--- +name: aws_secret_manager +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/YOUR_MR_ID +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/542277 +milestone: '18.1' +type: development +group: group::pipeline security +default_enabled: false -- GitLab From 7fbd9d34b3d723db53290ab564ca3a13d5c552a5 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 09:31:46 +0000 Subject: [PATCH 02/26] Fix MR ID --- config/feature_flags/development/aws_secret_manager.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/development/aws_secret_manager.yml b/config/feature_flags/development/aws_secret_manager.yml index c328bce43279ab..5be0f4b6f75c25 100644 --- a/config/feature_flags/development/aws_secret_manager.yml +++ b/config/feature_flags/development/aws_secret_manager.yml @@ -2,7 +2,7 @@ --- name: aws_secret_manager -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/YOUR_MR_ID +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/191761 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/542277 milestone: '18.1' type: development -- GitLab From b625d0cbd5937ab92ae66e153c850387588a6ecd Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 09:39:09 +0000 Subject: [PATCH 03/26] remove comments --- config/feature_flags/development/aws_secret_manager.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/feature_flags/development/aws_secret_manager.yml b/config/feature_flags/development/aws_secret_manager.yml index 5be0f4b6f75c25..e4144a3f4dd790 100644 --- a/config/feature_flags/development/aws_secret_manager.yml +++ b/config/feature_flags/development/aws_secret_manager.yml @@ -1,5 +1,3 @@ -# config/feature_flags/development/aws_secret_manager.yml - --- name: aws_secret_manager introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/191761 -- GitLab From 2f95c229bf14f41fdf4a595536d4918429043d72 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 09:48:15 +0000 Subject: [PATCH 04/26] Fix move to the ee folder --- config/feature_flags/development/aws_secret_manager.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 config/feature_flags/development/aws_secret_manager.yml diff --git a/config/feature_flags/development/aws_secret_manager.yml b/config/feature_flags/development/aws_secret_manager.yml deleted file mode 100644 index e4144a3f4dd790..00000000000000 --- a/config/feature_flags/development/aws_secret_manager.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: aws_secret_manager -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/191761 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/542277 -milestone: '18.1' -type: development -group: group::pipeline security -default_enabled: false -- GitLab From 1961f9e709c50276a23333b2f287fccb630cc1da Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 11:17:12 +0000 Subject: [PATCH 05/26] Move file to ee folder --- .../feature_flags/development/aws_secret_manager.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ee/config/feature_flags/development/aws_secret_manager.yml diff --git a/ee/config/feature_flags/development/aws_secret_manager.yml b/ee/config/feature_flags/development/aws_secret_manager.yml new file mode 100644 index 00000000000000..05e1c0e7688d12 --- /dev/null +++ b/ee/config/feature_flags/development/aws_secret_manager.yml @@ -0,0 +1,9 @@ +--- +name: aws_secret_manager +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/191761 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/542277 +feature_issue_url: https://gitlab.com/groups/gitlab-org/-/epics/17822 +milestone: '18.1' +type: development +group: group::pipeline security +default_enabled: false -- GitLab From 3a3cd21d13c7739e8dc6f0b97252b6d2be121d4a Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 12:14:29 +0000 Subject: [PATCH 06/26] Add Secretsmanager Files --- .../entry/aws_secrets_manager/secret.rb | 53 +++++++++++ ee/lib/gitlab/ci/config/entry/secret.rb | 8 +- .../lib/gitlab/ci/config/entry/secret_spec.rb | 89 +++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb new file mode 100644 index 00000000000000..4ff6cbb65e11bb --- /dev/null +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + module AWSSecretsManager + class Secret < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[secret_id region version_id version_stage].freeze + + attributes ALLOWED_KEYS + + validations do + # Required fields + validates :secret_id, type: String, presence: true + validates :region, type: String, presence: true + + # Optional fields + validates :version_id, type: String, allow_nil: true + validates :version_stage, type: String, allow_nil: true + end + + entry :secret_id, Entry::String, + description: 'Secret ID of the secret in AWS Secrets Manager.' + + entry :region, Entry::String, + description: 'AWS region where the secret is stored.' + + entry :version_id, Entry::String, + description: 'Specific version ID of the secret to retrieve.', + inherit: false + + entry :version_stage, Entry::String, + description: 'Stage of the secret version to retrieve (e.g., AWSCURRENT).', + inherit: false + + def value + { + secret_id: secret_id, + version: version_id, + version_stage: version_stage, + region: region + } + end + end + end + end + end + end +end diff --git a/ee/lib/gitlab/ci/config/entry/secret.rb b/ee/lib/gitlab/ci/config/entry/secret.rb index bf91e5b33b9ead..de3df59c9af0cf 100644 --- a/ee/lib/gitlab/ci/config/entry/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/secret.rb @@ -11,12 +11,15 @@ class Secret < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager token].freeze - SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager].freeze + ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager token + aws_secrets_manager].freeze + SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager + aws_secrets_manager].freeze attributes ALLOWED_KEYS entry :vault, Entry::Vault::Secret, description: 'Vault secrets engine configuration' + entry :aws_secrets_manager, Entry::AWSSecretsManager::Secret, description: 'AWS engine configuration' entry :file, ::Gitlab::Config::Entry::Boolean, description: 'Should the created variable be of file type' entry :azure_key_vault, Entry::AzureKeyVault::Secret, description: 'Azure Key Vault configuration' entry :gcp_secret_manager, Entry::GcpSecretManager::Secret, description: 'GCP Secrets Manager configuration' @@ -37,6 +40,7 @@ def value { vault: vault_value, gitlab_secrets_manager: gitlab_secrets_manager_value, + aws_secrets_manager: aws_secrets_manager_value, gcp_secret_manager: gcp_secret_manager_value, azure_key_vault: azure_key_vault_value, akeyless: akeyless_value, diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index e9035502c65700..6d29ec4497dd15 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -90,6 +90,95 @@ end end + context 'for AWS Secrets Manager' do + context 'when `token` is defined' do + let(:config) do + { + aws_secrets_manager: { + secret_id: 'name', + region: 'eu-central-1' + }, + token: '$TEST_ID_TOKEN' + } + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'with optional fields' do + let(:config) do + { + secret_id: 'db-password', + region: 'us-east-1', + version_id: 'abcdef1234567890', + version_stage: 'AWSCURRENT' + } + end + + it 'is valid' do + expect(secret).to be_valid + end + end + + context 'with invalid configuration' do + context 'when secret_id is missing' do + let(:config) do + { region: 'us-east-1' } + end + + it 'is not valid' do + expect(secret).not_to be_valid + expect(secret.errors[:aws_secret_manager]).to include(/secret_id is missing/) + end + end + + context 'when region is missing' do + let(:config) do + { secret_id: 'db-password' } + end + + it 'is not valid' do + expect(secret).not_to be_valid + expect(secret.errors[:aws_secret_manager]).to include(/region is missing/) + end + end + end + + context 'when `token` is not defined' do + let(:config) do + { + aws_secrets_manager: { + secret_id: 'name', + region: 'eu-central-1' + } + } + end + + describe '#value' do + it 'returns secret configuration' do + expect(entry.value).to eq( + { + aws_secrets_manager: { + secret_id: 'name', + region: 'eu-central-1' + } + } + ) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + end + context 'for Azure Key Vault' do context 'when `token` is defined' do let(:config) do -- GitLab From 3baf50ad449398e67e5863da0848a2bc58ea73cf Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 13:08:35 +0000 Subject: [PATCH 07/26] Fix Test Cases --- .../entry/aws_secrets_manager/secret.rb | 18 ++-------- ee/lib/gitlab/ci/config/entry/secret.rb | 2 +- .../lib/gitlab/ci/config/entry/secret_spec.rb | 36 +++++++++++-------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index 4ff6cbb65e11bb..21125c883e653f 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -4,7 +4,7 @@ module Gitlab module Ci class Config module Entry - module AWSSecretsManager + module AwsSecretsManager class Secret < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable @@ -23,24 +23,10 @@ class Secret < ::Gitlab::Config::Entry::Node validates :version_stage, type: String, allow_nil: true end - entry :secret_id, Entry::String, - description: 'Secret ID of the secret in AWS Secrets Manager.' - - entry :region, Entry::String, - description: 'AWS region where the secret is stored.' - - entry :version_id, Entry::String, - description: 'Specific version ID of the secret to retrieve.', - inherit: false - - entry :version_stage, Entry::String, - description: 'Stage of the secret version to retrieve (e.g., AWSCURRENT).', - inherit: false - def value { secret_id: secret_id, - version: version_id, + version_id: version_id, version_stage: version_stage, region: region } diff --git a/ee/lib/gitlab/ci/config/entry/secret.rb b/ee/lib/gitlab/ci/config/entry/secret.rb index de3df59c9af0cf..ef015982a6985e 100644 --- a/ee/lib/gitlab/ci/config/entry/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/secret.rb @@ -19,7 +19,7 @@ class Secret < ::Gitlab::Config::Entry::Node attributes ALLOWED_KEYS entry :vault, Entry::Vault::Secret, description: 'Vault secrets engine configuration' - entry :aws_secrets_manager, Entry::AWSSecretsManager::Secret, description: 'AWS engine configuration' + entry :aws_secrets_manager, Entry::AwsSecretsManager::Secret, description: 'AWS engine configuration' entry :file, ::Gitlab::Config::Entry::Boolean, description: 'Should the created variable be of file type' entry :azure_key_vault, Entry::AzureKeyVault::Secret, description: 'Azure Key Vault configuration' entry :gcp_secret_manager, Entry::GcpSecretManager::Secret, description: 'GCP Secrets Manager configuration' diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index 6d29ec4497dd15..ef52202f423004 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -112,38 +112,44 @@ context 'with optional fields' do let(:config) do { - secret_id: 'db-password', - region: 'us-east-1', - version_id: 'abcdef1234567890', - version_stage: 'AWSCURRENT' + aws_secrets_manager: { + secret_id: 'db-password', + region: 'us-east-1', + version_id: 'abcdef1234567890', + version_stage: 'AWSCURRENT' + } } end it 'is valid' do - expect(secret).to be_valid + expect(entry).to be_valid end end context 'with invalid configuration' do context 'when secret_id is missing' do let(:config) do - { region: 'us-east-1' } + { + aws_secrets_manager: { region: 'us-east-1' } + } end it 'is not valid' do - expect(secret).not_to be_valid - expect(secret.errors[:aws_secret_manager]).to include(/secret_id is missing/) + expect(entry).not_to be_valid + expect(entry.errors).to include(/aws_secrets_manager secret can't be blank/) end end context 'when region is missing' do let(:config) do - { secret_id: 'db-password' } + { + aws_secrets_manager: { secret_id: 'db_password' } + } end it 'is not valid' do - expect(secret).not_to be_valid - expect(secret.errors[:aws_secret_manager]).to include(/region is missing/) + expect(entry).not_to be_valid + expect(entry.errors).to include(/aws_secrets_manager region can't be blank/) end end end @@ -164,7 +170,9 @@ { aws_secrets_manager: { secret_id: 'name', - region: 'eu-central-1' + region: 'eu-central-1', + version_id: nil, + version_stage: nil } } ) @@ -438,7 +446,7 @@ it 'reports error' do expect(entry.errors) .to include 'secret config must use exactly one of these keys: ' \ - 'vault, azure_key_vault, gcp_secret_manager, akeyless, gitlab_secrets_manager' + 'vault, azure_key_vault, gcp_secret_manager, akeyless, gitlab_secrets_manager, aws_secrets_manager' end end @@ -449,7 +457,7 @@ it 'reports error' do expect(entry.errors) .to include "secret config must use exactly one of these keys: " \ - "#{Gitlab::Ci::Config::Entry::Secret::SUPPORTED_PROVIDERS.join(', ')}" + "#{Gitlab::Ci::Config::Entry::Secret::SUPPORTED_PROVIDERS.join(', ')}" end end end -- GitLab From 7b1511da8d03db5c114f9bd8d90a5e9d09c5ef31 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Mon, 19 May 2025 13:09:55 +0000 Subject: [PATCH 08/26] Add Gitlab CI config entry for AWS SSM Parameterstore --- .../entry/aws_ssm_parameter_store/secret.rb | 33 +++++++++ ee/lib/gitlab/ci/config/entry/secret.rb | 9 ++- .../lib/gitlab/ci/config/entry/secret_spec.rb | 68 ++++++++++++++++++- 3 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb diff --git a/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb new file mode 100644 index 00000000000000..066d413fcd58ba --- /dev/null +++ b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + module AwsSsmParameterStore + class Secret < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[name region with_decryption].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :name, type: String, presence: true + validates :region, type: String, presence: true + end + + def value + { + name: name, + region: region + } + end + end + end + end + end + end +end diff --git a/ee/lib/gitlab/ci/config/entry/secret.rb b/ee/lib/gitlab/ci/config/entry/secret.rb index bf91e5b33b9ead..247044703ff108 100644 --- a/ee/lib/gitlab/ci/config/entry/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/secret.rb @@ -11,8 +11,10 @@ class Secret < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager token].freeze - SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager].freeze + ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager token + aws_ssm_parameter_store].freeze + SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager + aws_ssm_parameter_store].freeze attributes ALLOWED_KEYS @@ -23,6 +25,8 @@ class Secret < ::Gitlab::Config::Entry::Node entry :akeyless, Entry::Akeyless::Secret, description: 'Akeyless Key Vault configuration' entry :gitlab_secrets_manager, Entry::GitlabSecretsManager::Secret, description: 'Gitlab Secrets Manager configuration' + entry :aws_ssm_parameter_store, Entry::AwsSsmParameterStore::Secret, + description: 'AWS SSM Parameter Store configuration' validations do validates :config, allowed_keys: ALLOWED_KEYS, only_one_of_keys: SUPPORTED_PROVIDERS @@ -40,6 +44,7 @@ def value gcp_secret_manager: gcp_secret_manager_value, azure_key_vault: azure_key_vault_value, akeyless: akeyless_value, + aws_ssm_parameter_store: aws_ssm_parameter_store_value, file: file_value, token: token }.compact diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index e9035502c65700..9f79d0af200a67 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -329,6 +329,70 @@ end end end + + context 'for AWS SSM ParameterStore' do + context 'when `token` is defined' do + let(:config) do + { + aws_ssm_parameter_store: { + name: 'name', + region: 'us-east-1' + }, + token: '$TEST_ID_TOKEN' + } + end + + describe '#value' do + it 'returns secret configuration' do + expect(entry.value).to eq( + { + aws_ssm_parameter_store: { + name: 'name', + region: 'us-east-1' + }, + token: '$TEST_ID_TOKEN' + } + ) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when `token` is not defined' do + let(:config) do + { + aws_ssm_parameter_store: { + name: 'name', + region: 'us-east-1' + } + } + end + + describe '#value' do + it 'returns secret configuration' do + expect(entry.value).to eq( + { + aws_ssm_parameter_store: { + name: 'name', + region: 'us-east-1' + } + } + ) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + end end end @@ -349,7 +413,7 @@ it 'reports error' do expect(entry.errors) .to include 'secret config must use exactly one of these keys: ' \ - 'vault, azure_key_vault, gcp_secret_manager, akeyless, gitlab_secrets_manager' + 'vault, azure_key_vault, gcp_secret_manager, akeyless, gitlab_secrets_manager, aws_ssm_parameter_store' end end @@ -360,7 +424,7 @@ it 'reports error' do expect(entry.errors) .to include "secret config must use exactly one of these keys: " \ - "#{Gitlab::Ci::Config::Entry::Secret::SUPPORTED_PROVIDERS.join(', ')}" + "#{Gitlab::Ci::Config::Entry::Secret::SUPPORTED_PROVIDERS.join(', ')}" end end end -- GitLab From 8050562f4d88f80c9cf5c4893f8e5219028b9439 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 13:26:59 +0000 Subject: [PATCH 09/26] Update CI Schema --- app/assets/javascripts/editor/schema/ci.json | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 3fee6762423fe6..0be4a6f5677c98 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1028,6 +1028,28 @@ ], "additionalProperties": false }, + "aws_secrets_manager": { + "type": "object", + "properties": { + "secret_id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "version_id": { + "type": "string" + }, + "version_stage": { + "type": "string" + } + }, + "required": [ + "secret_id", + "region" + ], + "additionalProperties": false + }, "akeyless": { "type": "object", "properties": { -- GitLab From d41ddb27e4c44c0e64e93d93019e2808df73be93 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Mon, 19 May 2025 13:39:03 +0000 Subject: [PATCH 10/26] Add AWS SSM Parameter Store support to CI schema and tests --- app/assets/javascripts/editor/schema/ci.json | 21 +++++++++++++++++++ .../ci/yaml_tests/negative_tests/secrets.yml | 15 +++++++++++++ .../ci/yaml_tests/positive_tests/secrets.yml | 11 ++++++++++ 3 files changed, 47 insertions(+) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 3fee6762423fe6..e49e80fac2f134 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1057,6 +1057,22 @@ "token": { "type": "string", "description": "Specifies the JWT variable that should be used to authenticate with the secret provider." + }, + "aws_ssm_parameter_store": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "required": [ + "name", + "region" + ], + "additionalProperties": false } }, "anyOf": [ @@ -1079,6 +1095,11 @@ "required": [ "akeyless" ] + }, + { + "required": [ + "aws_ssm_parameter_store" + ] } ], "dependencies": { diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml index 23d667eeeffaef..aa54d7d8f56ccc 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml @@ -89,3 +89,18 @@ job_with_gcp_secret_manager_secret_without_token: gcp_secret_manager: name: my-secret +job_with_secrets_with_invalid_aws_ssm_parameter_store_property: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_ssm_parameter_store: + invalid: TEST + +job_with_secrets_with_missing_required_aws_ssm_parameter_store_property: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_ssm_parameter_store: + name: test-db-password \ No newline at end of file diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index e615fa52dc5e36..5489329c782afd 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -72,3 +72,14 @@ valid_job_with_gcp_secret_manager_name_and_string_version: name: 'test' version: 'latest' token: $TEST_TOKEN + +valid_job_with_secrets_with_every_aws_ssm_parameter_store_keyword: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_ssm_parameter_store: + name: test-db-password + region: us-east-1 + file: true + token: $TEST_TOKEN \ No newline at end of file -- GitLab From 6bc006c8008ea2ad35c28d4a1e18380415124426 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 13:41:28 +0000 Subject: [PATCH 11/26] Update CI Schema and Add Tests --- app/assets/javascripts/editor/schema/ci.json | 5 +++ .../ci/yaml_tests/negative_tests/secrets.yml | 16 +++++++ .../ci/yaml_tests/positive_tests/secrets.yml | 44 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 0be4a6f5677c98..ac7f9310705262 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1097,6 +1097,11 @@ "gcp_secret_manager" ] }, + { + "required": [ + "aws_secrets_manager" + ] + }, { "required": [ "akeyless" diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml index 23d667eeeffaef..e866e57acee9d5 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml @@ -89,3 +89,19 @@ job_with_gcp_secret_manager_secret_without_token: gcp_secret_manager: name: my-secret +job_with_aws_secrets_manager_secret_without_secret_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + region: "us-east-1" + +job_with_aws_secrets_manager_secret_without_region: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: "database_password" + diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index e615fa52dc5e36..3b786317ea9ae1 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -72,3 +72,47 @@ valid_job_with_gcp_secret_manager_name_and_string_version: name: 'test' version: 'latest' token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: 'us-east-1' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id_and_version_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + version_id: "2" + region: 'us-east-1' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id_and_version_stage: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: "us-east-1" + version_stage: 'AWSCURRENT' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_with_every_keyword: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: "us-east-1" + version_stage: 'AWSCURRENT' + version_id: "2" + token: $TEST_TOKEN -- GitLab From 1b252318347df5f074d0f919e878fc99094b70de Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 19 May 2025 14:55:08 +0000 Subject: [PATCH 12/26] Add calls to runner --- app/assets/javascripts/editor/schema/ci.json | 6 +----- ee/app/models/ci/secrets/integration.rb | 5 +++++ .../presenters/ee/ci/build_runner_presenter.rb | 17 +++++++++++++++++ .../config/entry/aws_secrets_manager/secret.rb | 2 +- .../lib/gitlab/ci/config/entry/secret_spec.rb | 18 ++---------------- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index ac7f9310705262..60afbfd86fbfe2 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1034,9 +1034,6 @@ "secret_id": { "type": "string" }, - "region": { - "type": "string" - }, "version_id": { "type": "string" }, @@ -1045,8 +1042,7 @@ } }, "required": [ - "secret_id", - "region" + "secret_id" ], "additionalProperties": false }, diff --git a/ee/app/models/ci/secrets/integration.rb b/ee/app/models/ci/secrets/integration.rb index 981bfa211c3c2d..6c7c8724cfd03b 100644 --- a/ee/app/models/ci/secrets/integration.rb +++ b/ee/app/models/ci/secrets/integration.rb @@ -4,6 +4,7 @@ module Ci module Secrets class Integration PROVIDER_TYPE_MAP = { + "aws_secrets_manager" => :aws_secrets_manager, "azure_key_vault" => :azure_key_vault, "akeyless" => :akeyless, "gcp_secret_manager" => :gcp_secret_manager, @@ -60,6 +61,10 @@ def akeyless? variables['AKEYLESS_ACCESS_ID']&.value.present? end + def aws_secrets_manager? + variables['AWS_REGION']&.value.present? + end + def gitlab_secrets_manager? # TODO: figure out context for whether GitLab Secrets Manager is # globally enabled on this instance. diff --git a/ee/app/presenters/ee/ci/build_runner_presenter.rb b/ee/app/presenters/ee/ci/build_runner_presenter.rb index 26fbfa34593481..83d567f11f5539 100644 --- a/ee/app/presenters/ee/ci/build_runner_presenter.rb +++ b/ee/app/presenters/ee/ci/build_runner_presenter.rb @@ -15,6 +15,13 @@ def secrets_configuration secret['akeyless']['server'] = akeyless_server(secret) end + if ::Feature.enabled?(:aws_secret_manager, project) && (secret['aws_secrets_manager']) + secret['aws_secrets_manager']['server'] = { + 'region' => variables['AWS_REGION']&.value, + 'jwt' => aws_jwt(secret) + } + end + # For compatibility with the existing Vault integration in Runner, # template gitlab_secrets_manager data into the vault field. if secret.has_key?('gitlab_secrets_manager') @@ -82,6 +89,16 @@ def id_token_var(secret) secret['token'] || "$#{id_tokens.each_key.first}" end + def aws_jwt(secret) + return unless id_tokens? + + id_token_var(secret) + end + + def aws_id_token_var(secret) + secret['token'] || id_tokens['AWS_ID_TOKEN'] + end + def gcp_secret_manager_server(secret) @gcp_secret_manager_server ||= { 'project_number' => variables['GCP_PROJECT_NUMBER']&.value, diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index 21125c883e653f..ae24dd71cdd376 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -16,9 +16,9 @@ class Secret < ::Gitlab::Config::Entry::Node validations do # Required fields validates :secret_id, type: String, presence: true - validates :region, type: String, presence: true # Optional fields + validates :region, type: String, allow_nil: true validates :version_id, type: String, allow_nil: true validates :version_stage, type: String, allow_nil: true end diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index ef52202f423004..fb71e6ce68553b 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -95,8 +95,7 @@ let(:config) do { aws_secrets_manager: { - secret_id: 'name', - region: 'eu-central-1' + secret_id: 'name' }, token: '$TEST_ID_TOKEN' } @@ -130,7 +129,7 @@ context 'when secret_id is missing' do let(:config) do { - aws_secrets_manager: { region: 'us-east-1' } + aws_secrets_manager: {} } end @@ -139,19 +138,6 @@ expect(entry.errors).to include(/aws_secrets_manager secret can't be blank/) end end - - context 'when region is missing' do - let(:config) do - { - aws_secrets_manager: { secret_id: 'db_password' } - } - end - - it 'is not valid' do - expect(entry).not_to be_valid - expect(entry.errors).to include(/aws_secrets_manager region can't be blank/) - end - end end context 'when `token` is not defined' do -- GitLab From 482e39bd3d289f5fd5deb2b4d7c84aa2f9850af4 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 20 May 2025 07:40:41 +0000 Subject: [PATCH 13/26] Remove files and move to different MR --- app/assets/javascripts/editor/schema/ci.json | 23 ---------- .../ci/yaml_tests/negative_tests/secrets.yml | 16 ------- .../ci/yaml_tests/positive_tests/secrets.yml | 44 ------------------- 3 files changed, 83 deletions(-) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 60afbfd86fbfe2..3fee6762423fe6 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1028,24 +1028,6 @@ ], "additionalProperties": false }, - "aws_secrets_manager": { - "type": "object", - "properties": { - "secret_id": { - "type": "string" - }, - "version_id": { - "type": "string" - }, - "version_stage": { - "type": "string" - } - }, - "required": [ - "secret_id" - ], - "additionalProperties": false - }, "akeyless": { "type": "object", "properties": { @@ -1093,11 +1075,6 @@ "gcp_secret_manager" ] }, - { - "required": [ - "aws_secrets_manager" - ] - }, { "required": [ "akeyless" diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml index e866e57acee9d5..23d667eeeffaef 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml @@ -89,19 +89,3 @@ job_with_gcp_secret_manager_secret_without_token: gcp_secret_manager: name: my-secret -job_with_aws_secrets_manager_secret_without_secret_id: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - region: "us-east-1" - -job_with_aws_secrets_manager_secret_without_region: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - secret_id: "database_password" - diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index 3b786317ea9ae1..e615fa52dc5e36 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -72,47 +72,3 @@ valid_job_with_gcp_secret_manager_name_and_string_version: name: 'test' version: 'latest' token: $TEST_TOKEN - -valid_job_with_aws_secretsmanager_secret_id: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - secret_id: 'test' - region: 'us-east-1' - token: $TEST_TOKEN - -valid_job_with_aws_secretsmanager_secret_id_and_version_id: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - secret_id: 'test' - version_id: "2" - region: 'us-east-1' - token: $TEST_TOKEN - -valid_job_with_aws_secretsmanager_secret_id_and_version_stage: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - secret_id: 'test' - region: "us-east-1" - version_stage: 'AWSCURRENT' - token: $TEST_TOKEN - -valid_job_with_aws_secretsmanager_with_every_keyword: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_secrets_manager: - secret_id: 'test' - region: "us-east-1" - version_stage: 'AWSCURRENT' - version_id: "2" - token: $TEST_TOKEN -- GitLab From fe5fed2de20dc3663d0ccbd9e080b2bfed6a9c0f Mon Sep 17 00:00:00 2001 From: markus siebert Date: Tue, 20 May 2025 09:05:24 +0000 Subject: [PATCH 14/26] Improve the implementation and add tests --- .rubocop_todo/rspec/feature_category.yml | 1 + app/assets/javascripts/editor/schema/ci.json | 37 ++++-- ee/app/models/ci/secrets/integration.rb | 9 +- .../entry/aws_ssm_parameter_store/secret.rb | 61 +++++++--- .../aws_ssm_parameter_store/secret_spec.rb | 115 ++++++++++++++++++ .../lib/gitlab/ci/config/entry/secret_spec.rb | 12 +- .../ci/yaml_tests/negative_tests/secrets.yml | 8 -- .../ci/yaml_tests/positive_tests/secrets.yml | 12 +- 8 files changed, 211 insertions(+), 44 deletions(-) create mode 100644 ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb diff --git a/.rubocop_todo/rspec/feature_category.yml b/.rubocop_todo/rspec/feature_category.yml index 1ed9764798df6d..4dfbbf13aaf45b 100644 --- a/.rubocop_todo/rspec/feature_category.yml +++ b/.rubocop_todo/rspec/feature_category.yml @@ -529,6 +529,7 @@ RSpec/FeatureCategory: - 'ee/spec/lib/gitlab/ci/config/entry/dast_configuration_spec.rb' - 'ee/spec/lib/gitlab/ci/config/entry/vault/engine_spec.rb' - 'ee/spec/lib/gitlab/ci/config/entry/vault/secret_spec.rb' + - 'ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/license_compliance/license_scanning_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/metrics/generic_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/security/cluster_image_scanning_spec.rb' diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index e49e80fac2f134..2fc905d21b1c7c 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1059,20 +1059,33 @@ "description": "Specifies the JWT variable that should be used to authenticate with the secret provider." }, "aws_ssm_parameter_store": { - "type": "object", - "properties": { - "name": { - "type": "string" + "oneOf": [ + { + "type": "string", + "markdownDescription": "The secret to be fetched from AWS SSM Parameter Store (e.g. 'production/db/password'). [Learn More](https://docs.gitlab.com/ci/yaml/#secretsawsssmparameterstore)" }, - "region": { - "type": "string" + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "region": { + "type": "string" + }, + "role_arn": { + "type": "string" + }, + "role_session_name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false } - }, - "required": [ - "name", - "region" - ], - "additionalProperties": false + ] } }, "anyOf": [ diff --git a/ee/app/models/ci/secrets/integration.rb b/ee/app/models/ci/secrets/integration.rb index 981bfa211c3c2d..ecea2d231f7acd 100644 --- a/ee/app/models/ci/secrets/integration.rb +++ b/ee/app/models/ci/secrets/integration.rb @@ -8,7 +8,8 @@ class Integration "akeyless" => :akeyless, "gcp_secret_manager" => :gcp_secret_manager, "vault" => :hashicorp_vault, - "gitlab_secrets_manager" => :gitlab_secrets_manager + "gitlab_secrets_manager" => :gitlab_secrets_manager, + "aws_ssm_parameter_store" => :aws_ssm_parameter_store }.freeze def initialize(variables:, project:) @@ -60,6 +61,12 @@ def akeyless? variables['AKEYLESS_ACCESS_ID']&.value.present? end + def aws_ssm_parameter_store? + variables['AWS_REGION']&.value.present? && + variables['AWS_ROLE_ARN']&.value.present? && + variables['AWS_ROLE_SESSION_NAME']&.value.present? + end + def gitlab_secrets_manager? # TODO: figure out context for whether GitLab Secrets Manager is # globally enabled on this instance. diff --git a/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb index 066d413fcd58ba..f41f7a13f54347 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb @@ -5,25 +5,58 @@ module Ci class Config module Entry module AwsSsmParameterStore - class Secret < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable + ## + # Entry that represents AWS SSM ParameterStore. + # + class Secret < ::Gitlab::Config::Entry::Simplifiable + strategy :StringStrategy, if: ->(config) { config.is_a?(String) } + strategy :HashStrategy, if: ->(config) { config.is_a?(Hash) } - ALLOWED_KEYS = %i[name region with_decryption].freeze + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a hash or a string"] + end + end + + class StringStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable - attributes ALLOWED_KEYS + validations do + validates :config, presence: true + validates :config, type: String + end - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :name, type: String, presence: true - validates :region, type: String, presence: true + def value + { + name: config + } + end end - def value - { - name: name, - region: region - } + class HashStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[name region role_arn role_session_name].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :name, type: String, presence: true + validates :region, type: String, presence: true, allow_nil: true + validates :role_arn, type: String, presence: true, allow_nil: true + validates :role_session_name, type: String, presence: true, allow_nil: true + end + + def value + { + name: name, + region: region, + role_arn: role_arn, + role_session_name: role_session_name + } + end end end end diff --git a/ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb new file mode 100644 index 00000000000000..051fcd3a8647a8 --- /dev/null +++ b/ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::AwsSsmParameterStore::Secret do + let(:entry) { described_class.new(config) } + + describe 'validation' do + before do + entry.compose! + end + + context 'when entry config value is correct' do + let(:hash_config) do + { + name: 'production/db/password', + region: 'us-east-1', + role_arn: 'arn:aws:iam::123456789012:role/role-name', + role_session_name: 'session-name' + } + end + + let(:hash_config_minimal) do + { + name: 'production/db/password' + } + end + + context 'when config is a hash' do + let(:config) { hash_config } + + describe '#value' do + it 'returns AWS ParameterStore configuration' do + expect(entry.value).to eq(hash_config) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when config is a string' do + let(:config) { 'production/db/password' } + + describe '#value' do + it 'returns AWS ParameterStore secret configuration' do + expect(entry.value).to eq(hash_config_minimal) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + end + end + + context 'when entry value is not correct' do + describe '#errors' do + context 'when there is an unknown key present' do + let(:config) { { foo: :bar } } + + it 'reports error' do + expect(entry.errors) + .to include 'hash strategy config contains unknown keys: foo' + end + end + + context 'when name is not present' do + let(:config) { {} } + + it 'reports error' do + puts entry.errors + expect(entry.errors) + .to include 'hash strategy name can\'t be blank' + end + end + + context 'when name is is empty' do + let(:config) { { name: "" } } + + it 'reports error' do + expect(entry.errors) + .to include 'hash strategy name can\'t be blank' + end + end + + context 'when optional values are blank' do + let(:config) do + { + name: 'production/db/password', + region: '', + role_arn: '', + role_session_name: '' + } + end + + it 'reports error' do + [ + "hash strategy region can't be blank", + "hash strategy role arn can't be blank", + "hash strategy role session name can't be blank" + ].each do |error| + expect(entry.errors).to include error + end + end + end + end + end +end diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index 9f79d0af200a67..59c78b8f4614fa 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -335,8 +335,7 @@ let(:config) do { aws_ssm_parameter_store: { - name: 'name', - region: 'us-east-1' + name: 'name' }, token: '$TEST_ID_TOKEN' } @@ -347,8 +346,7 @@ expect(entry.value).to eq( { aws_ssm_parameter_store: { - name: 'name', - region: 'us-east-1' + name: 'name' }, token: '$TEST_ID_TOKEN' } @@ -367,8 +365,7 @@ let(:config) do { aws_ssm_parameter_store: { - name: 'name', - region: 'us-east-1' + name: 'name' } } end @@ -378,8 +375,7 @@ expect(entry.value).to eq( { aws_ssm_parameter_store: { - name: 'name', - region: 'us-east-1' + name: 'name' } } ) diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml index aa54d7d8f56ccc..b9ca767b10c62d 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml @@ -96,11 +96,3 @@ job_with_secrets_with_invalid_aws_ssm_parameter_store_property: TEST_DB_PASSWORD: aws_ssm_parameter_store: invalid: TEST - -job_with_secrets_with_missing_required_aws_ssm_parameter_store_property: - script: - - echo $TEST_DB_PASSWORD - secrets: - TEST_DB_PASSWORD: - aws_ssm_parameter_store: - name: test-db-password \ No newline at end of file diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index 5489329c782afd..729075bbff89b7 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -73,6 +73,15 @@ valid_job_with_gcp_secret_manager_name_and_string_version: version: 'latest' token: $TEST_TOKEN +valid_job_with_secrets_with_aws_ssm_parameter_store_keyword_string: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_ssm_parameter_store: test-db-password + token: $TEST_TOKEN + + valid_job_with_secrets_with_every_aws_ssm_parameter_store_keyword: script: - echo $TEST_DB_PASSWORD @@ -81,5 +90,6 @@ valid_job_with_secrets_with_every_aws_ssm_parameter_store_keyword: aws_ssm_parameter_store: name: test-db-password region: us-east-1 - file: true + role_arn: arn:aws:iam::123456789012:role/role-name + role_session_name: role-session-name token: $TEST_TOKEN \ No newline at end of file -- GitLab From 6c2a11bac38cc56ee4813517d0841a47d1d20617 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 20 May 2025 09:22:33 +0000 Subject: [PATCH 15/26] Pass data to runner --- .../json_schemas/build_metadata_secrets.json | 297 ++++++++++-------- .../ee/ci/build_runner_presenter.rb | 24 +- .../entry/aws_secrets_manager/secret.rb | 10 +- .../lib/gitlab/ci/config/entry/secret_spec.rb | 10 +- .../ci/build_runner_presenter_spec.rb | 94 +++++- 5 files changed, 296 insertions(+), 139 deletions(-) diff --git a/app/validators/json_schemas/build_metadata_secrets.json b/app/validators/json_schemas/build_metadata_secrets.json index f9122ff811c4f0..e20d9ead804b4a 100644 --- a/app/validators/json_schemas/build_metadata_secrets.json +++ b/app/validators/json_schemas/build_metadata_secrets.json @@ -1,223 +1,266 @@ +# frozen_string_literal: true + { "$schema": "http://json-schema.org/draft-07/schema#", - "description": "CI builds metadata secrets", - "type": "object", - "patternProperties": { + description: "CI builds metadata secrets", + type: "object", + patternProperties: { ".*": { - "type": "object", - "patternProperties": { + type: "object", + patternProperties: { "^vault$": { - "type": "object", - "required": [ - "path", - "field", - "engine" + type: "object", + required: %w[ + path + field + engine ], - "properties": { - "path": { - "type": "string" - }, - "field": { - "type": "string" - }, - "engine": { - "type": "object", - "required": [ - "name", - "path" + properties: { + path: { + type: "string" + }, + field: { + type: "string" + }, + engine: { + type: "object", + required: %w[ + name + path ], - "properties": { - "path": { - "type": "string" + properties: { + path: { + type: "string" }, - "name": { - "type": "string" + name: { + type: "string" } }, - "additionalProperties": false + additionalProperties: false } }, - "additionalProperties": false + additionalProperties: false }, "^gitlab_secrets_manager$": { - "type": "object", - "required": [ + type: "object", + required: [ "name" ], - "properties": { - "name": { - "type": "string" + properties: { + name: { + type: "string" } }, - "additionalProperties": false + additionalProperties: false }, "^gcp_secret_manager$": { - "type": "object", - "required": [ + type: "object", + required: [ "name" ], - "properties": { - "name": { - "type": "string" + properties: { + name: { + type: "string" }, - "version": { - "type": [ - "string", - "null" + version: { + type: %w[ + string + null ] } }, - "additionalProperties": false + additionalProperties: false }, "^azure_key_vault$": { - "type": "object", - "required": [ + type: "object", + required: [ "name" ], - "properties": { - "name": { - "type": "string" + properties: { + name: { + type: "string" }, - "version": { - "type": [ - "string", - "null" + version: { + type: %w[ + string + null ] } }, - "additionalProperties": false + additionalProperties: false + }, + "^aws_secrets_manager$": { + type: "object", + required: [ + "secret_id" + ], + properties: { + secret_id: { + type: "string" + }, + version_id: { + type: %w[ + string + null + ] + }, + role_session_name: { + type: %w[ + string + null + ] + }, + role_arn: { + type: %w[ + string + null + ] + }, + region: { + type: %w[ + string + null + ] + } + }, + additionalProperties: false }, "^akeyless$": { - "type": "object", - "properties": { - "name": { - "types": [ - "string", - "null" + type: "object", + properties: { + name: { + types: %w[ + string + null ] }, - "data_key": { - "type": [ - "string", - "null" + data_key: { + type: %w[ + string + null ] }, - "public_key_data": { - "type": [ - "string", - "null" + public_key_data: { + type: %w[ + string + null ] }, - "cert_user_name": { - "type": [ - "string", - "null" + cert_user_name: { + type: %w[ + string + null ] }, - "csr_data": { - "type": [ - "string", - "null" + csr_data: { + type: %w[ + string + null ] }, - "akeyless_api_url": { - "type": [ - "string", - "null" + akeyless_api_url: { + type: %w[ + string + null ] }, - "akeyless_access_type": { - "type": [ - "string", - "null" + akeyless_access_type: { + type: %w[ + string + null ] }, - "akeyless_token": { - "type": [ - "string", - "null" + akeyless_token: { + type: %w[ + string + null ] }, - "uid_token": { - "type": [ - "string", - "null" + uid_token: { + type: %w[ + string + null ] }, - "gcp_audience": { - "type": [ - "string", - "null" + gcp_audience: { + type: %w[ + string + null ] }, - "azure_object_id": { - "type": [ - "string", - "null" + azure_object_id: { + type: %w[ + string + null ] }, - "k8s_service_account_token": { - "type": [ - "string", - "null" + k8s_service_account_token: { + type: %w[ + string + null ] }, - "k8s_auth_config_name": { - "type": [ - "string", - "null" + k8s_auth_config_name: { + type: %w[ + string + null ] }, - "akeyless_access_key": { - "type": [ - "string", - "null" + akeyless_access_key: { + type: %w[ + string + null ] }, - "gateway_ca_certificate": { - "type": [ - "string", - "null" + gateway_ca_certificate: { + type: %w[ + string + null ] } }, - "additionalProperties": false + additionalProperties: false }, "^file$": { - "type": "boolean" + type: "boolean" }, "^token$": { - "type": "string" + type: "string" } }, - "anyOf": [ + anyOf: [ { - "required": [ + required: [ "vault" ] }, { - "required": [ + required: [ "gcp_secret_manager" ] }, { - "required": [ + required: [ "azure_key_vault" ] }, { - "required": [ + required: [ "akeyless" ] }, { - "required": [ + required: [ "gitlab_secrets_manager" ] + }, + { + required: [ + "aws_secrets_manager" + ] } ], - "additionalProperties": false + additionalProperties: false } } } diff --git a/ee/app/presenters/ee/ci/build_runner_presenter.rb b/ee/app/presenters/ee/ci/build_runner_presenter.rb index 83d567f11f5539..18581d57007000 100644 --- a/ee/app/presenters/ee/ci/build_runner_presenter.rb +++ b/ee/app/presenters/ee/ci/build_runner_presenter.rb @@ -16,10 +16,7 @@ def secrets_configuration end if ::Feature.enabled?(:aws_secret_manager, project) && (secret['aws_secrets_manager']) - secret['aws_secrets_manager']['server'] = { - 'region' => variables['AWS_REGION']&.value, - 'jwt' => aws_jwt(secret) - } + secret['aws_secrets_manager']['server'] = aws_secrets_manager_server(secret) end # For compatibility with the existing Vault integration in Runner, @@ -63,6 +60,19 @@ def vault_server(secret) } end + def aws_secrets_manager_server(secret) + @aws_secrets_manager_server ||= { + 'region' => secret['aws_secrets_manager']['region'] || variables['AWS_REGION']&.value, + 'jwt' => aws_token(secret), + 'role_arn' => secret['aws_secrets_manager']['role_arn'] || variables['AWS_ROLE_ARN']&.value, + 'role_session_name' => aws_role_session_name(secret) + } + end + + def aws_role_session_name(secret) + secret['aws_secrets_manager']['role_session_name'] || variables['AWS_ROLE_SESSION_NAME']&.value + end + def gitlab_secrets_manager_server(psm) @gitlab_secrets_manager_server ||= { 'url' => SecretsManagement::ProjectSecretsManager.server_url, @@ -89,14 +99,14 @@ def id_token_var(secret) secret['token'] || "$#{id_tokens.each_key.first}" end - def aws_jwt(secret) + def aws_token(secret) return unless id_tokens? - id_token_var(secret) + aws_id_token_var(secret) end def aws_id_token_var(secret) - secret['token'] || id_tokens['AWS_ID_TOKEN'] + secret['token'] || '$AWS_ID_TOKEN' end def gcp_secret_manager_server(secret) diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index ae24dd71cdd376..9910437fe84471 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -9,7 +9,7 @@ class Secret < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[secret_id region version_id version_stage].freeze + ALLOWED_KEYS = %i[secret_id region version_id version_stage role_arn role_session_name field].freeze attributes ALLOWED_KEYS @@ -21,6 +21,9 @@ class Secret < ::Gitlab::Config::Entry::Node validates :region, type: String, allow_nil: true validates :version_id, type: String, allow_nil: true validates :version_stage, type: String, allow_nil: true + validates :role_arn, type: String, allow_nil: true + validates :role_session_name, type: String, allow_nil: true + validates :field, type: String, allow_nil: true end def value @@ -28,7 +31,10 @@ def value secret_id: secret_id, version_id: version_id, version_stage: version_stage, - region: region + region: region, + role_arn: role_arn, + field: field, + role_session_name: role_session_name } end end diff --git a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb index fb71e6ce68553b..932650753d4339 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb @@ -115,7 +115,10 @@ secret_id: 'db-password', region: 'us-east-1', version_id: 'abcdef1234567890', - version_stage: 'AWSCURRENT' + version_stage: 'AWSCURRENT', + role_arn: 'arn:aws:iam::123456789012:role/role-name', + field: 'password', + role_session_name: 'session-name' } } end @@ -158,7 +161,10 @@ secret_id: 'name', region: 'eu-central-1', version_id: nil, - version_stage: nil + version_stage: nil, + role_arn: nil, + field: nil, + role_session_name: nil } } ) diff --git a/ee/spec/presenters/ci/build_runner_presenter_spec.rb b/ee/spec/presenters/ci/build_runner_presenter_spec.rb index 45fc0e86860e2b..7f59327408d984 100644 --- a/ee/spec/presenters/ci/build_runner_presenter_spec.rb +++ b/ee/spec/presenters/ci/build_runner_presenter_spec.rb @@ -223,7 +223,6 @@ it 'adds the first ID token to the Vault server payload' do jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'azure_key_vault', 'server', 'jwt') - expect(jwt).to eq('$VAULT_ID_TOKEN_1') end @@ -257,6 +256,99 @@ end end + context 'with AWS Secrets Manager' do + let(:secrets) do + { + DATABASE_PASSWORD: { + aws_secrets_manager: { + secret_id: 'key', + version_id: 'version' + } + } + } + end + + let(:aws_secrets_manager_server) { presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'aws_secrets_manager', 'server') } + + context 'Secrets Manager Region' do + context 'AWS_REGION CI variable is present' do + it 'returns the Region' do + create(:ci_variable, project: ci_build.project, key: 'AWS_REGION', value: 'test') + + expect(aws_secrets_manager_server.fetch('region')).to eq('test') + end + end + + context 'AWS_REGION CI variable is not present' do + it 'returns the nil' do + expect(aws_secrets_manager_server.fetch('region')).to eq(nil) + end + end + end + + context 'Vault tenant id' do + context 'AWS_ROLE_ARN CI variable is present' do + it 'returns the URL' do + create(:ci_variable, project: ci_build.project, key: 'AWS_ROLE_ARN', value: 'test') + + expect(aws_secrets_manager_server.fetch('role_arn')).to eq('test') + end + end + + context 'AWS_ROLE_ARN CI variable is not present' do + it 'returns the nil' do + expect(aws_secrets_manager_server.fetch('role_arn')).to eq(nil) + end + end + end + + context 'when there are ID tokens available' do + before do + rsa_key = OpenSSL::PKey::RSA.generate(3072).to_s + stub_application_setting(ci_jwt_signing_key: rsa_key) + ci_build.id_tokens = { + 'VAULT_ID_TOKEN_2' => { id_token: { aud: 'https://gitlab.link' } }, + 'AWS_ID_TOKEN' => { id_token: { aud: 'https://gitlab.test' } } + } + ci_build.runner = build_stubbed(:ci_runner) + end + + it 'adds the AWS_ID_TOKEN to the Vault server payload' do + jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'aws_secrets_manager', 'server', 'jwt') + + expect(jwt).to eq('$AWS_ID_TOKEN') + end + + context 'when the token variable is specified for the vault secret' do + let(:secrets) do + { + DATABASE_PASSWORD: { + token: '$VAULT_ID_TOKEN_2', + aws_secrets_manager: { + secret_id: 'key', + version_id: 'version' + } + } + } + end + + it 'uses the specified token variable' do + jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'aws_secrets_manager', 'server', 'jwt') + + expect(jwt).to eq('$VAULT_ID_TOKEN_2') + end + end + end + + context 'when there are no ID tokens available' do + it 'returns nil so instance profile could be used' do + jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'aws_secrets_manager', 'server', 'jwt') + + expect(jwt).to be_nil + end + end + end + context 'with GCP Secret Manager' do let(:secrets) do { -- GitLab From 67349d5abce4186ca4ea22b377a78c9341b2d0af Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 20 May 2025 09:31:52 +0000 Subject: [PATCH 16/26] Fix Json Schema --- .../json_schemas/build_metadata_secrets.json | 304 +++++++++--------- 1 file changed, 154 insertions(+), 150 deletions(-) diff --git a/app/validators/json_schemas/build_metadata_secrets.json b/app/validators/json_schemas/build_metadata_secrets.json index e20d9ead804b4a..1b174ec8b6e202 100644 --- a/app/validators/json_schemas/build_metadata_secrets.json +++ b/app/validators/json_schemas/build_metadata_secrets.json @@ -1,266 +1,270 @@ -# frozen_string_literal: true - { "$schema": "http://json-schema.org/draft-07/schema#", - description: "CI builds metadata secrets", - type: "object", - patternProperties: { + "description": "CI builds metadata secrets", + "type": "object", + "patternProperties": { ".*": { - type: "object", - patternProperties: { + "type": "object", + "patternProperties": { "^vault$": { - type: "object", - required: %w[ - path - field - engine + "type": "object", + "required": [ + "path", + "field", + "engine" ], - properties: { - path: { - type: "string" + "properties": { + "path": { + "type": "string" }, - field: { - type: "string" + "field": { + "type": "string" }, - engine: { - type: "object", - required: %w[ - name - path + "engine": { + "type": "object", + "required": [ + "name", + "path" ], - properties: { - path: { - type: "string" + "properties": { + "path": { + "type": "string" }, - name: { - type: "string" + "name": { + "type": "string" } }, - additionalProperties: false + "additionalProperties": false } }, - additionalProperties: false + "additionalProperties": false }, "^gitlab_secrets_manager$": { - type: "object", - required: [ + "type": "object", + "required": [ "name" ], - properties: { - name: { - type: "string" + "properties": { + "name": { + "type": "string" } }, - additionalProperties: false + "additionalProperties": false }, "^gcp_secret_manager$": { - type: "object", - required: [ + "type": "object", + "required": [ "name" ], - properties: { - name: { - type: "string" + "properties": { + "name": { + "type": "string" }, - version: { - type: %w[ - string - null + "version": { + "type": [ + "string", + "null" ] } }, - additionalProperties: false + "additionalProperties": false }, "^azure_key_vault$": { - type: "object", - required: [ + "type": "object", + "required": [ "name" ], - properties: { - name: { - type: "string" + "properties": { + "name": { + "type": "string" }, - version: { - type: %w[ - string - null + "version": { + "type": [ + "string", + "null" ] } }, - additionalProperties: false + "additionalProperties": false }, "^aws_secrets_manager$": { - type: "object", - required: [ + "type": "object", + "required": [ "secret_id" ], - properties: { - secret_id: { - type: "string" + "properties": { + "secret_id": { + "type": "string" }, - version_id: { - type: %w[ - string - null + "version_id": { + "type": [ + "string", + "null" ] }, - role_session_name: { - type: %w[ - string - null + "version_stage": { + "type": [ + "string", + "null" ] }, - role_arn: { - type: %w[ - string - null + "region": { + "type": [ + "string", + "null" ] }, - region: { - type: %w[ - string - null + "role_arn": { + "type": [ + "string", + "null" + ] + }, + "role_session_name": { + "type": [ + "string", + "null" ] } }, - additionalProperties: false + "additionalProperties": false }, "^akeyless$": { - type: "object", - properties: { - name: { - types: %w[ - string - null + "type": "object", + "properties": { + "name": { + "types": [ + "string", + "null" ] }, - data_key: { - type: %w[ - string - null + "data_key": { + "type": [ + "string", + "null" ] }, - public_key_data: { - type: %w[ - string - null + "public_key_data": { + "type": [ + "string", + "null" ] }, - cert_user_name: { - type: %w[ - string - null + "cert_user_name": { + "type": [ + "string", + "null" ] }, - csr_data: { - type: %w[ - string - null + "csr_data": { + "type": [ + "string", + "null" ] }, - akeyless_api_url: { - type: %w[ - string - null + "akeyless_api_url": { + "type": [ + "string", + "null" ] }, - akeyless_access_type: { - type: %w[ - string - null + "akeyless_access_type": { + "type": [ + "string", + "null" ] }, - akeyless_token: { - type: %w[ - string - null + "akeyless_token": { + "type": [ + "string", + "null" ] }, - uid_token: { - type: %w[ - string - null + "uid_token": { + "type": [ + "string", + "null" ] }, - gcp_audience: { - type: %w[ - string - null + "gcp_audience": { + "type": [ + "string", + "null" ] }, - azure_object_id: { - type: %w[ - string - null + "azure_object_id": { + "type": [ + "string", + "null" ] }, - k8s_service_account_token: { - type: %w[ - string - null + "k8s_service_account_token": { + "type": [ + "string", + "null" ] }, - k8s_auth_config_name: { - type: %w[ - string - null + "k8s_auth_config_name": { + "type": [ + "string", + "null" ] }, - akeyless_access_key: { - type: %w[ - string - null + "akeyless_access_key": { + "type": [ + "string", + "null" ] }, - gateway_ca_certificate: { - type: %w[ - string - null + "gateway_ca_certificate": { + "type": [ + "string", + "null" ] } }, - additionalProperties: false + "additionalProperties": false }, "^file$": { - type: "boolean" + "type": "boolean" }, "^token$": { - type: "string" + "type": "string" } }, - anyOf: [ + "anyOf": [ { - required: [ + "required": [ "vault" ] }, { - required: [ + "required": [ "gcp_secret_manager" ] }, { - required: [ + "required": [ "azure_key_vault" ] }, { - required: [ + "required": [ "akeyless" ] }, { - required: [ + "required": [ "gitlab_secrets_manager" ] }, { - required: [ + "required": [ "aws_secrets_manager" ] } ], - additionalProperties: false + "additionalProperties": false } } } -- GitLab From eca179bcb3c0b29ecf56503ea72f43c85e80e618 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 20 May 2025 07:42:00 +0000 Subject: [PATCH 17/26] Add JSON Schmema for AWS Secretsmanager --- app/assets/javascripts/editor/schema/ci.json | 23 ++++++++++ .../ci/yaml_tests/negative_tests/secrets.yml | 16 +++++++ .../ci/yaml_tests/positive_tests/secrets.yml | 44 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 3fee6762423fe6..60afbfd86fbfe2 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1028,6 +1028,24 @@ ], "additionalProperties": false }, + "aws_secrets_manager": { + "type": "object", + "properties": { + "secret_id": { + "type": "string" + }, + "version_id": { + "type": "string" + }, + "version_stage": { + "type": "string" + } + }, + "required": [ + "secret_id" + ], + "additionalProperties": false + }, "akeyless": { "type": "object", "properties": { @@ -1075,6 +1093,11 @@ "gcp_secret_manager" ] }, + { + "required": [ + "aws_secrets_manager" + ] + }, { "required": [ "akeyless" diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml index 23d667eeeffaef..e866e57acee9d5 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml @@ -89,3 +89,19 @@ job_with_gcp_secret_manager_secret_without_token: gcp_secret_manager: name: my-secret +job_with_aws_secrets_manager_secret_without_secret_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + region: "us-east-1" + +job_with_aws_secrets_manager_secret_without_region: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: "database_password" + diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index e615fa52dc5e36..3b786317ea9ae1 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -72,3 +72,47 @@ valid_job_with_gcp_secret_manager_name_and_string_version: name: 'test' version: 'latest' token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: 'us-east-1' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id_and_version_id: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + version_id: "2" + region: 'us-east-1' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id_and_version_stage: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: "us-east-1" + version_stage: 'AWSCURRENT' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_with_every_keyword: + script: + - echo $TEST_DB_PASSWORD + secrets: + TEST_DB_PASSWORD: + aws_secrets_manager: + secret_id: 'test' + region: "us-east-1" + version_stage: 'AWSCURRENT' + version_id: "2" + token: $TEST_TOKEN -- GitLab From 6218b5314d66919e9eb19864036f1c795260bd5b Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 20 May 2025 07:47:09 +0000 Subject: [PATCH 18/26] ReAdd Region --- app/assets/javascripts/editor/schema/ci.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 60afbfd86fbfe2..db43793fd960f8 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1039,6 +1039,9 @@ }, "version_stage": { "type": "string" + }, + "region": { + "type": "string" } }, "required": [ -- GitLab From 7166c739a90ac22a143101e80a388e5de9125f74 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Tue, 20 May 2025 13:40:16 +0000 Subject: [PATCH 19/26] fix --- ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index 9910437fe84471..a08fba0e94c0d8 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -14,6 +14,7 @@ class Secret < ::Gitlab::Config::Entry::Node attributes ALLOWED_KEYS validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS # Required fields validates :secret_id, type: String, presence: true -- GitLab From f32d405d1c06deeb828e49da3c7bcf0e788c4bef Mon Sep 17 00:00:00 2001 From: markus siebert Date: Wed, 21 May 2025 07:38:03 +0000 Subject: [PATCH 20/26] more --- app/assets/javascripts/editor/schema/ci.json | 9 +++++++++ .../json_schemas/build_metadata_secrets.json | 6 ++++++ .../ee/ci/build_runner_presenter.rb | 5 ++++- ...te_secrets_aws_secrets_manager_monthly.yml | 19 +++++++++++++++++++ .../entry/aws_ssm_parameter_store/secret.rb | 6 +++--- .../hll_redis_legacy_events.yml | 2 ++ 6 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 ee/config/metrics/counts_28d/count_total_create_secrets_aws_secrets_manager_monthly.yml diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 928430f2e625f6..0346a32813eb2a 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1042,6 +1042,12 @@ }, "region": { "type": "string" + }, + "role_arn": { + "type": "string" + }, + "role_session_name": { + "type": "string" } }, "required": [ @@ -1066,6 +1072,9 @@ }, "csr_data": { "type": "string" + }, + "field": { + "type": "string" } }, "additionalProperties": false diff --git a/app/validators/json_schemas/build_metadata_secrets.json b/app/validators/json_schemas/build_metadata_secrets.json index 1b174ec8b6e202..74ca6db612a7a4 100644 --- a/app/validators/json_schemas/build_metadata_secrets.json +++ b/app/validators/json_schemas/build_metadata_secrets.json @@ -125,6 +125,12 @@ "string", "null" ] + }, + "field": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": false diff --git a/ee/app/presenters/ee/ci/build_runner_presenter.rb b/ee/app/presenters/ee/ci/build_runner_presenter.rb index 18581d57007000..e591c876d4480c 100644 --- a/ee/app/presenters/ee/ci/build_runner_presenter.rb +++ b/ee/app/presenters/ee/ci/build_runner_presenter.rb @@ -62,10 +62,13 @@ def vault_server(secret) def aws_secrets_manager_server(secret) @aws_secrets_manager_server ||= { + 'secret_id' => secret['aws_secrets_manager']['secret_id'], + 'version_id' => secret['aws_secrets_manager']['version_id'] || variables['AWS_VERSION_ID']&.value, 'region' => secret['aws_secrets_manager']['region'] || variables['AWS_REGION']&.value, 'jwt' => aws_token(secret), 'role_arn' => secret['aws_secrets_manager']['role_arn'] || variables['AWS_ROLE_ARN']&.value, - 'role_session_name' => aws_role_session_name(secret) + 'role_session_name' => aws_role_session_name(secret), + 'field' => secret['aws_secrets_manager']['field'] } end diff --git a/ee/config/metrics/counts_28d/count_total_create_secrets_aws_secrets_manager_monthly.yml b/ee/config/metrics/counts_28d/count_total_create_secrets_aws_secrets_manager_monthly.yml new file mode 100644 index 00000000000000..f4ef3bd8e66061 --- /dev/null +++ b/ee/config/metrics/counts_28d/count_total_create_secrets_aws_secrets_manager_monthly.yml @@ -0,0 +1,19 @@ +--- +data_category: optional +key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_aws_secrets_manager_build_created_monthly +description: Monthly active users creating pipelines that that have the AWS SecretsManager secrets. +product_group: pipeline_security +value_type: number +status: active +time_frame: 28d +data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - i_ci_secrets_management_aws_secrets_manager_build_created +tiers: + - premium + - ultimate +performance_indicator_type: [] +milestone: "16.9" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/191905 diff --git a/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb index f41f7a13f54347..ce732b97611bd6 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret.rb @@ -44,9 +44,9 @@ class HashStrategy < ::Gitlab::Config::Entry::Node validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS validates :name, type: String, presence: true - validates :region, type: String, presence: true, allow_nil: true - validates :role_arn, type: String, presence: true, allow_nil: true - validates :role_session_name, type: String, presence: true, allow_nil: true + validates :region, type: String, allow_nil: true + validates :role_arn, type: String, allow_nil: true + validates :role_session_name, type: String, allow_nil: true end def value diff --git a/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml b/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml index e12b15f6d392c2..5e8fbf1e788d4d 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml +++ b/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml @@ -43,6 +43,8 @@ - i_ci_secrets_management_id_tokens_build_created - i_ci_secrets_management_vault_build_created - i_ci_secrets_management_gitlab_secrets_manager_build_created +- i_ci_secrets_management_aws_secrets_manager_build_created +- i_ci_secrets_management_aws_ssm_parameter_store_build_created - i_code_review_click_diff_view_setting - i_code_review_click_file_browser_setting - i_code_review_click_single_file_mode_setting -- GitLab From b2b4b8d930a6e5dae81005d40dcc8f5f694b9c9f Mon Sep 17 00:00:00 2001 From: markus siebert Date: Wed, 21 May 2025 08:38:51 +0000 Subject: [PATCH 21/26] more --- app/assets/javascripts/editor/schema/ci.json | 6 +++--- ee/app/presenters/ee/ci/build_runner_presenter.rb | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 0346a32813eb2a..3540243b1951d4 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1048,6 +1048,9 @@ }, "role_session_name": { "type": "string" + }, + "field": { + "type": "string" } }, "required": [ @@ -1072,9 +1075,6 @@ }, "csr_data": { "type": "string" - }, - "field": { - "type": "string" } }, "additionalProperties": false diff --git a/ee/app/presenters/ee/ci/build_runner_presenter.rb b/ee/app/presenters/ee/ci/build_runner_presenter.rb index e591c876d4480c..18581d57007000 100644 --- a/ee/app/presenters/ee/ci/build_runner_presenter.rb +++ b/ee/app/presenters/ee/ci/build_runner_presenter.rb @@ -62,13 +62,10 @@ def vault_server(secret) def aws_secrets_manager_server(secret) @aws_secrets_manager_server ||= { - 'secret_id' => secret['aws_secrets_manager']['secret_id'], - 'version_id' => secret['aws_secrets_manager']['version_id'] || variables['AWS_VERSION_ID']&.value, 'region' => secret['aws_secrets_manager']['region'] || variables['AWS_REGION']&.value, 'jwt' => aws_token(secret), 'role_arn' => secret['aws_secrets_manager']['role_arn'] || variables['AWS_ROLE_ARN']&.value, - 'role_session_name' => aws_role_session_name(secret), - 'field' => secret['aws_secrets_manager']['field'] + 'role_session_name' => aws_role_session_name(secret) } end -- GitLab From 5f36162d78e0a378c62f94abf2cf785a0e415eeb Mon Sep 17 00:00:00 2001 From: markus siebert Date: Wed, 21 May 2025 11:28:46 +0000 Subject: [PATCH 22/26] more --- ee/app/models/ci/secrets/integration.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ee/app/models/ci/secrets/integration.rb b/ee/app/models/ci/secrets/integration.rb index 483b5c2c5f45f0..4784735b34cdd8 100644 --- a/ee/app/models/ci/secrets/integration.rb +++ b/ee/app/models/ci/secrets/integration.rb @@ -4,13 +4,13 @@ module Ci module Secrets class Integration PROVIDER_TYPE_MAP = { - "aws_secrets_manager" => :aws_secrets_manager, "azure_key_vault" => :azure_key_vault, "akeyless" => :akeyless, "gcp_secret_manager" => :gcp_secret_manager, "vault" => :hashicorp_vault, "gitlab_secrets_manager" => :gitlab_secrets_manager, - "aws_ssm_parameter_store" => :aws_ssm_parameter_store + "aws_secrets_manager" => :aws, + "aws_ssm_parameter_store" => :aws }.freeze def initialize(variables:, project:) @@ -62,16 +62,10 @@ def akeyless? variables['AKEYLESS_ACCESS_ID']&.value.present? end - def aws_secrets_manager? + def aws? variables['AWS_REGION']&.value.present? end - def aws_ssm_parameter_store? - variables['AWS_REGION']&.value.present? && - variables['AWS_ROLE_ARN']&.value.present? && - variables['AWS_ROLE_SESSION_NAME']&.value.present? - end - def gitlab_secrets_manager? # TODO: figure out context for whether GitLab Secrets Manager is # globally enabled on this instance. -- GitLab From 18180e83c2a5e71b1ab2acbbe400c20b80a7f7f9 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Wed, 21 May 2025 13:00:15 +0000 Subject: [PATCH 23/26] more --- .../presenters/ee/ci/build_runner_presenter.rb | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/ee/app/presenters/ee/ci/build_runner_presenter.rb b/ee/app/presenters/ee/ci/build_runner_presenter.rb index 18581d57007000..a1edbacf44af1a 100644 --- a/ee/app/presenters/ee/ci/build_runner_presenter.rb +++ b/ee/app/presenters/ee/ci/build_runner_presenter.rb @@ -62,17 +62,13 @@ def vault_server(secret) def aws_secrets_manager_server(secret) @aws_secrets_manager_server ||= { - 'region' => secret['aws_secrets_manager']['region'] || variables['AWS_REGION']&.value, + 'region' => variables['AWS_REGION']&.value, 'jwt' => aws_token(secret), - 'role_arn' => secret['aws_secrets_manager']['role_arn'] || variables['AWS_ROLE_ARN']&.value, - 'role_session_name' => aws_role_session_name(secret) + 'role_arn' => variables['AWS_ROLE_ARN']&.value, + 'role_session_name' => variables['AWS_ROLE_SESSION_NAME']&.value } end - def aws_role_session_name(secret) - secret['aws_secrets_manager']['role_session_name'] || variables['AWS_ROLE_SESSION_NAME']&.value - end - def gitlab_secrets_manager_server(psm) @gitlab_secrets_manager_server ||= { 'url' => SecretsManagement::ProjectSecretsManager.server_url, @@ -100,12 +96,6 @@ def id_token_var(secret) end def aws_token(secret) - return unless id_tokens? - - aws_id_token_var(secret) - end - - def aws_id_token_var(secret) secret['token'] || '$AWS_ID_TOKEN' end -- GitLab From 20a5578d452c88232dccebeca4ba053b2ee1beb4 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Thu, 22 May 2025 07:05:45 +0000 Subject: [PATCH 24/26] Change secretsmanager code to accept string and map --- app/assets/javascripts/editor/schema/ci.json | 68 ++++++++------ .../entry/aws_secrets_manager/secret.rb | 89 ++++++++++++------- 2 files changed, 101 insertions(+), 56 deletions(-) diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 3540243b1951d4..d7afb331698fbc 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1029,34 +1029,50 @@ "additionalProperties": false }, "aws_secrets_manager": { - "type": "object", - "properties": { - "secret_id": { - "type": "string" - }, - "version_id": { - "type": "string" - }, - "version_stage": { - "type": "string" - }, - "region": { - "type": "string" - }, - "role_arn": { - "type": "string" - }, - "role_session_name": { - "type": "string" + "oneOf": [ + { + "type": "string", + "markdownDescription": "The secret to be fetched from AWS Secrets Manager (e.g. 'production/db/password'). [Learn More](https://docs.gitlab.com/ci/yaml/#secretsawsssmparameterstore)" }, - "field": { - "type": "string" + { + "type": "object", + "markdownDescription": "Defines the secret version to be fetched from AWS Secrets Manager. The secret_id refers to the ARN or name of the secret in AWS Secrets Manager. Version_id and version_stage are optional parameters that can be used to specify a specific version of the secret, else AWSCURRENT version will be returned.", + "properties": { + "secret_id": { + "type": "string", + "markdownDescription": "The ARN or name of the secret to retrieve. To retrieve a secret from another account, you must use an ARN." + }, + "version_id": { + "type": "string", + "markdownDescription": "The unique identifier of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." + }, + "version_stage": { + "type": "string", + "markdownDescription": "The staging label of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." + }, + "region": { + "type": "string", + "markdownDescription": "The AWS region where the secret is stored. Use this to override the region for a specific secret. Defaults to AWS_REGION variable." + }, + "role_arn": { + "type": "string", + "markdownDescription": "The ARN of the IAM role to assume before retrieving the secret. Use this to override the ARN. Defaults to AWS_ROLE_ARN variable." + }, + "role_session_name": { + "type": "string", + "markdownDescription": "The name of the session to use when assuming the role. Use this to override the session name. Defaults to AWS_ROLE_SESSION_NAME variable." + }, + "field": { + "type": "string", + "markdownDescription": "The name of the field to retrieve from the secret. If not specified, the entire secret is retrieved." + } + }, + "required": [ + "secret_id" + ], + "additionalProperties": false } - }, - "required": [ - "secret_id" - ], - "additionalProperties": false + ] }, "akeyless": { "type": "object", diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index a08fba0e94c0d8..1069dab53a6382 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -5,38 +5,67 @@ module Ci class Config module Entry module AwsSecretsManager - class Secret < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[secret_id region version_id version_stage role_arn role_session_name field].freeze - - attributes ALLOWED_KEYS - - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - # Required fields - validates :secret_id, type: String, presence: true - - # Optional fields - validates :region, type: String, allow_nil: true - validates :version_id, type: String, allow_nil: true - validates :version_stage, type: String, allow_nil: true - validates :role_arn, type: String, allow_nil: true - validates :role_session_name, type: String, allow_nil: true - validates :field, type: String, allow_nil: true + ## + # Entry that represents AWS SSM ParameterStore. + # + class Secret < ::Gitlab::Config::Entry::Simplifiable + strategy :StringStrategy, if: ->(config) { config.is_a?(String) } + strategy :HashStrategy, if: ->(config) { config.is_a?(Hash) } + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a hash or a string"] + end + end + + class StringStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: String + end + + def value + { + secret_id: config + } + end end - def value - { - secret_id: secret_id, - version_id: version_id, - version_stage: version_stage, - region: region, - role_arn: role_arn, - field: field, - role_session_name: role_session_name - } + class HashStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[secret_id region version_id version_stage role_arn role_session_name field].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + # Required fields + validates :secret_id, type: String, presence: true + + # Optional fields + validates :region, type: String, allow_nil: true + validates :version_id, type: String, allow_nil: true + validates :version_stage, type: String, allow_nil: true + validates :role_arn, type: String, allow_nil: true + validates :role_session_name, type: String, allow_nil: true + validates :field, type: String, allow_nil: true + end + + def value + { + secret_id: secret_id, + version_id: version_id, + version_stage: version_stage, + region: region, + role_arn: role_arn, + field: field, + role_session_name: role_session_name + } + end end end end -- GitLab From 88856e303376b556fe08c15f928312630286b64f Mon Sep 17 00:00:00 2001 From: markus siebert Date: Thu, 22 May 2025 07:43:41 +0000 Subject: [PATCH 25/26] Tests and last ci.json changes --- app/assets/javascripts/editor/schema/ci.json | 16 +-- .../entry/aws_secrets_manager/secret_spec.rb | 97 +++++++++++++++++++ .../ci/yaml_tests/positive_tests/secrets.yml | 5 +- 3 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index d7afb331698fbc..4613cdeb0088ad 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1032,7 +1032,7 @@ "oneOf": [ { "type": "string", - "markdownDescription": "The secret to be fetched from AWS Secrets Manager (e.g. 'production/db/password'). [Learn More](https://docs.gitlab.com/ci/yaml/#secretsawsssmparameterstore)" + "description": "The secret to be fetched from AWS Secrets Manager (e.g. 'production/db/password'). [Learn More](https://docs.gitlab.com/ci/yaml/#secretsawsssmparameterstore)" }, { "type": "object", @@ -1040,31 +1040,31 @@ "properties": { "secret_id": { "type": "string", - "markdownDescription": "The ARN or name of the secret to retrieve. To retrieve a secret from another account, you must use an ARN." + "description": "The ARN or name of the secret to retrieve. To retrieve a secret from another account, you must use an ARN." }, "version_id": { "type": "string", - "markdownDescription": "The unique identifier of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." + "description": "The unique identifier of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." }, "version_stage": { "type": "string", - "markdownDescription": "The staging label of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." + "description": "The staging label of the version of the secret to retrieve. If you include both this parameter and VersionStage, the two parameters must refer to the same secret version. If you don't specify either a VersionStage or VersionId, Secrets Manager returns the AWSCURRENT version." }, "region": { "type": "string", - "markdownDescription": "The AWS region where the secret is stored. Use this to override the region for a specific secret. Defaults to AWS_REGION variable." + "description": "The AWS region where the secret is stored. Use this to override the region for a specific secret. Defaults to AWS_REGION variable." }, "role_arn": { "type": "string", - "markdownDescription": "The ARN of the IAM role to assume before retrieving the secret. Use this to override the ARN. Defaults to AWS_ROLE_ARN variable." + "description": "The ARN of the IAM role to assume before retrieving the secret. Use this to override the ARN. Defaults to AWS_ROLE_ARN variable." }, "role_session_name": { "type": "string", - "markdownDescription": "The name of the session to use when assuming the role. Use this to override the session name. Defaults to AWS_ROLE_SESSION_NAME variable." + "description": "The name of the session to use when assuming the role. Use this to override the session name. Defaults to AWS_ROLE_SESSION_NAME variable." }, "field": { "type": "string", - "markdownDescription": "The name of the field to retrieve from the secret. If not specified, the entire secret is retrieved." + "description": "The name of the field to retrieve from the secret. If not specified, the entire secret is retrieved." } }, "required": [ diff --git a/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb new file mode 100644 index 00000000000000..87a90df4006ae2 --- /dev/null +++ b/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::AwsSecretsManager::Secret do + let(:entry) { described_class.new(config) } + + describe 'validation' do + before do + entry.compose! + end + + context 'when entry config value is correct' do + let(:hash_config) do + { + secret_id: 'production/db/password', + version_id: 'version_id', + version_stage: 'AWSCURRENT', + field: 'some_field', + region: 'us-east-1', + role_arn: 'arn:aws:iam::123456789012:role/role-name', + role_session_name: 'session-name' + } + end + + let(:hash_config_minimal) do + { + secret_id: 'production/db/password' + } + end + + context 'when config is a hash' do + let(:config) { hash_config } + + describe '#value' do + it 'returns AWS SecretsManager configuration' do + expect(entry.value).to eq(hash_config) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when config is a string' do + let(:config) { 'production/db/password' } + + describe '#value' do + it 'returns AWS SecretsManager secret configuration' do + expect(entry.value).to eq(hash_config_minimal) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + end + end + + context 'when entry value is not correct' do + describe '#errors' do + context 'when there is an unknown key present' do + let(:config) { { foo: :bar } } + + it 'reports error' do + expect(entry.errors) + .to include 'hash strategy config contains unknown keys: foo' + end + end + + context 'when name is not present' do + let(:config) { {} } + + it 'reports error' do + puts entry.errors + expect(entry.errors) + .to include 'hash strategy secret can\'t be blank' + end + end + + context 'when secret_id is is blank' do + let(:config) { { secret_id: '' } } + + it 'reports error' do + expect(entry.errors) + .to include "hash strategy secret can't be blank" + end + end + end + end +end diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml index 7e4adf0315a4ec..6090f4cb6090cd 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml @@ -76,7 +76,10 @@ valid_job_with_gcp_secret_manager_name_and_string_version: valid_job_with_aws_secretsmanager_secret_id: aws_secrets_manager: secret_id: 'test' - region: 'us-east-1' + token: $TEST_TOKEN + +valid_job_with_aws_secretsmanager_secret_id_short: + aws_secrets_manager: 'test' token: $TEST_TOKEN valid_job_with_aws_secretsmanager_secret_id_and_version_id: -- GitLab From 418886e6a476671dd746fb90c271fd58ebd3fc30 Mon Sep 17 00:00:00 2001 From: markus siebert Date: Thu, 22 May 2025 08:11:00 +0000 Subject: [PATCH 26/26] Allow passing field to secretsmanager secret via # in string --- .rubocop_todo/rspec/feature_category.yml | 1 + app/assets/javascripts/editor/schema/ci.json | 4 +-- .../entry/aws_secrets_manager/secret.rb | 21 +++++++++++--- .../entry/aws_secrets_manager/secret_spec.rb | 29 +++++++++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.rubocop_todo/rspec/feature_category.yml b/.rubocop_todo/rspec/feature_category.yml index 4dfbbf13aaf45b..3913bd0cf9f1d6 100644 --- a/.rubocop_todo/rspec/feature_category.yml +++ b/.rubocop_todo/rspec/feature_category.yml @@ -530,6 +530,7 @@ RSpec/FeatureCategory: - 'ee/spec/lib/gitlab/ci/config/entry/vault/engine_spec.rb' - 'ee/spec/lib/gitlab/ci/config/entry/vault/secret_spec.rb' - 'ee/spec/lib/gitlab/ci/config/entry/aws_ssm_parameter_store/secret_spec.rb' + - 'ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/license_compliance/license_scanning_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/metrics/generic_spec.rb' - 'ee/spec/lib/gitlab/ci/parsers/security/cluster_image_scanning_spec.rb' diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 4613cdeb0088ad..1e8afb5b864fa4 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -1032,11 +1032,11 @@ "oneOf": [ { "type": "string", - "description": "The secret to be fetched from AWS Secrets Manager (e.g. 'production/db/password'). [Learn More](https://docs.gitlab.com/ci/yaml/#secretsawsssmparameterstore)" + "description": "The ARN or name of the secret to retrieve. To retrieve a secret from another account, you must use an ARN." }, { "type": "object", - "markdownDescription": "Defines the secret version to be fetched from AWS Secrets Manager. The secret_id refers to the ARN or name of the secret in AWS Secrets Manager. Version_id and version_stage are optional parameters that can be used to specify a specific version of the secret, else AWSCURRENT version will be returned.", + "markdownDescription": "Defines the secret to be fetched from AWS Secrets Manager. The secret_id refers to the ARN or name of the secret in AWS Secrets Manager. Version_id and version_stage are optional parameters that can be used to specify a specific version of the secret, else AWSCURRENT version will be returned.", "properties": { "secret_id": { "type": "string", diff --git a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb index 1069dab53a6382..4f86fff8f8f8ac 100644 --- a/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb +++ b/ee/lib/gitlab/ci/config/entry/aws_secrets_manager/secret.rb @@ -23,13 +23,26 @@ class StringStrategy < ::Gitlab::Config::Entry::Node validations do validates :config, presence: true - validates :config, type: String + validates :config, type: String, + format: { with: /\A[^#]*(#[^#]*)?\z/, message: "must contain at most one '#'" } end def value - { - secret_id: config - } + # expect to return 1 element for secret without field and 2 elements for secret with field + parts = config.split('#') + + # input "/my/secret" + if parts.size == 1 + { + secret_id: parts[0] + } + # input "/my/secret#field" + else + { + secret_id: parts[0], + field: parts[1] + } + end end end diff --git a/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb index 87a90df4006ae2..b1495e5c6e08e8 100644 --- a/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/entry/aws_secrets_manager/secret_spec.rb @@ -60,6 +60,25 @@ end end end + + context 'when config is a string with field' do + let(:config) { 'production/db/password#field' } + + describe '#value' do + it 'returns AWS SecretsManager secret configuration' do + expect(entry.value).to eq({ + secret_id: 'production/db/password', + field: 'field' + }) + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end end end @@ -74,6 +93,16 @@ end end + context 'when config is a string and contains more than one #' do + let(:config) { 'production/db/password#field#foo' } + + it 'reports error' do + puts entry.errors + expect(entry.errors) + .to include "string strategy config must contain at most one '#'" + end + end + context 'when name is not present' do let(:config) { {} } -- GitLab