From 55221ac4d387cf0d385ee422cc51a87018a6ee45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20C=CC=8Cavoj?= Date: Tue, 21 Oct 2025 18:08:32 +0200 Subject: [PATCH 1/3] Add SEP policy source to job options Add `sha` and `project_id` to SEP-triggered jobs `options`. This data matches the job_options for PEP and will be used for id_tokens to verify the source of the policy-injected jobs. --- .../security/scan_execution_policy/config.rb | 19 ++++- .../ci_action/template.rb | 8 ++ ...and_scan_pipeline_configuration_service.rb | 9 ++- .../ci/pipeline/chain/set_build_sources.rb | 2 +- ee/lib/ee/gitlab/ci/yaml_processor/result.rb | 8 +- .../processor.rb | 48 ++++++++---- .../execution_policies/pipeline_context.rb | 4 + .../pipeline_context.rb | 39 +++++++--- .../pipeline/chain/set_build_sources_spec.rb | 2 +- .../gitlab/ci/yaml_processor/result_spec.rb | 11 +++ .../processor_spec.rb | 13 +++- .../pipeline_context_spec.rb | 26 +++++++ .../pipeline_context_spec.rb | 76 ++++++++++++++++--- .../scan_execution_policy/config_spec.rb | 20 ++++- .../ci_action/template_spec.rb | 19 +++++ ...can_pipeline_configuration_service_spec.rb | 25 ++++++ 16 files changed, 272 insertions(+), 57 deletions(-) diff --git a/ee/app/models/security/scan_execution_policy/config.rb b/ee/app/models/security/scan_execution_policy/config.rb index 100ce27d83eede..41892ee9c98df9 100644 --- a/ee/app/models/security/scan_execution_policy/config.rb +++ b/ee/app/models/security/scan_execution_policy/config.rb @@ -8,16 +8,29 @@ class Config DEFAULT_SKIP_CI_STRATEGY = { allowed: true }.freeze - attr_reader :actions, :skip_ci_strategy + attr_reader :actions, :configuration, :skip_ci_strategy - def initialize(policy:) - @actions = policy.fetch(:actions) + def initialize(policy:, configuration: nil) + @configuration = configuration @skip_ci_strategy = policy[:skip_ci].presence || DEFAULT_SKIP_CI_STRATEGY + @actions = policy.fetch(:actions, []).map { |action| action.merge(metadata: action_metadata) } end def skip_ci_allowed?(user_id) skip_ci_allowed_for_strategy?(skip_ci_strategy, user_id) end + + private + + delegate :security_policy_management_project_id, :configuration_sha, to: :configuration, allow_nil: true + + def action_metadata + # Metadata used for id_tokens. It matches the attributes in `pipeline_execution_context.job_options`. + { + project_id: security_policy_management_project_id, + sha: configuration_sha + }.compact + end end end end diff --git a/ee/app/services/security/security_orchestration_policies/ci_action/template.rb b/ee/app/services/security/security_orchestration_policies/ci_action/template.rb index 19ec4041b7ebcc..08b10e5f1ae95d 100644 --- a/ee/app/services/security/security_orchestration_policies/ci_action/template.rb +++ b/ee/app/services/security/security_orchestration_policies/ci_action/template.rb @@ -46,6 +46,7 @@ def config apply_defaults!(job_configuration, @action[:scan_settings]) remove_extends!(job_configuration) remove_rule_to_disable_job!(job_configuration) + apply_metadata!(job_configuration, @action[:metadata]) end ci_configuration @@ -95,6 +96,13 @@ def remove_rule_to_disable_job!(job_configuration) EXCLUDED_VARIABLES_PATTERNS.any? { |pattern| rule[:if]&.include?(pattern) } end end + + def apply_metadata!(job_configuration, action_metadata) + return if action_metadata.nil? + + # Store the policy configuration URI and SHA as a metadata in the job configuration, removed then in processor + job_configuration[:_metadata] = action_metadata + end end end end diff --git a/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb b/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb index 7378cfff9b5f20..7b9d414d1e40d0 100644 --- a/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb +++ b/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb @@ -43,9 +43,12 @@ def prepare_on_demand_scan_configuration(action) .merge(action[:tags] ? { tags: action[:tags] } : {}) .merge(ignore_default_before_after_script?(action) ? { before_script: [], after_script: [] } : {}) .deep_merge( - 'stage' => 'dast', - 'variables' => dast_on_demand_variables(template, action_variables), - 'dast_configuration' => ci_configuration['dast']['dast_configuration'] + { + 'stage' => 'dast', + 'variables' => dast_on_demand_variables(template, action_variables), + 'dast_configuration' => ci_configuration['dast']['dast_configuration'], + '_metadata' => action[:metadata] + }.compact ) end diff --git a/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb b/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb index a9ff6e0688abcc..5b787e7e36d088 100644 --- a/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb +++ b/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb @@ -16,7 +16,7 @@ def pipeline_execution_policy_build?(build) override :scan_execution_policy_build? def scan_execution_policy_build?(build) command.pipeline_policy_context.scan_execution_context(pipeline.source_ref_path) - .job_injected?(build) + .job_injected?(build.name) end end end diff --git a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb index 37a3a901c1c60d..c08ab4d437f4fd 100644 --- a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb +++ b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb @@ -18,15 +18,17 @@ def build_attributes(name) options: { dast_configuration: job[:dast_configuration], identity: job[:identity], - policy: execution_policy_job_options + policy: execution_policy_job_options(name) }.compact, secrets: job[:secrets] }.compact ) end - def execution_policy_job_options - ci_config&.pipeline_policy_context&.pipeline_execution_context&.job_options + def execution_policy_job_options(job_name) + return unless ci_config + + ci_config.pipeline_policy_context&.job_options(ref: ci_config.source_ref_path, job_name: job_name) end end end diff --git a/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb b/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb index c21b1361ce076d..1ae1cda0054f5f 100644 --- a/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb +++ b/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb @@ -93,37 +93,53 @@ def merge_policies_with_stages(config) def merge_on_demand_scan_template(merged_config, defined_stages) on_demand_scan_template = prepare_on_demand_scans_template + return if on_demand_scan_template.blank? + on_demand_scan_job_names = job_names(on_demand_scan_template.keys) + on_demand_scan_jobs_without_metadata = template_without_metadata(on_demand_scan_template) - if on_demand_scan_template.present? - insert_stage_before_or_append(defined_stages, DEFAULT_ON_DEMAND_STAGE, ['.post']) - merged_config.except!(*on_demand_scan_job_names).deep_merge!(on_demand_scan_template) - scan_execution_policy_context.collect_injected_job_names(on_demand_scan_job_names) - end + insert_stage_before_or_append(defined_stages, DEFAULT_ON_DEMAND_STAGE, ['.post']) + merged_config.except!(*on_demand_scan_job_names).deep_merge!(on_demand_scan_jobs_without_metadata) + scan_execution_policy_context.collect_injected_job_names_with_metadata( + fetch_job_names_and_metadata(on_demand_scan_template) + ) end def merge_pipeline_scan_template(merged_config, defined_stages) pipeline_scan_template = prepare_pipeline_scans_template - pipeline_scan_job_names = job_names(prepare_pipeline_scans_template.keys) - - if pipeline_scan_template.present? - unless defined_stages.include?(DEFAULT_SECURITY_JOB_STAGE) - insert_stage_after_or_prepend(defined_stages, DEFAULT_SCAN_POLICY_STAGE, ['.pre', DEFAULT_BUILD_STAGE]) - pipeline_scan_template = pipeline_scan_template.transform_values do |job_config| - job_config.merge(stage: DEFAULT_SCAN_POLICY_STAGE) - end - end + return if pipeline_scan_template.blank? - merged_config.except!(*pipeline_scan_job_names).deep_merge!(pipeline_scan_template) + pipeline_scan_job_names = job_names(pipeline_scan_template.keys) + scan_template_without_metadata = template_without_metadata(pipeline_scan_template) - scan_execution_policy_context.collect_injected_job_names(pipeline_scan_job_names) + unless defined_stages.include?(DEFAULT_SECURITY_JOB_STAGE) + insert_stage_after_or_prepend(defined_stages, DEFAULT_SCAN_POLICY_STAGE, ['.pre', DEFAULT_BUILD_STAGE]) + scan_template_without_metadata = scan_template_without_metadata.transform_values do |job_config| + job_config.merge(stage: DEFAULT_SCAN_POLICY_STAGE) + end end + + merged_config.except!(*pipeline_scan_job_names).deep_merge!(scan_template_without_metadata) + + scan_execution_policy_context.collect_injected_job_names_with_metadata( + fetch_job_names_and_metadata(pipeline_scan_template) + ) end def job_names(keys) keys - %i[variables] end + def template_without_metadata(template) + template.transform_values { |job_config| job_config.except(:_metadata) } + end + + def fetch_job_names_and_metadata(template) + template + .except(:variables, :stages, :workflow) + .transform_values { |job_config| job_config.delete(:_metadata) } + end + def insert_stage_after_or_prepend(stages, insert_stage_name, after_stages) stage_index = after_stages.filter_map { |stage| stages.index(stage) }.max diff --git a/ee/lib/gitlab/ci/pipeline/execution_policies/pipeline_context.rb b/ee/lib/gitlab/ci/pipeline/execution_policies/pipeline_context.rb index 4cd9cb5cf6cbca..b01f990096d0ef 100644 --- a/ee/lib/gitlab/ci/pipeline/execution_policies/pipeline_context.rb +++ b/ee/lib/gitlab/ci/pipeline/execution_policies/pipeline_context.rb @@ -57,6 +57,10 @@ def skip_ci_allowed?(ref:) pipeline_execution_context.skip_ci_allowed? && scan_execution_context(ref).skip_ci_allowed? end + def job_options(ref:, job_name:) + pipeline_execution_context.job_options || scan_execution_context(ref).job_options(job_name) + end + private attr_reader :project, :source, :current_user, :ref, :sha_context, :variables_attributes, :chat_data, diff --git a/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb b/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb index a655b9bb34e1bc..d7cc7a6ddbe398 100644 --- a/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb +++ b/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb @@ -13,7 +13,7 @@ def initialize(project:, ref:, current_user:, source:) @ref = ref @current_user = current_user @source = source - @injected_job_names = [] + @injected_job_names_with_metadata = {} end def has_scan_execution_policies? @@ -21,7 +21,10 @@ def has_scan_execution_policies? end def active_scan_execution_actions - policies.flat_map { |policy| limited_actions(policy.actions) }.compact.uniq + # If there are multiple SEP policies with the same scanner, + # we may end up with duplicates due to different metadata. + # Remove the duplicates by only taking unique scans. + policies.flat_map { |policy| limited_actions(policy.actions) }.compact.uniq { |action| action[:scan] } end strong_memoize_attr :active_scan_execution_actions @@ -31,12 +34,18 @@ def skip_ci_allowed? policies.all? { |policy| policy.skip_ci_allowed?(current_user&.id) } end - def collect_injected_job_names(job_names) - @injected_job_names.concat(job_names.map(&:to_s)) + def collect_injected_job_names_with_metadata(job_names_with_metadata) + @injected_job_names_with_metadata.merge!(job_names_with_metadata) end - def job_injected?(job) - @injected_job_names.include?(job.name) + def job_injected?(name) + @injected_job_names_with_metadata.key?(name.to_sym) + end + + def job_options(name) + return unless job_injected?(name) + + @injected_job_names_with_metadata[name.to_sym] end private @@ -61,13 +70,19 @@ def apply_scan_execution_policies? def policies return [] if valid_security_orchestration_policy_configurations.blank? - policies = valid_security_orchestration_policy_configurations - .flat_map do |configuration| - configuration.active_pipeline_policies_for_project(ref, project, source) - end.compact + configurations_with_policies = valid_security_orchestration_policy_configurations + .filter_map do |configuration| + [ + configuration, + configuration.active_pipeline_policies_for_project(ref, project, source) + ] + end - policies.map do |policy| - ::Security::ScanExecutionPolicy::Config.new(policy: policy) + configurations_with_policies.flat_map do |configuration, policies| + policies.map do |policy| + ::Security::ScanExecutionPolicy::Config + .new(policy: policy, configuration: configuration) + end end end strong_memoize_attr :policies diff --git a/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb b/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb index d86e98d6c924e3..0aceb1e6d651b1 100644 --- a/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb +++ b/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb @@ -72,7 +72,7 @@ pipeline_seed.stages.flat_map(&:statuses).each do |build| allow(scan_execution_context).to receive(:job_injected?) - .with(build) + .with(build.name) .and_return(expected_sources[build.name] == "scan_execution_policy") expect(build).to receive(:build_job_source).with( diff --git a/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb b/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb index 1ed1b5d47288ea..857738c34c6066 100644 --- a/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb +++ b/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb @@ -57,6 +57,17 @@ it 'does not mark the build as `policy` via :options' do expect(build.dig(:options, :policy)).to be_nil end + + context 'when job was injected by scan execution policies' do + before do + pipeline_policy_context.scan_execution_context('refs/heads/master') + .collect_injected_job_names_with_metadata(test: { sha: 'SEP SHA', project_id: 123 }) + end + + it 'saves the policy data in :options' do + expect(build.dig(:options, :policy)).to eq(sha: 'SEP SHA', project_id: 123) + end + end end end diff --git a/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb b/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb index 53f727c11f27e9..9f3f14cee59742 100644 --- a/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb +++ b/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb @@ -45,7 +45,8 @@ let(:policies) { {} } let_it_be(:namespace) { create(:group) } - let_it_be(:namespace_policies_repository) { create(:project, :repository) } + let_it_be(:policy_files) { { Security::OrchestrationPolicyConfiguration::POLICY_PATH => '' } } + let_it_be(:namespace_policies_repository) { create(:project, :custom_repo, files: policy_files) } let_it_be(:namespace_security_orchestration_policy_configuration) do create( :security_orchestration_policy_configuration, @@ -63,8 +64,7 @@ end let_it_be_with_refind(:project) { create(:project, :repository, group: namespace) } - - let_it_be(:policies_repository) { create(:project, :repository, group: namespace) } + let_it_be(:policies_repository) { create(:project, :custom_repo, files: policy_files, group: namespace) } let_it_be(:security_orchestration_policy_configuration) do create( :security_orchestration_policy_configuration, @@ -119,6 +119,13 @@ context "when #{ci_source} pipeline is created and affects CI status of the ref" do let(:source) { ci_source } + it 'collects the injected jobs and metadata in the pipeline context' do + perform_service + pipeline_context = pipeline_policy_context.scan_execution_context(ref) + expect(pipeline_context.job_injected?(extended_job)).to be(true) + expect(pipeline_context.job_options(extended_job)).to match a_hash_including(:project_id, :sha) + end + context 'when config already have jobs with names provided by policies' do let(:config) do { diff --git a/ee/spec/lib/gitlab/ci/pipeline/execution_policies/pipeline_context_spec.rb b/ee/spec/lib/gitlab/ci/pipeline/execution_policies/pipeline_context_spec.rb index 2d6ea7080ab49a..7e305ccefd5198 100644 --- a/ee/spec/lib/gitlab/ci/pipeline/execution_policies/pipeline_context_spec.rb +++ b/ee/spec/lib/gitlab/ci/pipeline/execution_policies/pipeline_context_spec.rb @@ -170,4 +170,30 @@ end end end + + describe '#job_options' do + subject { context.job_options(ref: pipeline.ref, job_name: 'test-job') } + + it { is_expected.to be_nil } + + context 'when there are pipeline execution policies' do + before do + allow(context.pipeline_execution_context).to receive(:job_options) + .and_return({ name: 'Policy', project_id: 123, sha: 'sha' }) + end + + it { is_expected.to eq(name: 'Policy', project_id: 123, sha: 'sha') } + end + + context 'when there are scan execution policies' do + before do + scan_execution_context_double = + instance_double(::Gitlab::Ci::Pipeline::ScanExecutionPolicies::PipelineContext, + job_options: { project_id: 123, sha: 'sha' }) + allow(context).to receive(:scan_execution_context).with(pipeline.ref).and_return(scan_execution_context_double) + end + + it { is_expected.to eq(project_id: 123, sha: 'sha') } + end + end end diff --git a/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb b/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb index 07db79861aca99..181be13291fe9e 100644 --- a/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb +++ b/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb @@ -13,7 +13,8 @@ let(:source) { 'push' } let(:pipeline) { build(:ci_pipeline, source: source, project: project, ref: ref, user: user) } - let_it_be(:policies_repository) { create(:project, :repository) } + let_it_be(:policy_files) { { Security::OrchestrationPolicyConfiguration::POLICY_PATH => '' } } + let_it_be(:policies_repository) { create(:project, :custom_repo, files: policy_files) } let(:feature_licensed) { true } let_it_be(:security_orchestration_policy_configuration) do create( @@ -23,6 +24,13 @@ ) end + let(:expected_metadata) do + { + sha: security_orchestration_policy_configuration.configuration_sha, + project_id: policies_repository.id + } + end + let(:policy) do build(:scan_execution_policy, actions: [ { scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile' }, @@ -107,7 +115,14 @@ describe '#active_scan_execution_actions' do subject(:actions) { context.active_scan_execution_actions } - it { is_expected.to match_array(policy[:actions]) } + it 'contains active actions with metadata' do + expect(actions).to match_array([ + { scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile', + metadata: expected_metadata }, + { scan: 'secret_detection', metadata: expected_metadata }, + { scan: 'dependency_scanning', metadata: expected_metadata } + ]) + end describe 'action limits' do let(:policies) { [policy, other_policy] } @@ -120,19 +135,35 @@ end let(:action_limit) { 2 } - let(:all_actions) { policy[:actions] + other_policy[:actions] } - let(:limited_actions) { policy[:actions].first(action_limit) + other_policy[:actions].first(action_limit) } before do allow(Gitlab::CurrentSettings).to receive(:scan_execution_policies_action_limit).and_return(action_limit) end - it { is_expected.to match_array(limited_actions) } + it 'contains active actions with metadata' do + expect(actions).to match_array([ + { scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile', + metadata: expected_metadata }, + { scan: 'secret_detection', metadata: expected_metadata }, + { scan: 'sast', metadata: expected_metadata }, + { scan: 'sast_iac', metadata: expected_metadata } + ]) + end context 'when value is zero' do let(:action_limit) { 0 } - it { is_expected.to match_array(all_actions) } + it 'contains active actions with metadata' do + expect(actions).to match_array([ + { scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile', + metadata: expected_metadata }, + { scan: 'secret_detection', metadata: expected_metadata }, + { scan: 'dependency_scanning', metadata: expected_metadata }, + { scan: 'sast', metadata: expected_metadata }, + { scan: 'sast_iac', metadata: expected_metadata }, + { scan: 'container_scanning', metadata: expected_metadata } + ]) + end end end @@ -154,7 +185,12 @@ let(:policy_pipeline_source) { source } it 'returns the active scan execution actions' do - expect(context.active_scan_execution_actions).to match_array(policy[:actions]) + expect(context.active_scan_execution_actions).to match_array([ + { scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile', + metadata: expected_metadata }, + { scan: 'secret_detection', metadata: expected_metadata }, + { scan: 'dependency_scanning', metadata: expected_metadata } + ]) end end @@ -218,12 +254,28 @@ end describe '#job_injected?' do - it 'stores array of job names' do - context.collect_injected_job_names([:job1, "job-2"]) + it 'stores hash of job names with metadata' do + context.collect_injected_job_names_with_metadata({ + job1: { some_key: 'value' }, + 'job-2': { some_other_key: 'other value ' } + }) + + expect(context.job_injected?('job1')).to be(true) + expect(context.job_injected?('job-2')).to be(true) + expect(context.job_injected?('job3')).to be(false) + end + end + + describe '#job_options' do + it 'returns the collected metadata' do + context.collect_injected_job_names_with_metadata({ + job1: { some_key: 'value' }, + 'job-2': { some_other_key: 'other value' } + }) - expect(context.job_injected?(instance_double(::Ci::Build, name: 'job1'))).to be(true) - expect(context.job_injected?(instance_double(::Ci::Build, name: 'job-2'))).to be(true) - expect(context.job_injected?(instance_double(::Ci::Build, name: 'job3'))).to be(false) + expect(context.job_options('job1')).to eq(some_key: 'value') + expect(context.job_options('job-2')).to eq(some_other_key: 'other value') + expect(context.job_options('job3')).to be_nil end end end diff --git a/ee/spec/models/security/scan_execution_policy/config_spec.rb b/ee/spec/models/security/scan_execution_policy/config_spec.rb index 1f354f82faa331..872862cc525fc3 100644 --- a/ee/spec/models/security/scan_execution_policy/config_spec.rb +++ b/ee/spec/models/security/scan_execution_policy/config_spec.rb @@ -3,15 +3,29 @@ require 'spec_helper' RSpec.describe Security::ScanExecutionPolicy::Config, feature_category: :security_policy_management do + let_it_be(:policy_files) { { Security::OrchestrationPolicyConfiguration::POLICY_PATH => '' } } + let_it_be(:security_policy_project) { create(:project, :custom_repo, files: policy_files) } + let_it_be(:security_orchestration_policy_configuration) do + create(:security_orchestration_policy_configuration, security_policy_management_project: security_policy_project) + end + let(:config) { described_class.new(**params) } - let(:params) { { policy: policy } } + let(:params) { { policy: policy, configuration: security_orchestration_policy_configuration } } describe '#actions' do - subject { config.actions } + subject(:actions) { config.actions } let(:policy) { build(:scan_execution_policy, actions: [{ scan: 'secret_detection' }]) } - it { is_expected.to eq([{ scan: 'secret_detection' }]) } + it 'returns actions with configuration metadata' do + expect(actions).to eq([{ + scan: 'secret_detection', + metadata: { + project_id: security_policy_project.id, + sha: security_orchestration_policy_configuration.configuration_sha + } + }]) + end end describe '#skip_ci_allowed?' do diff --git a/ee/spec/services/security/security_orchestration_policies/ci_action/template_spec.rb b/ee/spec/services/security/security_orchestration_policies/ci_action/template_spec.rb index c8d317a8c962b6..e0ac25fd0bd960 100644 --- a/ee/spec/services/security/security_orchestration_policies/ci_action/template_spec.rb +++ b/ee/spec/services/security/security_orchestration_policies/ci_action/template_spec.rb @@ -75,6 +75,23 @@ end end + shared_examples 'with metadata added to action' do + let(:action_metadata) do + { + sha: '8dc6668b57dbeec74feb7c68119cf0c28cbe2a29', + project_id: 123 + } + end + + before do + action.merge!(metadata: action_metadata) + end + + it 'adds _metadata section to each job' do + expect(config.values).to all(include(_metadata: action_metadata)) + end + end + describe '.scan_template_path' do let(:scan_type) { 'dependency_scanning' } let(:template) { 'default' } @@ -138,6 +155,7 @@ it_behaves_like 'with template name for scan type' it_behaves_like 'removes rules which disable jobs' it_behaves_like 'with scan_settings.ignore_default_before_after_script set to true' + it_behaves_like 'with metadata added to action' it 'merges template variables with ci variables and returns them as string' do expect(config[:'secret-detection-0']).to include( @@ -190,6 +208,7 @@ it_behaves_like 'with template name for scan type' it_behaves_like 'removes rules which disable jobs' it_behaves_like 'with scan_settings.ignore_default_before_after_script set to true' + it_behaves_like 'with metadata added to action' it 'merges template variables with ci variables and returns them as string' do expect(config[:'container-scanning-0']).to include( diff --git a/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb b/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb index 5627ad97098787..12a49305851efd 100644 --- a/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb +++ b/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb @@ -168,6 +168,31 @@ end end + context 'when metadata is added to action' do + let(:action_metadata) do + { + sha: '8dc6668b57dbeec74feb7c68119cf0c28cbe2a29', + project_id: 123 + } + end + + let(:actions) do + [ + { + scan: 'dast', + site_profile: site_profile.name, + scanner_profile: scanner_profile.name, + tags: ['runner-tag'], + metadata: action_metadata + } + ] + end + + it 'adds _metadata section to job configuration' do + expect(pipeline_configuration[:'dast-on-demand-0']).to include(_metadata: action_metadata) + end + end + describe "variable injection and precedence" do let(:actions) do [ -- GitLab From 4374e03f3d9ef382db4d3671a9156785c3b3be00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20C=CC=8Cavoj?= Date: Fri, 12 Dec 2025 14:34:26 +0100 Subject: [PATCH 2/3] Refactoring to remove duplication and make methods more readable --- .../ci_action/template.rb | 11 ++---- .../ci_configuration_metadata.rb | 14 +++++++ ...and_scan_pipeline_configuration_service.rb | 12 +++--- .../ci/pipeline/chain/set_build_sources.rb | 4 +- .../processor.rb | 39 +++++++------------ .../pipeline_context.rb | 17 +++++--- .../pipeline/chain/set_build_sources_spec.rb | 12 ++++++ .../gitlab/ci/yaml_processor/result_spec.rb | 2 +- .../pipeline_context_spec.rb | 8 ++-- .../set_build_sources_spec.rb | 4 +- ...can_pipeline_configuration_service_spec.rb | 6 +++ 11 files changed, 77 insertions(+), 52 deletions(-) create mode 100644 ee/app/services/security/security_orchestration_policies/ci_configuration_metadata.rb diff --git a/ee/app/services/security/security_orchestration_policies/ci_action/template.rb b/ee/app/services/security/security_orchestration_policies/ci_action/template.rb index 08b10e5f1ae95d..3bb07d5d471121 100644 --- a/ee/app/services/security/security_orchestration_policies/ci_action/template.rb +++ b/ee/app/services/security/security_orchestration_policies/ci_action/template.rb @@ -4,6 +4,8 @@ module Security module SecurityOrchestrationPolicies module CiAction class Template < Base + include CiConfigurationMetadata + SCAN_TEMPLATES = { 'secret_detection' => 'Jobs/Secret-Detection', 'container_scanning' => 'Jobs/Container-Scanning', @@ -46,7 +48,7 @@ def config apply_defaults!(job_configuration, @action[:scan_settings]) remove_extends!(job_configuration) remove_rule_to_disable_job!(job_configuration) - apply_metadata!(job_configuration, @action[:metadata]) + merge_configuration_metadata!(job_configuration, @action[:metadata]) end ci_configuration @@ -96,13 +98,6 @@ def remove_rule_to_disable_job!(job_configuration) EXCLUDED_VARIABLES_PATTERNS.any? { |pattern| rule[:if]&.include?(pattern) } end end - - def apply_metadata!(job_configuration, action_metadata) - return if action_metadata.nil? - - # Store the policy configuration URI and SHA as a metadata in the job configuration, removed then in processor - job_configuration[:_metadata] = action_metadata - end end end end diff --git a/ee/app/services/security/security_orchestration_policies/ci_configuration_metadata.rb b/ee/app/services/security/security_orchestration_policies/ci_configuration_metadata.rb new file mode 100644 index 00000000000000..ac7d62bddf1b93 --- /dev/null +++ b/ee/app/services/security/security_orchestration_policies/ci_configuration_metadata.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Security + module SecurityOrchestrationPolicies + module CiConfigurationMetadata + def merge_configuration_metadata!(config, metadata) + return config if metadata.nil? + + # Store the policy project ID and SHA as a metadata in the job configuration, removed then in processor + config[:_metadata] = metadata + end + end + end +end diff --git a/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb b/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb index 7b9d414d1e40d0..ffaefe8fbc9f31 100644 --- a/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb +++ b/ee/app/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service.rb @@ -4,6 +4,7 @@ module Security module SecurityOrchestrationPolicies class OnDemandScanPipelineConfigurationService include Gitlab::Utils::StrongMemoize + include CiConfigurationMetadata def initialize(project) @project = project @@ -33,7 +34,7 @@ def prepare_on_demand_scan_configuration(action) scanner_profile = dast_scanner_profile(action[:scanner_profile]) result = prepare_base_configuration(site_profile, scanner_profile) - return error_script(result.message) unless result.success? + return error_script(result.message, action) unless result.success? action_variables = action[:variables].to_h.stringify_keys ci_configuration = YAML.safe_load(result.payload[:ci_configuration]) @@ -46,10 +47,9 @@ def prepare_on_demand_scan_configuration(action) { 'stage' => 'dast', 'variables' => dast_on_demand_variables(template, action_variables), - 'dast_configuration' => ci_configuration['dast']['dast_configuration'], - '_metadata' => action[:metadata] + 'dast_configuration' => ci_configuration['dast']['dast_configuration'] }.compact - ) + ).tap { |config| merge_configuration_metadata!(config, action[:metadata]) } end def dast_site_profile(site_profile_name) @@ -95,11 +95,11 @@ def dast_on_demand_variables(template, action_variables) .merge(action_variables) end - def error_script(error_message) + def error_script(error_message, action) { 'script' => "echo \"Error during On-Demand Scan execution: #{error_message}\" && false", 'allow_failure' => true - } + }.tap { |config| merge_configuration_metadata!(config, action[:metadata]) } end end end diff --git a/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb b/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb index 5b787e7e36d088..3e41880f25212b 100644 --- a/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb +++ b/ee/lib/ee/gitlab/ci/pipeline/chain/set_build_sources.rb @@ -9,8 +9,8 @@ module SetBuildSources extend ::Gitlab::Utils::Override override :pipeline_execution_policy_build? - def pipeline_execution_policy_build?(build) - build.options&.dig(:policy).present? + def pipeline_execution_policy_build?(_build) + command.pipeline_policy_context.pipeline_execution_context.creating_policy_pipeline? end override :scan_execution_policy_build? diff --git a/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb b/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb index 1ae1cda0054f5f..fdd19d61dc78a6 100644 --- a/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb +++ b/ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb @@ -92,41 +92,36 @@ def merge_policies_with_stages(config) end def merge_on_demand_scan_template(merged_config, defined_stages) - on_demand_scan_template = prepare_on_demand_scans_template - return if on_demand_scan_template.blank? + template = prepare_on_demand_scans_template + return if template.blank? - on_demand_scan_job_names = job_names(on_demand_scan_template.keys) - on_demand_scan_jobs_without_metadata = template_without_metadata(on_demand_scan_template) + job_names = extract_job_names(template.keys) + jobs_without_metadata = template_without_metadata(template) insert_stage_before_or_append(defined_stages, DEFAULT_ON_DEMAND_STAGE, ['.post']) - merged_config.except!(*on_demand_scan_job_names).deep_merge!(on_demand_scan_jobs_without_metadata) - scan_execution_policy_context.collect_injected_job_names_with_metadata( - fetch_job_names_and_metadata(on_demand_scan_template) - ) + merged_config.except!(*job_names).deep_merge!(jobs_without_metadata) + scan_execution_policy_context.collect_injected_job_names_with_metadata(template) end def merge_pipeline_scan_template(merged_config, defined_stages) - pipeline_scan_template = prepare_pipeline_scans_template - return if pipeline_scan_template.blank? + template = prepare_pipeline_scans_template + return if template.blank? - pipeline_scan_job_names = job_names(pipeline_scan_template.keys) - scan_template_without_metadata = template_without_metadata(pipeline_scan_template) + job_names = extract_job_names(template.keys) + jobs_without_metadata = template_without_metadata(template) unless defined_stages.include?(DEFAULT_SECURITY_JOB_STAGE) insert_stage_after_or_prepend(defined_stages, DEFAULT_SCAN_POLICY_STAGE, ['.pre', DEFAULT_BUILD_STAGE]) - scan_template_without_metadata = scan_template_without_metadata.transform_values do |job_config| + jobs_without_metadata = jobs_without_metadata.transform_values do |job_config| job_config.merge(stage: DEFAULT_SCAN_POLICY_STAGE) end end - merged_config.except!(*pipeline_scan_job_names).deep_merge!(scan_template_without_metadata) - - scan_execution_policy_context.collect_injected_job_names_with_metadata( - fetch_job_names_and_metadata(pipeline_scan_template) - ) + merged_config.except!(*job_names).deep_merge!(jobs_without_metadata) + scan_execution_policy_context.collect_injected_job_names_with_metadata(template) end - def job_names(keys) + def extract_job_names(keys) keys - %i[variables] end @@ -134,12 +129,6 @@ def template_without_metadata(template) template.transform_values { |job_config| job_config.except(:_metadata) } end - def fetch_job_names_and_metadata(template) - template - .except(:variables, :stages, :workflow) - .transform_values { |job_config| job_config.delete(:_metadata) } - end - def insert_stage_after_or_prepend(stages, insert_stage_name, after_stages) stage_index = after_stages.filter_map { |stage| stages.index(stage) }.max diff --git a/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb b/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb index d7cc7a6ddbe398..c681ec655626ec 100644 --- a/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb +++ b/ee/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context.rb @@ -13,7 +13,7 @@ def initialize(project:, ref:, current_user:, source:) @ref = ref @current_user = current_user @source = source - @injected_job_names_with_metadata = {} + @injected_job_names_metadata_map = {} end def has_scan_execution_policies? @@ -34,18 +34,19 @@ def skip_ci_allowed? policies.all? { |policy| policy.skip_ci_allowed?(current_user&.id) } end - def collect_injected_job_names_with_metadata(job_names_with_metadata) - @injected_job_names_with_metadata.merge!(job_names_with_metadata) + def collect_injected_job_names_with_metadata(template_with_metadata) + job_name_with_metadata = extract_job_names_and_metadata(template_with_metadata) + @injected_job_names_metadata_map.merge!(job_name_with_metadata) end def job_injected?(name) - @injected_job_names_with_metadata.key?(name.to_sym) + @injected_job_names_metadata_map.key?(name.to_sym) end def job_options(name) return unless job_injected?(name) - @injected_job_names_with_metadata[name.to_sym] + @injected_job_names_metadata_map[name.to_sym] end private @@ -91,6 +92,12 @@ def valid_security_orchestration_policy_configurations @valid_security_orchestration_policy_configurations ||= ::Gitlab::Security::Orchestration::ProjectPolicyConfigurations.new(project).all end + + def extract_job_names_and_metadata(template) + template + .except(*Gitlab::Ci::Config::Entry::Root::ALLOWED_KEYS) + .transform_values { |job_config| job_config.delete(:_metadata) } + end end end end diff --git a/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb b/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb index 0aceb1e6d651b1..2560a1f3f74c97 100644 --- a/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb +++ b/ee/spec/lib/ee/gitlab/ci/pipeline/chain/set_build_sources_spec.rb @@ -50,6 +50,10 @@ end context 'with security policy' do + let(:pipeline_execution_context) do + instance_double(::Gitlab::Ci::Pipeline::PipelineExecutionPolicies::PipelineContext) + end + let(:scan_execution_context) do instance_double(::Gitlab::Ci::Pipeline::ScanExecutionPolicies::PipelineContext) end @@ -70,6 +74,14 @@ .at_least(:once) .and_return(scan_execution_context) + expect(command.pipeline_policy_context).to receive(:pipeline_execution_context) + .at_least(:once) + .and_return(pipeline_execution_context) + + allow(pipeline_execution_context).to receive(:creating_policy_pipeline?) + .exactly(7).times + .and_return(false, true, false, false, true, true, false) + pipeline_seed.stages.flat_map(&:statuses).each do |build| allow(scan_execution_context).to receive(:job_injected?) .with(build.name) diff --git a/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb b/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb index 857738c34c6066..c52957e36e768c 100644 --- a/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb +++ b/ee/spec/lib/ee/gitlab/ci/yaml_processor/result_spec.rb @@ -61,7 +61,7 @@ context 'when job was injected by scan execution policies' do before do pipeline_policy_context.scan_execution_context('refs/heads/master') - .collect_injected_job_names_with_metadata(test: { sha: 'SEP SHA', project_id: 123 }) + .collect_injected_job_names_with_metadata(test: { _metadata: { sha: 'SEP SHA', project_id: 123 } }) end it 'saves the policy data in :options' do diff --git a/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb b/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb index 181be13291fe9e..519106bdfe1b73 100644 --- a/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb +++ b/ee/spec/lib/gitlab/ci/pipeline/scan_execution_policies/pipeline_context_spec.rb @@ -254,7 +254,7 @@ end describe '#job_injected?' do - it 'stores hash of job names with metadata' do + it 'returns the collected job names' do context.collect_injected_job_names_with_metadata({ job1: { some_key: 'value' }, 'job-2': { some_other_key: 'other value ' } @@ -267,14 +267,14 @@ end describe '#job_options' do - it 'returns the collected metadata' do + it 'returns the metadata corresponding to the collected jobs, ignoring other attributes' do context.collect_injected_job_names_with_metadata({ - job1: { some_key: 'value' }, + job1: { _metadata: { some_key: 'value' }, some_other_key: 'value2' }, 'job-2': { some_other_key: 'other value' } }) expect(context.job_options('job1')).to eq(some_key: 'value') - expect(context.job_options('job-2')).to eq(some_other_key: 'other value') + expect(context.job_options('job-2')).to be_nil expect(context.job_options('job3')).to be_nil end end diff --git a/ee/spec/services/ci/create_pipeline_service/set_build_sources_spec.rb b/ee/spec/services/ci/create_pipeline_service/set_build_sources_spec.rb index ff51ee133b0402..6a61f0d4e35ad7 100644 --- a/ee/spec/services/ci/create_pipeline_service/set_build_sources_spec.rb +++ b/ee/spec/services/ci/create_pipeline_service/set_build_sources_spec.rb @@ -143,7 +143,9 @@ pipeline.builds.each do |build| source = Ci::BuildSource.find_by(build_id: build.id, project_id: project.id) - expect(source.source).to eq(expected_sources[build.name] || pipeline.source) + expected = expected_sources[build.name] || pipeline.source + expect(source.source).to eq(expected), + "expected source for build #{build.name} to match #{expected}, but it was #{source.source}" end end end diff --git a/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb b/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb index 12a49305851efd..9776d4ce8e2a2d 100644 --- a/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb +++ b/ee/spec/services/security/security_orchestration_policies/on_demand_scan_pipeline_configuration_service_spec.rb @@ -184,12 +184,18 @@ scanner_profile: scanner_profile.name, tags: ['runner-tag'], metadata: action_metadata + }, + { + scan: 'dast', + site_profile: 'Site Profile B', + metadata: action_metadata } ] end it 'adds _metadata section to job configuration' do expect(pipeline_configuration[:'dast-on-demand-0']).to include(_metadata: action_metadata) + expect(pipeline_configuration[:'dast-on-demand-1']).to include(_metadata: action_metadata) end end -- GitLab From e986d211253863e6cb5f297926b775b7437177f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20C=CC=8Cavoj?= Date: Mon, 15 Dec 2025 10:12:05 +0100 Subject: [PATCH 3/3] Remove unnecessary nil check for ci_config --- ee/lib/ee/gitlab/ci/yaml_processor/result.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb index c08ab4d437f4fd..f6bca2d76dc118 100644 --- a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb +++ b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb @@ -26,8 +26,6 @@ def build_attributes(name) end def execution_policy_job_options(job_name) - return unless ci_config - ci_config.pipeline_policy_context&.job_options(ref: ci_config.source_ref_path, job_name: job_name) end end -- GitLab