diff --git a/app/services/ci/pipeline_creation/find_pipeline_inputs_service.rb b/app/services/ci/pipeline_creation/find_pipeline_inputs_service.rb index 363d4cf42da9472537fe25269bfa43961bb8838b..89562fc0b8a2660feffee649eea7929386227a75 100644 --- a/app/services/ci/pipeline_creation/find_pipeline_inputs_service.rb +++ b/app/services/ci/pipeline_creation/find_pipeline_inputs_service.rb @@ -39,7 +39,10 @@ def execute yaml_result = yaml_result_of_internal_include(yaml_content) return error_response(s_('Pipelines|Invalid YAML syntax')) unless yaml_result&.valid? - spec_inputs = Ci::Inputs::Builder.new(yaml_result.spec[:inputs]) + # Process header includes to merge external input definitions + spec = process_header_includes(yaml_result.spec) + + spec_inputs = Ci::Inputs::Builder.new(spec[:inputs]) return error_response(spec_inputs.errors.join(', ')) if spec_inputs.errors.any? success_response(spec_inputs) @@ -48,6 +51,8 @@ def execute end rescue ::Gitlab::Ci::Config::Yaml::LoadError => e error_response("YAML load error: #{e.message}") + rescue ::Gitlab::Ci::Config::External::Header::Processor::IncludeError => e + error_response(e.message) end private @@ -91,6 +96,14 @@ def sha project.commit(ref)&.sha end strong_memoize_attr :sha + + def process_header_includes(spec) + return spec unless Feature.enabled?(:ci_file_inputs, project) + return spec unless spec[:include].present? + + processor = ::Gitlab::Ci::Config::External::Header::Processor.new(spec, context) + processor.perform + end end end end diff --git a/config/feature_flags/gitlab_com_derisk/ci_file_inputs.yml b/config/feature_flags/gitlab_com_derisk/ci_file_inputs.yml new file mode 100644 index 0000000000000000000000000000000000000000..4ad3ce973cee38dfebd7f85566cd5a9f07c2284c --- /dev/null +++ b/config/feature_flags/gitlab_com_derisk/ci_file_inputs.yml @@ -0,0 +1,10 @@ +--- +name: ci_file_inputs +description: +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/415636 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/206931 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/579240 +milestone: '18.6' +group: group::pipeline authoring +type: gitlab_com_derisk +default_enabled: false diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index dbbe60ec31fbd4d0403c708d229130b7590b3386..94a8ccc331c8e2fe3a7dd9e44c3a1b52142f78a3 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -174,8 +174,10 @@ def expand_config(config, inputs) def build_config(config, inputs) initial_config = logger.instrument(:config_yaml_load, once: true) do - yaml_context = Config::Yaml::Context.new(variables: @context.variables) - Config::Yaml.load!(config, yaml_context, inputs) + yaml_context = Config::Yaml::Context.new( + variables: @context.variables + ) + Config::Yaml.load!(config, yaml_context, inputs, @context) end initial_config = logger.instrument(:config_external_process, once: true) do diff --git a/lib/gitlab/ci/config/entry/concerns/base_include.rb b/lib/gitlab/ci/config/entry/concerns/base_include.rb index 8817d6ffd10828226f0869360de6533603405edd..4c54ca8272b6544732fc6f80370695359c34e507 100644 --- a/lib/gitlab/ci/config/entry/concerns/base_include.rb +++ b/lib/gitlab/ci/config/entry/concerns/base_include.rb @@ -17,13 +17,13 @@ module Concerns module BaseInclude extend ActiveSupport::Concern - COMMON_ALLOWED_KEYS = %i[local file remote project component integrity].freeze + COMMON_ALLOWED_KEYS = %i[local file remote project ref integrity].freeze included do include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - attributes :local, :file, :remote, :project, :component, :integrity + attributes :local, :file, :remote, :project, :ref, :component, :integrity validations do validates :config, hash_or_string: true diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index a8ee967a84336d5243864a7feee2fc0300c6b281..a675f66f8d6e6876bced008a07c8786226e89604 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -13,7 +13,7 @@ class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Concerns::BaseInclude # Additional keys beyond the common ones - ADDITIONAL_ALLOWED_KEYS = %i[template artifact inputs job ref rules].freeze + ADDITIONAL_ALLOWED_KEYS = %i[template artifact inputs job rules component].freeze ALLOWED_KEYS = (COMMON_ALLOWED_KEYS + ADDITIONAL_ALLOWED_KEYS).freeze validations do diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index f7978b5822386dc232ddee35ec0817eccb9e6f49..911f822978ac2cfc77c251a7609cc97af93db22e 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,8 +9,6 @@ class Context TimeoutError = Class.new(StandardError) - include ::Gitlab::Utils::StrongMemoize - attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config, :parallel_requests, :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes, :pipeline_policy_context, :component_data diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 2cba0eff695b16dbf8dcbb3c96ea67fb33d46690..88872b7de13b178b0700e3841404a046ba8f6b36 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -16,6 +16,16 @@ def initialize(params, context) @params = params @context = context @errors = [] + @inputs_only = false + end + + def inputs_only! + @inputs_only = true + self + end + + def inputs_only? + @inputs_only end def matching? @@ -102,6 +112,18 @@ def load_and_validate_expanded_hash! end validate_hash! + validate_content_keys! if inputs_only? + end + + def validate_content_keys! + return unless expanded_content_hash + + allowed_keys = %i[inputs] + unknown_keys = expanded_content_hash.keys - allowed_keys + + return unless unknown_keys.any? + + errors.push("Header include file `#{masked_location}` contains unknown keys: #{unknown_keys}") end def load_uninterpolated_yaml @@ -117,7 +139,7 @@ def content_inputs def content_result context.logger.instrument(:config_file_fetch_content_hash) do ::Gitlab::Ci::Config::Yaml::Loader.new( - content, inputs: content_inputs, context: yaml_context + content, inputs: content_inputs, context: yaml_context, external_context: context ).load end end @@ -149,6 +171,8 @@ def validate_hash! end def expand_includes(hash) + return hash if inputs_only? + External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end diff --git a/lib/gitlab/ci/config/external/header/mapper.rb b/lib/gitlab/ci/config/external/header/mapper.rb new file mode 100644 index 0000000000000000000000000000000000000000..6f149e23a7879d27f29df442163d66783c98abe0 --- /dev/null +++ b/lib/gitlab/ci/config/external/header/mapper.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module Header + ## + # This method just gets the files using the header-specific Matcher and verifies them. + class Mapper < ::Gitlab::Ci::Config::External::Mapper + private + + def get_files_and_verify_locations(locations) + files = Header::Mapper::Matcher.new(context).process(locations) + External::Mapper::Verifier.new(context).process(files) + + files + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/header/mapper/matcher.rb b/lib/gitlab/ci/config/external/header/mapper/matcher.rb new file mode 100644 index 0000000000000000000000000000000000000000..7d3f4453b9173fb67836fc94171f38127351bbf6 --- /dev/null +++ b/lib/gitlab/ci/config/external/header/mapper/matcher.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module Header + class Mapper + ## + # Header-specific matcher that reuses the core matching logic + # but only matches header-compatible file types + class Matcher < ::Gitlab::Ci::Config::External::Mapper::Matcher + private + + def new_file_class(file_class, location) + file_class.new(location, context).inputs_only! + end + + # Override to provide header-compatible file classes + def file_classes + [ + External::File::Local, + External::File::Remote, + External::File::Project + ] + end + + def include_type + 'header include' + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/header/processor.rb b/lib/gitlab/ci/config/external/header/processor.rb new file mode 100644 index 0000000000000000000000000000000000000000..f06d292e8b19bde50f5856323ff40cee026876bc --- /dev/null +++ b/lib/gitlab/ci/config/external/header/processor.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module Header + ## + # Header include processor + # + # Processes header includes to merge input definitions. + # Inherits from External::Processor and overrides only the Mapper instantiation. + class Processor < ::Gitlab::Ci::Config::External::Processor + private + + def mapper + Header::Mapper.new(@values, @context) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index fcd65b2f2d7e33bd4d7d8429a7a8b1d826080be9..dd483fd486107705f1c279fa06ff1b81f2960947 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -38,6 +38,10 @@ def process_without_instrumentation locations = LocationExpander.new(context).process(locations) locations = VariablesExpander.new(context).process(locations) + get_files_and_verify_locations(locations) + end + + def get_files_and_verify_locations(locations) files = Matcher.new(context).process(locations) Verifier.new(context).process(files) diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index f9d5b0ebd015d8f480c0337ab5d8ce63817863f7..9a9ce9116bcb9815a9a590e5396ffbb408ecb0f9 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -14,14 +14,14 @@ class Matcher < Base def process_without_instrumentation(locations) locations.map do |location| matching = file_classes.map do |file_class| - file_class.new(location, context) + new_file_class(file_class, location) end.select(&:matching?) if matching.one? matching.first elsif matching.empty? raise Mapper::AmbigiousSpecificationError, - "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "`#{masked_location(location.to_json)}` does not have a valid subkey for #{include_type}. " \ "Valid subkeys are: `#{file_subkeys.join('`, `')}`" else raise Mapper::AmbigiousSpecificationError, @@ -30,6 +30,10 @@ def process_without_instrumentation(locations) end end + def new_file_class(file_class, location) + file_class.new(location, context) + end + def masked_location(location) context.mask_variables_from(location) end @@ -39,6 +43,10 @@ def file_subkeys end strong_memoize_attr :file_subkeys + def include_type + 'include' + end + def file_classes [ External::File::Local, diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 9d7ae0189de4ad0b1ba9bcfb9e2aa4d2ce38ba58..bfd04781f29e4d5746a6a77c43d115edc5920d52 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -11,7 +11,8 @@ class Processor def initialize(values, context) @values = values - @external_files = External::Mapper.new(values, context).process + @context = context + @external_files = mapper.process @content = {} @logger = context.logger rescue External::Mapper::Error, @@ -30,6 +31,10 @@ def perform private + def mapper + External::Mapper.new(@values, @context) + end + def validate_external_files! @external_files.each do |file| raise IncludeError, file.error_message unless file.valid? diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb index 32e034e319774fe7cc0abbbce3e0ab4a4c882d23..265753c41c39d60ef28883570b371e2f2c668080 100644 --- a/lib/gitlab/ci/config/header/spec.rb +++ b/lib/gitlab/ci/config/header/spec.rb @@ -7,10 +7,17 @@ module Header class Spec < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[inputs component].freeze + ALLOWED_KEYS = %i[inputs include component].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + + validate on: :composed do + if config.is_a?(Hash) && config.key?(:include) && + !Gitlab::Ci::Config::FeatureFlags.enabled?(:ci_file_inputs) + errors.add(:config, "contains unknown keys: include") + end + end end entry :inputs, ::Gitlab::Config::Entry::ComposableHash, @@ -18,6 +25,10 @@ class Spec < ::Gitlab::Config::Entry::Node inherit: false, metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input } + entry :include, ::Gitlab::Ci::Config::Header::Includes, + description: 'List of input files to include.', + inherit: false + entry :component, Header::Component, description: 'The available component context used for interpolation.', inherit: false, diff --git a/lib/gitlab/ci/config/interpolation/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb index 4589be2a45208243284c2ff3cf76b2388889d8aa..a170fa7658ebebab314f662bed2c5aa150ba5438 100644 --- a/lib/gitlab/ci/config/interpolation/interpolator.rb +++ b/lib/gitlab/ci/config/interpolation/interpolator.rb @@ -8,12 +8,13 @@ module Interpolation # Performs CI config file interpolation, and surfaces all possible interpolation errors. # class Interpolator - attr_reader :config, :args, :yaml_context, :errors + attr_reader :config, :args, :yaml_context, :external_context, :errors - def initialize(config, args, yaml_context) + def initialize(config, args, yaml_context, external_context: nil) @config = config @args = args.nil? ? {} : args @yaml_context = yaml_context + @external_context = external_context @errors = [] @interpolated = false end @@ -49,6 +50,8 @@ def interpolate! return @errors.concat(header.errors) unless header.valid? return @errors.concat(inputs.errors) unless inputs.valid? + + return if @errors.any? return @errors.concat(context.errors) unless context.valid? return @errors.concat(template.errors) unless template.valid? @@ -68,7 +71,7 @@ def inputs_without_header? end def header - @entry ||= Header::Root.new(config.header).tap do |header| + @entry ||= Header::Root.new(config.header || {}).tap do |header| header.key = 'header' header.compose! @@ -80,7 +83,19 @@ def content end def spec - @spec ||= header.spec_inputs_value + @spec ||= begin + full_spec = header.spec_entry.value || {} + if full_spec[:include].present? && external_context + processor = External::Header::Processor.new(full_spec, external_context) + processed_spec = processor.perform + processed_spec[:inputs] || {} + else + full_spec[:inputs] || {} + end + end + rescue External::Header::Processor::IncludeError => e + @errors.push(e.message) + {} end def inputs diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index bf3aba542b9ef931636aa535b630885fa4c6bc83..36ecab2bac659a428e58d16072096cc6169323c7 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -7,8 +7,9 @@ module Yaml LoadError = Class.new(StandardError) class << self - def load!(content, context, inputs = {}) - Loader.new(content, inputs: inputs, context: context).load.then do |result| + def load!(content, context, inputs = {}, external_context = nil) + Loader.new(content, inputs: inputs, context: context, + external_context: external_context).load.then do |result| raise result.error_class, result.error if !result.valid? && result.error_class.present? raise LoadError, result.error unless result.valid? diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index 6757c4c55e8929621a1a289d4f4a79b9ba0d1a4d..61f3b33e66960f233e7f5e4731058172646b02fc 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -10,10 +10,11 @@ class Loader AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, inputs: {}, context: Config::Yaml::Context.new) + def initialize(content, inputs: {}, context: Config::Yaml::Context.new, external_context: nil) @content = content @inputs = inputs @context = context + @external_context = external_context end def load @@ -21,7 +22,8 @@ def load return yaml_result unless yaml_result.valid? - interpolator = Interpolation::Interpolator.new(yaml_result, inputs, context) + interpolator = Interpolation::Interpolator.new(yaml_result, inputs, context, + external_context: external_context) interpolator.interpolate! @@ -42,7 +44,7 @@ def load_uninterpolated_yaml private - attr_reader :content, :inputs, :context + attr_reader :content, :inputs, :context, :external_context def load_yaml! ensure_custom_tags diff --git a/spec/lib/gitlab/ci/config/entry/include_spec.rb b/spec/lib/gitlab/ci/config/entry/include_spec.rb index 8f1cbb02e0ae31490f4513f800a0fb59f8ea3f54..86976f53f1a0e9e8e3317113091379aff22eacff 100644 --- a/spec/lib/gitlab/ci/config/entry/include_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/include_spec.rb @@ -20,6 +20,12 @@ it { is_expected.to be_valid } end + context 'when using "component"' do + let(:config) { { component: 'path/to/component@1.0' } } + + it { is_expected.to be_valid } + end + context 'when using "project" with "ref"' do let(:config) { { project: 'my-group/my-pipeline-library', ref: 'master', file: 'test.yml' } } diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 7f467c581bd6122dd406262ad3e87b6090e1de91..495b11b64e88de78cead5840d19ef86c5a9ae30b 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -289,4 +289,95 @@ def initialize(params, ctx) end end end + + describe '#inputs_only!' do + let(:location) { 'some/file/config.yml' } + + it 'sets the inputs_only flag' do + expect(file.inputs_only?).to be_falsy + + file.inputs_only! + + expect(file.inputs_only?).to be_truthy + end + + it 'returns self for chaining' do + expect(file.inputs_only!).to eq(file) + end + end + + describe '#inputs_only?' do + let(:location) { 'some/file/config.yml' } + + it 'returns false by default' do + expect(file.inputs_only?).to be_falsy + end + + it 'returns true after calling inputs_only!' do + file.inputs_only! + expect(file.inputs_only?).to be_truthy + end + end + + describe '#validate_content_keys!' do + let(:location) { 'some/file/inputs.yml' } + + before do + file.inputs_only! + end + + context 'when content has only inputs key' do + let(:content) do + <<~YAML + inputs: + environment: + default: 'production' + YAML + end + + it 'does not add errors' do + file.load_and_validate_expanded_hash! + expect(file.errors).to be_empty + end + end + + context 'when content has unknown keys' do + let(:content) do + <<~YAML + inputs: + environment: + default: 'production' + image: 'ruby:3.0' + script: + - echo "test" + YAML + end + + it 'adds an error for unknown keys' do + file.load_and_validate_expanded_hash! + expect(file.errors).to include( + match(/Header include file .* contains unknown keys: .*image.*script/) + ) + end + end + + context 'when not in inputs_only mode' do + let(:content) do + <<~YAML + image: 'ruby:3.0' + script: + - echo "test" + YAML + end + + before do + allow(file).to receive(:inputs_only?).and_return(false) + end + + it 'does not validate content keys' do + file.load_and_validate_expanded_hash! + expect(file.errors).to be_empty + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/header/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/header/mapper/matcher_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..97f34fe06dc39b603415040a4781090f8f02638f --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/header/mapper/matcher_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Header::Mapper::Matcher, feature_category: :pipeline_composition do + let_it_be(:variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'A_MASKED_VAR', value: 'this-is-secret', masked: true) + end + end + + let_it_be(:context) do + Gitlab::Ci::Config::External::Context.new(variables: variables) + end + + subject(:matcher) { described_class.new(context) } + + describe '#process' do + let(:masked_variable_value) { 'this-is-secret.yml' } + + # Header only supports local, remote, and project includes + let(:supported_file_types) do + { + { local: 'file.yml' } => Gitlab::Ci::Config::External::File::Local, + { file: 'file.yml', project: 'namespace/project' } => Gitlab::Ci::Config::External::File::Project, + { remote: 'https://example.com/.gitlab-ci.yml' } => Gitlab::Ci::Config::External::File::Remote + } + end + + # Use shared examples + it_behaves_like 'processes supported file types' + it_behaves_like 'handles invalid locations' + it_behaves_like 'handles ambiguous locations' + it_behaves_like 'masks variables in error messages' + + # Header::Mapper::Matcher specific tests + context 'when using unsupported file types for header includes' do + context 'with template include' do + let(:locations) { [{ template: 'Auto-DevOps.gitlab-ci.yml' }] } + + it 'raises an ambiguous specification error' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /does not have a valid subkey for header include/ + ) + end + end + + context 'with component include' do + let(:locations) { [{ component: 'gitlab.com/org/component@1.0' }] } + + it 'raises an ambiguous specification error' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /does not have a valid subkey for header include/ + ) + end + end + + context 'with artifact include' do + let(:locations) { [{ artifact: 'generated.yml', job: 'test' }] } + + it 'raises an ambiguous specification error' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /does not have a valid subkey for header include/ + ) + end + end + end + + context 'when files are returned' do + let(:locations) { [{ local: 'inputs.yml' }] } + + it 'returns files with inputs_only mode enabled' do + files = matcher.process(locations) + expect(files).to all(be_a(Gitlab::Ci::Config::External::File::Base)) + expect(files.first).to be_inputs_only + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/header/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/header/mapper_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..dde2e17de839e0501127427915d7684af9cbf026 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/header/mapper_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Header::Mapper, feature_category: :pipeline_composition do + include StubRequests + include RepoHelpers + + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { project.owner } + + let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' } + let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } + let(:variables) { project.predefined_variables } + let(:context_params) { { project: project, sha: project.commit.sha, user: user, variables: variables } } + let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } + + let(:file_content) do + <<~YAML + inputs: + environment: + default: 'production' + YAML + end + + subject(:mapper) { described_class.new(values, context) } + + before do + stub_full_request(remote_url).to_return(body: file_content) + + allow_next_instance_of(Gitlab::Ci::Config::External::Context) do |instance| + allow(instance).to receive(:check_execution_time!) + end + end + + describe '#process' do + subject(:process) { mapper.process } + + # Use shared examples + it_behaves_like 'processes local file includes' + it_behaves_like 'processes remote file includes' + it_behaves_like 'processes project file includes' + it_behaves_like 'handles empty includes' + it_behaves_like 'handles invalid include types' + it_behaves_like 'handles ambiguous specifications' + it_behaves_like 'processes array of includes' + + # Header::Mapper specific tests + context 'when using template includes' do + let(:values) do + { include: { 'template' => 'Auto-DevOps.gitlab-ci.yml' }, + inputs: { environment: { default: 'production' } } } + end + + it 'raises an error for unsupported include type' do + expect { process }.to raise_error(described_class::AmbigiousSpecificationError) + end + end + + context 'when using component includes' do + let(:values) do + { include: { 'component' => 'path/to/component@1.0' }, + inputs: { environment: { default: 'production' } } } + end + + it 'raises an error for unsupported include type' do + expect { process }.to raise_error(described_class::AmbigiousSpecificationError) + end + end + + context 'when files are returned' do + let(:project_files) { { '/inputs.yml' => file_content } } + let(:values) do + { include: { 'local' => '/inputs.yml' }, + inputs: { environment: { default: 'production' } } } + end + + around do |example| + create_and_delete_files(project, project_files) do + example.run + end + end + + it 'returns files with inputs_only mode enabled' do + files = process + expect(files).to all(be_a(Gitlab::Ci::Config::External::File::Base)) + expect(files.first).to be_inputs_only + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/header/processor_spec.rb b/spec/lib/gitlab/ci/config/external/header/processor_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..49600ac8fa0af884cf1fe2556be0b33c7e4206ff --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/header/processor_spec.rb @@ -0,0 +1,323 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Header::Processor, feature_category: :pipeline_composition do + include StubRequests + include RepoHelpers + + let_it_be(:user) { create(:user) } + + let_it_be_with_reload(:project) { create(:project, :repository) } + let_it_be_with_reload(:another_project) { create(:project, :repository) } + + let(:project_files) { {} } + let(:other_project_files) { {} } + + let(:sha) { project.commit.sha } + let(:context_params) { { project: project, sha: sha, user: user } } + let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } + + subject(:processor) { described_class.new(values, context) } + + around do |example| + create_and_delete_files(project, project_files) do + create_and_delete_files(another_project, other_project_files) do + example.run + end + end + end + + before_all do + project.add_developer(user) + end + + before do + allow_next_instance_of(Gitlab::Ci::Config::External::Context) do |instance| + allow(instance).to receive(:check_execution_time!) + end + end + + describe '#perform' do + # Shared examples setup + let(:base_values) do + { + inputs: { + inline_input: { default: 'inline_value' } + } + } + end + + let(:invalid_local_file_include) { '/non-existent-inputs.yml' } + let(:invalid_remote_file) { 'http://doesntexist.com/inputs.yml' } + + let(:valid_local_file_include) { '/inputs.yml' } + let(:valid_remote_file) { 'https://example.com/inputs.yml' } + + let(:valid_remote_file_content) do + <<~YAML + inputs: + remote_input: + default: 'remote_value' + YAML + end + + let(:project_files) do + { + '/inputs.yml' => <<~YAML + inputs: + local_input: + default: 'local_value' + YAML + } + end + + let(:multiple_includes) do + [ + { local: '/inputs.yml' }, + { remote: valid_remote_file } + ] + end + + # Use shared examples + it_behaves_like 'returns values when no includes defined' + it_behaves_like 'handles invalid local files' + it_behaves_like 'handles invalid remote files' + it_behaves_like 'processes valid external files' + it_behaves_like 'processes multiple external files' + + # Header-specific tests + context 'with valid local file containing inputs' do + let(:values) do + { + include: [{ local: '/shared-inputs.yml' }], + inputs: { + region: { default: 'us-west-2' } + } + } + end + + let(:project_files) do + { + '/shared-inputs.yml' => <<~YAML + inputs: + database: + default: 'postgres' + region: + default: 'us-east-1' + cache_enabled: + default: true + YAML + } + end + + it 'merges inputs with inline inputs taking precedence' do + result = processor.perform + expect(result[:inputs]).to eq({ + database: { default: 'postgres' }, + region: { default: 'us-west-2' }, + cache_enabled: { default: true } + }) + end + + it 'removes the include key' do + expect(processor.perform[:include]).to be_nil + end + end + + context 'with multiple local files' do + let(:values) do + { + include: [ + { local: '/base-inputs.yml' }, + { local: '/override-inputs.yml' } + ], + inputs: { + timeout: { default: 3600 } + } + } + end + + let(:project_files) do + { + '/base-inputs.yml' => <<~YAML, + inputs: + database: + default: 'postgres' + cache_enabled: + default: true + YAML + '/override-inputs.yml' => <<~YAML + inputs: + database: + default: 'mysql' + region: + default: 'eu-west-1' + YAML + } + end + + it 'merges all inputs in order with later files taking precedence' do + result = processor.perform + expect(result[:inputs]).to eq({ + database: { default: 'mysql' }, + cache_enabled: { default: true }, + region: { default: 'eu-west-1' }, + timeout: { default: 3600 } + }) + end + end + + context 'with a valid project file' do + let(:values) do + { + include: [ + { + project: another_project.full_path, + file: '/shared-inputs.yml', + ref: 'master' + } + ], + inputs: { + local_input: { default: 'value' } + } + } + end + + let(:other_project_files) do + { + '/shared-inputs.yml' => <<~YAML + inputs: + shared_database: + default: 'postgres' + shared_cache: + default: true + YAML + } + end + + before_all do + another_project.add_developer(user) + end + + it 'merges project inputs with inline inputs' do + result = processor.perform + expect(result[:inputs]).to eq({ + shared_database: { default: 'postgres' }, + shared_cache: { default: true }, + local_input: { default: 'value' } + }) + end + end + + context 'when included file contains non-input keys' do + let(:values) do + { + include: [{ local: '/invalid-inputs.yml' }], + inputs: { + foo: { default: 'bar' } + } + } + end + + let(:project_files) do + { + '/invalid-inputs.yml' => <<~YAML + inputs: + database: + default: 'postgres' + test: + script: echo "test" + YAML + } + end + + it 'raises an error about unknown keys' do + expect { processor.perform }.to raise_error( + described_class::IncludeError, + /Header include file .* contains unknown keys: \[:test\]/ + ) + end + end + + context 'when included file is empty' do + let(:values) do + { + include: [{ local: '/empty-inputs.yml' }], + inputs: { + foo: { default: 'bar' } + } + } + end + + let(:project_files) do + { + '/empty-inputs.yml' => '' + } + end + + it 'raises an error' do + expect { processor.perform }.to raise_error( + described_class::IncludeError, + /Local file .* is empty/ + ) + end + end + + context 'when included file has only inputs key with no content' do + let(:values) do + { + include: [{ local: '/empty-content-inputs.yml' }], + inputs: { + foo: { default: 'bar' } + } + } + end + + let(:project_files) do + { + '/empty-content-inputs.yml' => <<~YAML + inputs: + YAML + } + end + + it 'merges with inline inputs only' do + result = processor.perform + expect(result[:inputs]).to eq({ + foo: { default: 'bar' } + }) + end + end + + context 'with nested input structures' do + let(:values) do + { + include: [{ local: '/nested-inputs.yml' }], + inputs: { + config: { + options: ['opt3'] + } + } + } + end + + let(:project_files) do + { + '/nested-inputs.yml' => <<~YAML + inputs: + config: + default: 'value' + options: ['opt1', 'opt2'] + YAML + } + end + + it 'performs deep merge of input structures' do + result = processor.perform + expect(result[:inputs][:config][:options]).to eq(['opt3']) + expect(result[:inputs][:config][:default]).to eq('value') + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb index 077e3d245f73d659603a9c07aca0b537f6d5525d..f5e31ebf100f7f47877996b082951a6ac7d65a10 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb @@ -16,61 +16,24 @@ subject(:matcher) { described_class.new(context) } describe '#process' do - subject(:process) { matcher.process(locations) } - - let(:locations) do - [ - { local: 'file.yml' }, - { file: 'file.yml', project: 'namespace/project' }, - { component: 'gitlab.com/org/component@1.0' }, - { remote: 'https://example.com/.gitlab-ci.yml' }, - { template: 'file.yml' }, - { artifact: 'generated.yml', job: 'test' } - ] - end - - it 'returns an array of file objects' do - is_expected.to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Local), - an_instance_of(Gitlab::Ci::Config::External::File::Project), - an_instance_of(Gitlab::Ci::Config::External::File::Component), - an_instance_of(Gitlab::Ci::Config::External::File::Remote), - an_instance_of(Gitlab::Ci::Config::External::File::Template), - an_instance_of(Gitlab::Ci::Config::External::File::Artifact) - ) - end - - context 'when a location is not valid' do - let(:locations) { [{ invalid: 'file.yml' }] } - - it 'raises an error' do - expect { process }.to raise_error( - Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, - /`{"invalid":"file.yml"}` does not have a valid subkey for include. Valid subkeys are:/ - ) - end - - context 'when the invalid location includes a masked variable' do - let(:locations) { [{ invalid: 'this-is-secret.yml' }] } - - it 'raises an error with a masked sentence' do - expect { process }.to raise_error( - Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, - /`{"invalid":"\[MASKED\]xxxxxx.yml"}` does not have a valid subkey for include. Valid subkeys are:/ - ) - end - end + let(:masked_variable_value) { 'this-is-secret.yml' } + + # External supports all file types + let(:supported_file_types) do + { + { local: 'file.yml' } => Gitlab::Ci::Config::External::File::Local, + { file: 'file.yml', project: 'namespace/project' } => Gitlab::Ci::Config::External::File::Project, + { component: 'gitlab.com/org/component@1.0' } => Gitlab::Ci::Config::External::File::Component, + { remote: 'https://example.com/.gitlab-ci.yml' } => Gitlab::Ci::Config::External::File::Remote, + { template: 'file.yml' } => Gitlab::Ci::Config::External::File::Template, + { artifact: 'generated.yml', job: 'test' } => Gitlab::Ci::Config::External::File::Artifact + } end - context 'when a location is ambiguous' do - let(:locations) { [{ local: 'file.yml', remote: 'https://example.com/.gitlab-ci.yml' }] } - - it 'raises an error' do - expect { process }.to raise_error( - Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, - /Each include must use only one of:/ - ) - end - end + # Use shared examples + it_behaves_like 'processes supported file types' + it_behaves_like 'handles invalid locations' + it_behaves_like 'handles ambiguous locations' + it_behaves_like 'masks variables in error messages' end end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index 25e78d2bdb4622c0ad7efc5f76f24358f88e27a5..e91d034f9a9312ee200c1bc2990cf9b9c58db774 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -48,6 +48,16 @@ end end + # Use shared examples + it_behaves_like 'processes local file includes' + it_behaves_like 'processes remote file includes' + it_behaves_like 'processes project file includes' + it_behaves_like 'handles empty includes' + it_behaves_like 'handles invalid include types' + it_behaves_like 'handles ambiguous specifications' + it_behaves_like 'processes array of includes' + + # External::Mapper specific tests context "when single 'include' keyword is defined" do context 'when the string is a local file' do let(:values) do @@ -55,49 +65,9 @@ image: 'image:1.0' } end - it 'returns File instances' do - expect(subject).to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Local)) - end - it_behaves_like 'logging config file fetch', 'config_file_fetch_local_content_duration_s', 1 end - context 'when the key is a local file hash' do - let(:values) do - { include: { 'local' => local_file }, - image: 'image:1.0' } - end - - it 'returns File instances' do - expect(subject).to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Local)) - end - end - - context 'when the string is a remote file' do - let(:values) do - { include: remote_url, image: 'image:1.0' } - end - - it 'returns File instances' do - expect(subject).to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Remote)) - end - end - - context 'when the key is a remote file hash' do - let(:values) do - { include: { 'remote' => remote_url }, - image: 'image:1.0' } - end - - it 'returns File instances' do - expect(subject).to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Remote)) - end - end - context 'when the key is a template file hash' do let(:values) do { include: { 'template' => template_file }, @@ -112,43 +82,12 @@ it_behaves_like 'logging config file fetch', 'config_file_fetch_template_content_duration_s', 1 end - context 'when the key is not valid' do - let(:local_file) { 'secret-file.yml' } - let(:values) do - { include: { invalid: local_file }, - image: 'image:1.0' } - end - - it 'returns ambigious specification error' do - expect { subject }.to raise_error(described_class::AmbigiousSpecificationError, /`{"invalid":"secret-file.yml"}` does not have a valid subkey for include. Valid subkeys are:/) - end - end - - context 'when the key is a hash of local and remote' do - let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file', 'masked' => true }]) } - let(:local_file) { 'secret-file.yml' } - let(:remote_url) { 'https://gitlab.com/secret-file.yml' } - let(:values) do - { include: { 'local' => local_file, 'remote' => remote_url }, - image: 'image:1.0' } - end - - it 'returns ambigious specification error' do - expect { subject }.to raise_error(described_class::AmbigiousSpecificationError, /Each include must use only one of/) - end - end - context "when the key is a project's file" do let(:values) do { include: { project: project.full_path, file: local_file }, image: 'image:1.0' } end - it 'returns File instances' do - expect(subject).to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Project)) - end - it_behaves_like 'logging config file fetch', 'config_file_fetch_project_content_duration_s', 1 end @@ -166,71 +105,6 @@ it_behaves_like 'logging config file fetch', 'config_file_fetch_project_content_duration_s', 1 end - - context 'when the include value is a Boolean' do - let(:values) { { include: true } } - - it 'raises an error' do - expect { process }.to raise_error( - Gitlab::Ci::Config::External::Mapper::InvalidTypeError, /Each include must be a hash or a string/) - end - end - end - - context "when 'include' is defined as an array" do - let(:values) do - { include: [remote_url, local_file], - image: 'image:1.0' } - end - - it 'returns Files instances' do - expect(subject).to all(respond_to(:valid?)) - expect(subject).to all(respond_to(:content)) - end - - context 'when an include value is an Array' do - let(:values) { { include: [remote_url, [local_file]] } } - - it 'raises an error' do - expect { process }.to raise_error( - Gitlab::Ci::Config::External::Mapper::InvalidTypeError, /Each include must be a hash or a string/) - end - end - end - - context "when 'include' is defined as an array of hashes" do - let(:values) do - { include: [{ remote: remote_url }, { local: local_file }], - image: 'image:1.0' } - end - - it 'returns Files instances' do - expect(subject).to all(respond_to(:valid?)) - expect(subject).to all(respond_to(:content)) - end - - context 'when it has ambigious match' do - let(:values) do - { include: [{ remote: remote_url, local: local_file }], - image: 'image:1.0' } - end - - it 'returns ambigious specification error' do - expect { subject }.to raise_error(described_class::AmbigiousSpecificationError) - end - end - end - - context "when 'include' is not defined" do - let(:values) do - { - image: 'image:1.0' - } - end - - it 'returns an empty array' do - expect(subject).to be_empty - end end context "when duplicate 'include's are defined" do diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index 7550657fb1ee305507c2ae13b4a92b00871e6faa..a3d408ea87647d4cb7bfb4c522c72fce30745f0c 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -38,41 +38,58 @@ describe "#perform" do subject(:perform) { processor.perform } - context 'when no external files defined' do - let(:values) { { image: 'image:1.0' } } - - it 'returns the same values' do - expect(processor.perform).to eq(values) - end + # Shared examples setup + let(:base_values) { { image: 'image:1.0' } } + let(:invalid_local_file_include) { '/lib/gitlab/ci/templates/non-existent-file.yml' } + let(:invalid_remote_file) { 'http://doesntexist.com/.gitlab-ci-1.yml' } + let(:valid_local_file_include) { '/lib/gitlab/ci/templates/template.yml' } + let(:valid_remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } + + let(:valid_remote_file_content) do + <<~YAML + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + YAML end - context 'when an invalid local file is defined' do - let(:values) { { include: '/lib/gitlab/ci/templates/non-existent-file.yml', image: 'image:1.0' } } + let(:local_file_content) do + <<~YAML + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - bundle install --jobs $(nproc) "${FLAGS[@]}" + YAML + end - it 'raises an error' do - expect { processor.perform }.to raise_error( - described_class::IncludeError, - "Local file `lib/gitlab/ci/templates/non-existent-file.yml` does not exist!" - ) - end + let(:multiple_local_file_content) do + File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml')) end - context 'when an invalid remote file is defined' do - let(:remote_file) { 'http://doesntexist.com/.gitlab-ci-1.yml' } - let(:values) { { include: remote_file, image: 'image:1.0' } } + let(:multiple_includes) do + [ + { local: '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }, + { remote: valid_remote_file } + ] + end - before do - stub_full_request(remote_file).and_raise(SocketError.new('Some HTTP error')) + # Use shared examples + context 'with common external processor behavior' do + let(:project_files) do + { + valid_local_file_include => local_file_content, + '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' => multiple_local_file_content + } end - it 'raises an error' do - expect { processor.perform }.to raise_error( - described_class::IncludeError, - "Remote file `#{remote_file}` could not be fetched because of a socket error!" - ) - end + it_behaves_like 'returns values when no includes defined' + it_behaves_like 'handles invalid local files' + it_behaves_like 'handles invalid remote files' + it_behaves_like 'processes valid external files' + it_behaves_like 'processes multiple external files' end + # External::Processor specific tests context 'with a valid remote external file is defined' do let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:values) { { include: remote_file, image: 'image:1.0' } } @@ -138,79 +155,6 @@ end end - context 'with a valid local external file is defined' do - let(:values) { { include: '/lib/gitlab/ci/templates/template.yml', image: 'image:1.0' } } - let(:local_file_content) do - <<-YAML - before_script: - - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - - ruby -v - - which ruby - - bundle install --jobs $(nproc) "${FLAGS[@]}" - YAML - end - - let(:project_files) { { '/lib/gitlab/ci/templates/template.yml' => local_file_content } } - - it 'appends the file to the values' do - output = processor.perform - expect(output.keys).to match_array([:image, :before_script]) - end - - it "removes the 'include' keyword" do - expect(processor.perform[:include]).to be_nil - end - end - - context 'with multiple external files are defined' do - let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } - - let(:local_file_content) do - File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml')) - end - - let(:external_files) do - [ - '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml', - remote_file - ] - end - - let(:values) do - { - include: external_files, - image: 'image:1.0' - } - end - - let(:remote_file_content) do - <<-YAML - stages: - - build - - review - - cleanup - YAML - end - - let(:project_files) do - { - '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' => local_file_content - } - end - - before do - stub_full_request(remote_file).to_return(body: remote_file_content) - end - - it 'appends the files to the values' do - expect(processor.perform.keys).to match_array([:image, :stages, :before_script, :rspec]) - end - - it "removes the 'include' keyword" do - expect(processor.perform[:include]).to be_nil - end - end - context 'when external files are defined but not valid' do let(:values) { { include: '/lib/gitlab/ci/templates/template.yml', image: 'image:1.0' } } diff --git a/spec/lib/gitlab/ci/config/header/spec_spec.rb b/spec/lib/gitlab/ci/config/header/spec_spec.rb index 5925c67af0b6930f68091b230f8ca7763d46a617..0bf2efdc20dab5a1371cbf988529158b1329aa48 100644 --- a/spec/lib/gitlab/ci/config/header/spec_spec.rb +++ b/spec/lib/gitlab/ci/config/header/spec_spec.rb @@ -126,4 +126,109 @@ expect(config.value).to eq(spec_hash) end end + + context 'when spec contains include' do + let(:spec_hash) do + { + inputs: { + environment: { default: 'production' } + }, + include: [ + { local: '/inputs.yml' } + ] + } + end + + before do + allow(Gitlab::Ci::Config::FeatureFlags).to receive(:enabled?) + .with(:ci_file_inputs) + .and_return(feature_flag_enabled) + end + + context 'when ci_file_inputs feature flag is enabled' do + let(:feature_flag_enabled) { true } + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value with include' do + expect(config.value).to eq(spec_hash) + end + + it 'has include entry defined' do + expect(config.include_value).to eq([{ local: '/inputs.yml' }]) + end + end + + context 'when ci_file_inputs feature flag is disabled' do + let(:feature_flag_enabled) { false } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to include('spec config contains unknown keys: include') + end + + it 'still returns the value' do + expect(config.value).to eq(spec_hash) + end + end + end + + context 'when spec contains only include without inputs' do + let(:spec_hash) do + { + include: [ + { local: '/inputs.yml' } + ] + } + end + + before do + allow(Gitlab::Ci::Config::FeatureFlags).to receive(:enabled?) + .with(:ci_file_inputs) + .and_return(true) + end + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the include value' do + expect(config.include_value).to eq([{ local: '/inputs.yml' }]) + end + end + + context 'when spec contains inputs, include, and component' do + let(:spec_hash) do + { + inputs: { + environment: { default: 'production' } + }, + include: [ + { local: '/inputs.yml' } + ], + component: %w[name version] + } + end + + before do + allow(Gitlab::Ci::Config::FeatureFlags).to receive(:enabled?) + .with(:ci_file_inputs) + .and_return(true) + end + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns all values correctly' do + expect(config.inputs_value).to eq({ environment: { default: 'production' } }) + expect(config.include_value).to eq([{ local: '/inputs.yml' }]) + expect(config.component_value).to match_array([:name, :version]) + end + end end diff --git a/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb b/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb index 942948a68ed8f516cdc5e865625443ba601fec4d..966d07d2ffb5c81cda3b475e109ed9a8d9a61f32 100644 --- a/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb +++ b/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb @@ -282,4 +282,153 @@ end end end + + describe 'external context and header includes' do + let(:external_context) do + Gitlab::Ci::Config::External::Context.new(project: project, sha: 'HEAD', user: create(:user)) + end + + subject do + described_class.new(result, arguments, yaml_context, external_context: external_context) + end + + before do + allow_next_instance_of(Gitlab::Ci::Config::External::Context) do |instance| + allow(instance).to receive(:check_execution_time!) + end + + allow(Gitlab::Ci::Config::FeatureFlags).to receive(:enabled?) + .with(:ci_file_inputs) + .and_return(true) + end + + context 'when spec includes external input files' do + let(:header) do + { + spec: { + include: [{ local: '/inputs.yml' }], + inputs: { + inline_input: { default: 'inline_value' } + } + } + } + end + + let(:content) do + { test: 'deploy $[[ inputs.inline_input ]] $[[ inputs.external_input ]]' } + end + + let(:arguments) do + { inline_input: 'inline', external_input: 'external' } + end + + let(:external_inputs) do + { inputs: { external_input: { default: 'external_value' } } } + end + + before do + allow_next_instance_of(Gitlab::Ci::Config::External::Header::Processor) do |processor| + allow(processor).to receive(:perform).and_return( + inputs: { + inline_input: { default: 'inline_value' }, + external_input: { default: 'external_value' } + } + ) + end + end + + it 'processes header includes and merges external inputs' do + subject.interpolate! + + expect(subject).to be_interpolated + expect(subject).to be_valid + expect(subject.to_hash).to eq({ test: 'deploy inline external' }) + end + end + + context 'when spec has includes but no external_context is provided' do + let(:header) do + { + spec: { + include: [{ local: '/inputs.yml' }], + inputs: { + website: { default: 'gitlab.com' } + } + } + } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'example.com' } + end + + subject do + described_class.new(result, arguments, yaml_context, external_context: nil) + end + + it 'uses inline inputs without processing includes' do + subject.interpolate! + + expect(subject).to be_interpolated + expect(subject).to be_valid + expect(subject.to_hash).to eq({ test: 'deploy example.com' }) + end + end + + context 'when header include processing fails' do + let(:header) do + { + spec: { + include: [{ local: '/non-existent.yml' }], + inputs: { + website: nil + } + } + } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + before do + allow_next_instance_of(Gitlab::Ci::Config::External::Header::Processor) do |processor| + allow(processor).to receive(:perform).and_raise( + Gitlab::Ci::Config::External::Header::Processor::IncludeError.new('Local file does not exist') + ) + end + end + + it 'captures the error and marks interpolation as invalid' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.errors).to include('Local file does not exist') + end + end + + context 'when header is nil and external_context is provided' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(config: content) + end + + let(:content) { { test: 'deploy production' } } + let(:arguments) { nil } + + it 'handles nil header gracefully' do + subject.interpolate! + + expect(subject).to be_valid + expect(subject.to_hash).to eq({ test: 'deploy production' }) + end + end + end end diff --git a/spec/requests/api/graphql/project/ci/pipeline_creation/input_spec.rb b/spec/requests/api/graphql/project/ci/pipeline_creation/input_spec.rb index 12940dafb16dd0d7a50e1f90283f5146534b70a2..7c3e6f61917f08fc921a1953c25eb3b2eba73410 100644 --- a/spec/requests/api/graphql/project/ci/pipeline_creation/input_spec.rb +++ b/spec/requests/api/graphql/project/ci/pipeline_creation/input_spec.rb @@ -92,6 +92,12 @@ project.add_developer(user) end + before do + allow(Gitlab::Ci::Config::FeatureFlags).to receive(:enabled?) + .with(:ci_file_inputs) + .and_return(true) + end + context 'when inputs exist' do let(:ref) { 'master' } diff --git a/spec/services/ci/pipeline_creation/find_pipeline_inputs_service_spec.rb b/spec/services/ci/pipeline_creation/find_pipeline_inputs_service_spec.rb index 704919072a12b0b00ffd1f9a9781cbb787c93a88..a74e2c17d782e1c0934c66cac5e739e46828725d 100644 --- a/spec/services/ci/pipeline_creation/find_pipeline_inputs_service_spec.rb +++ b/spec/services/ci/pipeline_creation/find_pipeline_inputs_service_spec.rb @@ -194,6 +194,150 @@ expect(result.message).to eq(s_('Pipelines|Inputs not supported for this CI config source')) end end + + context 'when spec contains header includes' do + let(:config_yaml) do + <<~YAML + spec: + include: + - local: /inputs.yml + inputs: + inline_input: + default: inline_value + --- + job: + script: echo $[[ inputs.inline_input ]] + YAML + end + + let(:external_inputs_yaml) do + <<~YAML + inputs: + external_input: + default: external_value + YAML + end + + before do + project.repository.create_file( + project.creator, + '.gitlab-ci.yml', + config_yaml, + message: 'Add CI with header includes', + branch_name: 'master') + + project.repository.create_file( + project.creator, + 'inputs.yml', + external_inputs_yaml, + message: 'Add external inputs', + branch_name: 'master') + end + + it 'processes header includes and merges external inputs' do + result = service.execute + + expect(result).to be_success + + spec_inputs = result.payload.fetch(:inputs) + expect(spec_inputs.errors).to be_empty + + input_names = spec_inputs.all_inputs.map(&:name) + expect(input_names).to include(:external_input, :inline_input) + end + + it 'gives precedence to inline inputs over external inputs' do + result = service.execute + + expect(result).to be_success + + spec_inputs = result.payload.fetch(:inputs) + inline_input = spec_inputs.all_inputs.find { |i| i.name == :inline_input } + expect(inline_input.default).to eq('inline_value') + end + + context 'when ci_file_inputs feature flag is disabled' do + before do + stub_feature_flags(ci_file_inputs: false) + end + + it 'does not process header includes and only uses inline inputs' do + result = service.execute + + expect(result).to be_success + + spec_inputs = result.payload.fetch(:inputs) + expect(spec_inputs.errors).to be_empty + + input_names = spec_inputs.all_inputs.map(&:name) + expect(input_names).to include(:inline_input) + expect(input_names).not_to include(:external_input) + end + end + end + + context 'when header include processing fails' do + let(:config_yaml) do + <<~YAML + spec: + include: + - local: /non-existent-inputs.yml + --- + job: + script: echo test + YAML + end + + before do + project.repository.create_file( + project.creator, + '.gitlab-ci.yml', + config_yaml, + message: 'Add CI with invalid header include', + branch_name: 'master') + end + + it 'returns error response with include error message' do + result = service.execute + + expect(result).to be_error + expect(result.message).to match(/Local file .* does not exist/) + end + end + + context 'when spec does not contain header includes' do + let(:config_yaml) do + <<~YAML + spec: + inputs: + environment: + default: production + --- + job: + script: echo $[[ inputs.environment ]] + YAML + end + + before do + project.repository.create_file( + project.creator, + '.gitlab-ci.yml', + config_yaml, + message: 'Add CI without header includes', + branch_name: 'master') + end + + it 'processes inputs without header include processing' do + result = service.execute + + expect(result).to be_success + + spec_inputs = result.payload.fetch(:inputs) + expect(spec_inputs.errors).to be_empty + expect(spec_inputs.all_inputs.first.name).to eq(:environment) + expect(spec_inputs.all_inputs.first.default).to eq('production') + end + end end end end diff --git a/spec/support/shared_examples/ci/config/external/mapper_examples.rb b/spec/support/shared_examples/ci/config/external/mapper_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..32534c4bd55a6e87b84b5873e611bdd55c478291 --- /dev/null +++ b/spec/support/shared_examples/ci/config/external/mapper_examples.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +# Shared examples for External::Mapper and Header::Mapper +# +# These examples test common behavior across both mapper types. +# The including spec must define: +# - mapper: instance of the mapper being tested +# - values: the input hash to process +# - context: the external context +# - local_file: path to a local file +# - remote_url: URL for remote file tests +# - file_content: content to stub for remote files +# +RSpec.shared_examples 'processes local file includes' do + context 'when the string is a local file' do + let(:values) do + { include: local_file, + image: 'image:1.0' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local)) + end + end + + context 'when the key is a local file hash' do + let(:values) do + { include: { 'local' => local_file }, + image: 'image:1.0' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local)) + end + end +end + +RSpec.shared_examples 'processes remote file includes' do + context 'when the string is a remote file' do + let(:values) do + { include: remote_url, image: 'image:1.0' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Remote)) + end + end + + context 'when the key is a remote file hash' do + let(:values) do + { include: { 'remote' => remote_url }, + image: 'image:1.0' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Remote)) + end + end +end + +RSpec.shared_examples 'processes project file includes' do + context "when the key is a project's file" do + let(:values) do + { include: { project: project.full_path, file: local_file }, + image: 'image:1.0' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Project)) + end + end +end + +RSpec.shared_examples 'handles empty includes' do + context "when 'include' is not defined" do + let(:values) do + { + image: 'image:1.0' + } + end + + it 'returns an empty array' do + expect(subject).to be_empty + end + end +end + +RSpec.shared_examples 'handles invalid include types' do + context 'when the include value is a Boolean' do + let(:values) { { include: true } } + + it 'raises an error' do + expect { mapper.process }.to raise_error( + Gitlab::Ci::Config::External::Mapper::InvalidTypeError, /Each include must be a hash or a string/) + end + end + + context 'when an include value is an Array' do + let(:values) { { include: [remote_url, [local_file]] } } + + it 'raises an error' do + expect { mapper.process }.to raise_error( + Gitlab::Ci::Config::External::Mapper::InvalidTypeError, /Each include must be a hash or a string/) + end + end +end + +RSpec.shared_examples 'handles ambiguous specifications' do + context 'when the key is not valid' do + let(:local_file) { 'secret-file.yml' } + let(:values) do + { include: { invalid: local_file }, + image: 'image:1.0' } + end + + it 'returns ambigious specification error' do + expect { subject }.to raise_error( + described_class::AmbigiousSpecificationError, + /does not have a valid subkey for.*include/ + ) + end + end + + context 'when the key is a hash of local and remote' do + let(:local_file) { 'secret-file.yml' } + let(:remote_url) { 'https://gitlab.com/secret-file.yml' } + let(:values) do + { include: { 'local' => local_file, 'remote' => remote_url }, + image: 'image:1.0' } + end + + it 'returns ambigious specification error' do + expect { subject }.to raise_error( + described_class::AmbigiousSpecificationError, + /Each include must use only one of/ + ) + end + end +end + +RSpec.shared_examples 'processes array of includes' do + context "when 'include' is defined as an array" do + let(:values) do + { include: [remote_url, local_file], + image: 'image:1.0' } + end + + it 'returns Files instances' do + expect(subject).to all(respond_to(:valid?)) + expect(subject).to all(respond_to(:content)) + end + end + + context "when 'include' is defined as an array of hashes" do + let(:values) do + { include: [{ remote: remote_url }, { local: local_file }], + image: 'image:1.0' } + end + + it 'returns Files instances' do + expect(subject).to all(respond_to(:valid?)) + expect(subject).to all(respond_to(:content)) + end + end +end diff --git a/spec/support/shared_examples/ci/config/external/matcher_examples.rb b/spec/support/shared_examples/ci/config/external/matcher_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..74c27bd3541b95970ef645ac2ed97b93a2bd2f96 --- /dev/null +++ b/spec/support/shared_examples/ci/config/external/matcher_examples.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Shared examples for External::Mapper::Matcher and Header::Mapper::Matcher +# +# These examples test common behavior across both matcher types. +# The including spec must define: +# - matcher: instance of the matcher being tested +# - context: the external context +# - supported_file_types: hash mapping location types to expected file classes +# +RSpec.shared_examples 'processes supported file types' do + context 'when processing supported file types' do + let(:locations) { supported_file_types.keys } + + it 'returns correct file objects for each type' do + result = matcher.process(locations) + + supported_file_types.each_value do |file_class| + expect(result).to include(an_instance_of(file_class)) + end + end + end +end + +RSpec.shared_examples 'handles invalid locations' do + context 'when a location is not valid' do + let(:locations) { [{ invalid: 'file.yml' }] } + + it 'raises an ambiguous specification error' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /does not have a valid subkey for.*include/ + ) + end + end +end + +RSpec.shared_examples 'handles ambiguous locations' do + context 'when a location is ambiguous' do + let(:locations) { [{ local: 'file.yml', remote: 'https://example.com/.gitlab-ci.yml' }] } + + it 'raises an ambiguous specification error' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /Each include must use only one of:/ + ) + end + end +end + +RSpec.shared_examples 'masks variables in error messages' do + context 'when the invalid location includes a masked variable' do + let(:locations) { [{ invalid: masked_variable_value }] } + + it 'raises an error with a masked sentence' do + expect { matcher.process(locations) }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + /\[MASKED\]/ + ) + end + end +end diff --git a/spec/support/shared_examples/ci/config/external/processor_examples.rb b/spec/support/shared_examples/ci/config/external/processor_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..a5f90c8c7971c86aeee9a57bb6d081dfdf787a25 --- /dev/null +++ b/spec/support/shared_examples/ci/config/external/processor_examples.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Shared examples for External::Processor and Header::Processor +# +# These examples test common behavior across both processor types. +# The including spec must define: +# - processor: instance of the processor being tested +# - values: the input hash to process +# - project_files: hash of files to create in the project +# - remote_file: URL for remote file tests +# +RSpec.shared_examples 'handles invalid local files' do + context 'when an invalid local file is defined' do + let(:values) do + { + include: invalid_local_file_include, + **base_values + } + end + + it 'raises an error' do + expect { processor.perform }.to raise_error( + described_class::IncludeError, + /Local file .* does not exist/ + ) + end + end +end + +RSpec.shared_examples 'handles invalid remote files' do + context 'when an invalid remote file is defined' do + let(:remote_file) { invalid_remote_file } + let(:values) do + { + include: { remote: remote_file }, + **base_values + } + end + + before do + stub_full_request(remote_file).and_raise(SocketError.new('Some HTTP error')) + end + + it 'raises an error' do + expect { processor.perform }.to raise_error( + described_class::IncludeError, + /Remote file .* could not be fetched because of a socket error/ + ) + end + end +end + +RSpec.shared_examples 'processes valid external files' do + context 'with a valid local external file' do + let(:values) do + { + include: valid_local_file_include, + **base_values + } + end + + it 'merges the external file content' do + result = processor.perform + expect(result).to be_a(Hash) + expect(result).not_to be_empty + end + + it "removes the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end + + context 'with a valid remote external file' do + let(:remote_file) { valid_remote_file } + let(:values) do + { + include: { remote: remote_file }, + **base_values + } + end + + before do + stub_full_request(remote_file).to_return(body: valid_remote_file_content) + end + + it 'merges the external file content' do + result = processor.perform + expect(result).to be_a(Hash) + expect(result).not_to be_empty + end + + it "removes the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end +end + +RSpec.shared_examples 'processes multiple external files' do + context 'with multiple external files' do + let(:remote_file) { valid_remote_file } + let(:values) do + { + include: multiple_includes, + **base_values + } + end + + before do + if respond_to?(:valid_remote_file_content) + stub_full_request(remote_file).to_return(body: valid_remote_file_content) + end + end + + it 'merges all external files' do + result = processor.perform + expect(result).to be_a(Hash) + expect(result).not_to be_empty + end + + it "removes the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end +end + +RSpec.shared_examples 'returns values when no includes defined' do + context 'when no external files defined' do + let(:values) { base_values } + + it 'returns the same values' do + expect(processor.perform).to eq(values) + end + end +end diff --git a/spec/support/shared_examples/lib/gitlab/ci/config/include_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/ci/config/include_shared_examples.rb index 74e2fbc77315d1d56b12bbaf396723c071060ac2..a9d43d506e67ef089d000fde617b6d6cc4dc7339 100644 --- a/spec/support/shared_examples/lib/gitlab/ci/config/include_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/ci/config/include_shared_examples.rb @@ -42,12 +42,6 @@ it { is_expected.to be_valid } end - context 'when using "component"' do - let(:config) { { component: 'path/to/component@1.0' } } - - it { is_expected.to be_valid } - end - context 'when using "project"' do context 'and specifying "file"' do let(:config) { { project: 'my-group/my-project', file: 'test.yml' } }