diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index c80344bef8aee4f277efaabedb36b08137f3eceb..7b0a4add4811c6b52936311f3d0c1e3727d54057 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -111,8 +111,12 @@ def expanded_content_hash def content_hash strong_memoize(:content_hash) do - ::Gitlab::Ci::Config::Yaml.load!(content) + ::Gitlab::Ci::Config::Yaml.load!(content, input_values: input_values, project: project) end + rescue Gitlab::Ci::Config::Yaml::InputInterpolator::RequiredInputsNotMetError => e + errors.push("Included file `#{masked_location}` has required inputs `#{e.message}` that are not given!") + + nil rescue Gitlab::Config::Loader::FormatError nil end @@ -149,6 +153,12 @@ def masked_location context.mask_variables_from(location) end end + + def input_values + return {} unless params.is_a?(Hash) + + params.fetch(:with, {}) + end end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 94ef0afe7f9fa45e3f5e3c99ca4b2d2ab9d015a2..357fbf0dfdb0ed4c6d02cc5ef7b276077378a08a 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -6,24 +6,29 @@ class Config module Yaml AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 + MULTI_DOC_DIVIDER = /^---$/ - class << self - def load!(content) + class Loader + def initialize(content, input_values: {}, project: nil) + @content = content + @project = project + @input_values = input_values + end + + def load! ensure_custom_tags - if ::Feature.enabled?(:ci_multi_doc_yaml) - Gitlab::Config::Loader::MultiDocYaml.new( - content, - max_documents: MAX_DOCUMENTS, - additional_permitted_classes: AVAILABLE_TAGS - ).load!.first + if multi_document_yaml_loading_enabled? + load_multi_document_yaml! else - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + yaml_loader(content).load! end end private + attr_reader :content, :input_values, :project + def ensure_custom_tags @ensure_custom_tags ||= begin AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } @@ -31,6 +36,50 @@ def ensure_custom_tags true end end + + def multi_document_yaml_loading_enabled? + project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) + end + + def load_multi_document_yaml! + if contains_config_spec? + InputInterpolator.new( + spec_inputs, + yaml_loader(documents.second).load_raw!, + input_values: input_values + ).interpolate!.deep_symbolize_keys! + else + yaml_loader(documents.first).load! + end + end + + def contains_config_spec? + documents.count == 2 + end + + def spec_inputs + spec.dig(:spec, :inputs) || {} + end + + def spec + yaml_loader(documents.first).load! + rescue Gitlab::Config::Loader::FormatError + {} + end + + def documents + content.split(MULTI_DOC_DIVIDER).reject(&:empty?) + end + + def yaml_loader(yaml_content) + Gitlab::Config::Loader::Yaml.new(yaml_content, additional_permitted_classes: AVAILABLE_TAGS) + end + end + + class << self + def load!(content, input_values: {}, project: nil) + Loader.new(content, input_values: input_values, project: project).load! + end end end end diff --git a/lib/gitlab/ci/config/yaml/input_interpolator.rb b/lib/gitlab/ci/config/yaml/input_interpolator.rb new file mode 100644 index 0000000000000000000000000000000000000000..5c1f6f38c7ad8cc193d06684247b92c0c8ddc05b --- /dev/null +++ b/lib/gitlab/ci/config/yaml/input_interpolator.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class InputInterpolator + RequiredInputsNotMetError = Class.new(StandardError) + + def initialize(spec_inputs, content, input_values: {}) + @spec_inputs = spec_inputs + @content = content + @input_values = input_values + end + + def interpolate! + return content if spec_inputs.empty? + + raise RequiredInputsNotMetError, missing_required_inputs.join(', ') unless required_inputs_satisfied? + + interpolated_content + end + + private + + attr_reader :spec_inputs, :content, :input_values + + def required_inputs_satisfied? + missing_required_inputs.empty? + end + + def missing_required_inputs + required_inputs - input_values.keys + end + + def required_inputs + spec_inputs.keys.filter { |key| !spec_inputs[key]&.has_key?(:default) } + end + + def interpolated_content + ::Gitlab::Ci::Interpolation::Template.new(content, inputs: input_values).interpolated + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 8475c3a8b19dac0f112814b89a3c6ac98988fb67..bb513d0ea0762f5bced4cd691cdd97fc9707ec5d 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -117,6 +117,50 @@ def initialize(params, context) expect { valid? }.to raise_error(NotImplementedError) end end + + context 'when the file has required inputs' do + let(:test_class) do + Class.new(described_class) do + def initialize(params, context) + @location = params[:local] + + super + end + + def validate_context! + # no-op + end + end + end + + context 'when the inputs are given' do + let(:location) { { local: 'included.gitlab-ci.yml', with: { test_input: 'hello test' } } } + + before do + allow(::Gitlab::Ci::Config::Yaml).to receive(:load!).and_return({ test_job: { script: 'echo "hello test"' } }) + end + + it 'returns true' do + expect(subject).to be_truthy + end + end + + context 'when the inputs are missing' do + let(:location) { { local: 'included.gitlab-ci.yml' } } + + before do + allow(::Gitlab::Ci::Config::Yaml).to receive(:load!) + .and_raise(::Gitlab::Ci::Config::Yaml::InputInterpolator::RequiredInputsNotMetError, 'test_input') + end + + it 'returns false' do + expect(subject).to be_falsey + expect(file.error_message).to eq( + 'Included file `included.gitlab-ci.yml` has required inputs `test_input` that are not given!' + ) + end + end + end end describe '#to_hash' do diff --git a/spec/lib/gitlab/ci/config/yaml/input_interpolator_spec.rb b/spec/lib/gitlab/ci/config/yaml/input_interpolator_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..9b670856dec55a2e870afa9a150f0e96472d1dc7 --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml/input_interpolator_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Yaml::InputInterpolator, feature_category: :pipeline_authoring do + describe '#interpolate!' do + it 'interpolates all keys and values' do + content = { + '$[[ inputs.job_name ]]' => { + 'script' => 'echo "$[[ inputs.test_output ]]"' + } + } + spec_inputs = { + job_name: nil, + test_output: nil + } + input_values = { + job_name: 'test_job', + test_output: 'hello test' + } + + interpolated_content = described_class.new(spec_inputs, content, input_values: input_values).interpolate! + + expect(interpolated_content).to eq({ + 'test_job' => { + 'script' => 'echo "hello test"' + } + }) + end + + context 'when not given inputs' do + it 'returns the content with no changes' do + content = { + '$[[ inputs.job_name ]]' => { + 'script' => 'echo "$[[ inputs.test_output ]]"' + } + } + + interpolated_content = described_class.new({}, content).interpolate! + + expect(interpolated_content).to eq(content) + end + end + + context 'when there are required inputs missing' do + it 'raises a RequiredInputsNotMetError' do + content = { + '$[[ inputs.job_name ]]' => { + '$[[ inputs.script_keyword ]]' => 'echo "$[[ inputs.test_output ]]"' + } + } + spec_inputs = { + job_name: nil, + script_keyword: nil, + test_output: nil + } + input_values = { job_name: 'test_job' } + + interpolator = described_class.new(spec_inputs, content, input_values: input_values) + + expect { interpolator.interpolate! }.to raise_error( + described_class::RequiredInputsNotMetError, + 'script_keyword, test_output' + ) + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb index 4b34553f55e58660e165fd65d6775038324795d2..017aa069b1c3b4dc5c0fbb85b89ae67e84fa2374 100644 --- a/spec/lib/gitlab/ci/config/yaml_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -26,11 +26,10 @@ }) end - it 'loads the first document from a multi-doc YAML file' do + it 'loads the second document from a multi-doc YAML file' do yaml = <<~YAML spec: - inputs: - test_input: + version: 3.0 --- image: 'image:1.0' texts: @@ -42,14 +41,86 @@ config = described_class.load!(yaml) expect(config).to eq({ - spec: { - inputs: { - test_input: nil + image: 'image:1.0', + texts: { + nested_key: 'value1', + more_text: { + more_nested_key: 'value2' } } }) end + context 'when the loaded file accepts inputs' do + it 'interpolates the input values' do + yaml = <<~YAML + spec: + inputs: + job_name: + test_input: + --- + $[[ inputs.job_name ]]: + script: + - echo "$[[ inputs.test_input ]]" + YAML + + config = described_class.load!(yaml, input_values: { + job_name: 'test_job', + test_input: 'hello interpolation' + }) + + expect(config).to eq({ + test_job: { + script: ['echo "hello interpolation"'] + } + }) + end + + context 'when required inputs are missing values' do + it 'raises a RequiredInputsNotMetError' do + yaml = <<~YAML + spec: + inputs: + job_name: + test_input: + --- + $[[ inputs.job_name ]]: + script: + - echo "$[[ inputs.test_input ]]" + YAML + + expect { described_class.load!(yaml) }.to raise_error( + described_class::InputInterpolator::RequiredInputsNotMetError, + 'job_name, test_input' + ) + end + end + end + + context 'when the YAML file has an invalid `spec`' do + it 'treats the file as if it had no `spec`' do + yaml = <<~YAML + # invalid spec + # only comments :) + --- + test_job: + script: 'echo "hello"' + YAML + + config = described_class.load!(yaml) + + expect(config).to eq({ + test_job: { + script: 'echo "hello"' + } + }) + end + end + + context 'when not given a project' do + it 'only loads ' + end + context 'when ci_multi_doc_yaml is disabled' do before do stub_feature_flags(ci_multi_doc_yaml: false) diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 360686ce65c1002483aacb1ea720d0ba5eb3d8e7..0b652bdc58042e9118e0271409320128ba8e164a 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1429,6 +1429,57 @@ module Ci subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config), opts).execute } + context 'when inputs are required' do + let_it_be(:project) { create(:project, :repository) } + + let(:opts) { { project: project, sha: project.commit.sha } } + + let(:project_files) do + { + 'local.gitlab-ci.yml' => <<~YAML + spec: + inputs: + test_text: + --- + job1: + script: + - echo "$[[ inputs.test_text ]]" + YAML + } + end + + around do |example| + create_and_delete_files(project, project_files) do + example.run + end + end + + context 'when the required inputs are given' do + let(:include_content) do + [{ + local: '/local.gitlab-ci.yml', + with: { test_text: 'hello CI job' } + }] + end + + it 'interpolates the inputs' do + expect(subject.builds).to include(a_hash_including( + name: 'job1', + options: { script: ['echo "hello CI job"'] } + )) + end + end + + context 'when the required inputs are missing' do + let(:include_content) { [{ local: '/local.gitlab-ci.yml' }] } + + it_behaves_like( + 'returns errors', + 'Included file `local.gitlab-ci.yml` has required inputs `test_text` that are not given!' + ) + end + end + context "when validating a ci config file with no project context" do context "when a single string is provided" do let(:include_content) { "/local.gitlab-ci.yml" }