diff --git a/config/feature_flags/development/ci_multi_doc_yaml.yml b/config/feature_flags/development/ci_multi_doc_yaml.yml new file mode 100644 index 0000000000000000000000000000000000000000..4e6289abefa492366579afcba23e65984584eeb9 --- /dev/null +++ b/config/feature_flags/development/ci_multi_doc_yaml.yml @@ -0,0 +1,8 @@ +--- +name: ci_multi_doc_yaml +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109137 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/388836 +milestone: '15.9' +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 142f0b8dfd8c3e0960bcf3b4fe0f3e9a3f628598..585e671ce426be3b81f27e631d153889eb5a544d 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -117,7 +117,8 @@ def metadata def expand_config(config) build_config(config) - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError, + Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index de833619c8df2469f7223cedd790caf4b5332c87..94ef0afe7f9fa45e3f5e3c99ca4b2d2ab9d015a2 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -5,12 +5,21 @@ module Ci class Config module Yaml AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + MAX_DOCUMENTS = 2 class << self def load!(content) ensure_custom_tags - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + if ::Feature.enabled?(:ci_multi_doc_yaml) + Gitlab::Config::Loader::MultiDocYaml.new( + content, + max_documents: MAX_DOCUMENTS, + additional_permitted_classes: AVAILABLE_TAGS + ).load!.first + else + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end end private diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb new file mode 100644 index 0000000000000000000000000000000000000000..346adc79896c5bdd70000a35ee7d8dcd13db88bd --- /dev/null +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Loader + class MultiDocYaml + TooManyDocumentsError = Class.new(Loader::FormatError) + DataTooLargeError = Class.new(Loader::FormatError) + NotHashError = Class.new(Loader::FormatError) + + MULTI_DOC_DIVIDER = /^---$/.freeze + + def initialize(config, max_documents:, additional_permitted_classes: []) + @max_documents = max_documents + @safe_config = load_config(config, additional_permitted_classes) + end + + def load! + raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents? + raise DataTooLargeError, 'The parsed YAML is too big' if too_big? + raise NotHashError, 'Invalid configuration format' unless all_hashes? + + safe_config.map(&:deep_symbolize_keys) + end + + private + + attr_reader :safe_config, :max_documents + + def load_config(config, additional_permitted_classes) + config.split(MULTI_DOC_DIVIDER).filter_map do |document| + YAML.safe_load(document, + permitted_classes: [Symbol, *additional_permitted_classes], + permitted_symbols: [], + aliases: true + ) + end + rescue Psych::Exception => e + raise Loader::FormatError, e.message + end + + def all_hashes? + safe_config.all?(Hash) + end + + def too_many_documents? + safe_config.count > max_documents + end + + def too_big? + !deep_sizes.all?(&:valid?) + end + + def deep_sizes + safe_config.map do |config| + Gitlab::Utils::DeepSize.new(config, + max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, + max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..4b34553f55e58660e165fd65d6775038324795d2 --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring do + describe '.load!' do + it 'loads a single-doc YAML file' do + yaml = <<~YAML + image: 'image:1.0' + texts: + nested_key: 'value1' + more_text: + more_nested_key: 'value2' + YAML + + config = described_class.load!(yaml) + + expect(config).to eq({ + image: 'image:1.0', + texts: { + nested_key: 'value1', + more_text: { + more_nested_key: 'value2' + } + } + }) + end + + it 'loads the first document from a multi-doc YAML file' do + yaml = <<~YAML + spec: + inputs: + test_input: + --- + image: 'image:1.0' + texts: + nested_key: 'value1' + more_text: + more_nested_key: 'value2' + YAML + + config = described_class.load!(yaml) + + expect(config).to eq({ + spec: { + inputs: { + test_input: nil + } + } + }) + end + + context 'when ci_multi_doc_yaml is disabled' do + before do + stub_feature_flags(ci_multi_doc_yaml: false) + end + + it 'loads a single-doc YAML file' do + yaml = <<~YAML + image: 'image:1.0' + texts: + nested_key: 'value1' + more_text: + more_nested_key: 'value2' + YAML + + config = described_class.load!(yaml) + + expect(config).to eq({ + image: 'image:1.0', + texts: { + nested_key: 'value1', + more_text: { + more_nested_key: 'value2' + } + } + }) + end + + it 'loads the first document from a multi-doc YAML file' do + yaml = <<~YAML + spec: + inputs: + test_input: + --- + image: 'image:1.0' + texts: + nested_key: 'value1' + more_text: + more_nested_key: 'value2' + YAML + + config = described_class.load!(yaml) + + expect(config).to eq({ + spec: { + inputs: { + test_input: nil + } + } + }) + end + end + end +end diff --git a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..bae98f9bc3557590f2d77ad8e8fa280d897da2c1 --- /dev/null +++ b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_authoring do + let(:loader) { described_class.new(yml, max_documents: 2) } + + describe '#load!' do + let(:yml) do + <<~YAML + spec: + inputs: + test_input: + --- + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to eq([ + { spec: { inputs: { test_input: nil } } }, + { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } + ]) + end + + context 'when the YAML file is empty' do + let(:yml) { '' } + + it 'returns an empty array' do + expect(loader.load!).to be_empty + end + end + + context 'when the parsed YAML is too big' do + let(:yml) do + <<~YAML + a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] + b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] + c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] + d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] + e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] + f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] + g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] + h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] + i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] + --- + a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] + b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] + c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] + d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] + e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] + f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] + g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] + h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] + i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] + YAML + end + + it 'raises a DataTooLargeError' do + expect { loader.load! }.to raise_error(described_class::DataTooLargeError, 'The parsed YAML is too big') + end + end + + context 'when a document is not a hash' do + let(:yml) do + <<~YAML + not_a_hash + --- + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'raises a NotHashError' do + expect { loader.load! }.to raise_error(described_class::NotHashError, 'Invalid configuration format') + end + end + + context 'when there are too many documents' do + let(:yml) do + <<~YAML + a: b + --- + c: d + --- + e: f + YAML + end + + it 'raises a TooManyDocumentsError' do + expect { loader.load! }.to raise_error( + described_class::TooManyDocumentsError, + 'The parsed YAML has too many documents' + ) + end + end + end +end diff --git a/spec/lib/gitlab/config/loader/yaml_spec.rb b/spec/lib/gitlab/config/loader/yaml_spec.rb index c7f84cd583c9d1163b7935a0675667993f2453ef..346424d1681429568a2a85f48100ebec2173b90d 100644 --- a/spec/lib/gitlab/config/loader/yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Loader::Yaml do +RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_authoring do let(:loader) { described_class.new(yml) } let(:yml) do