diff --git a/config/feature_flags/development/ci_refactor_external_rules.yml b/config/feature_flags/development/ci_refactor_external_rules.yml new file mode 100644 index 0000000000000000000000000000000000000000..117bff63648e3334c2a55b3cd641a6ea11d4afc3 --- /dev/null +++ b/config/feature_flags/development/ci_refactor_external_rules.yml @@ -0,0 +1,8 @@ +--- +name: ci_refactor_external_rules +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129145 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/421731 +milestone: '16.3' +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb index 8eaf9e35aaf55331cc75b2cd4b9fa3368988c6cc..71418e6752d08186990ecbec35768207a798d1dc 100644 --- a/lib/gitlab/ci/config/entry/include/rules.rb +++ b/lib/gitlab/ci/config/entry/include/rules.rb @@ -5,6 +5,12 @@ module Ci class Config module Entry class Include + ## + # Include rules are validated separately from all other entries. This + # is because included files are expanded before `@root.compose!` runs + # in Ci::Config. As such, this class is directly instantiated and + # composed in lib/gitlab/ci/config/external/rules.rb. + # class Rules < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable @@ -13,8 +19,9 @@ class Rules < ::Gitlab::Config::Entry::ComposableArray validates :config, type: Array end + # Remove this method when FF `ci_refactor_external_rules` is removed def value - @config + Feature.enabled?(:ci_refactor_external_rules) ? super : @config end def composable_class diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index 9cdbd8cd037dcd31a429f83d64f915dc389a079f..1a68e95913cfc86118253baf14c8f8bcaa078afe 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -14,9 +14,6 @@ class Rules::Rule < ::Gitlab::Config::Entry::Node attributes :if, :exists, :when - # Include rules are validated before Entry validations. This is because - # the include files are expanded before `compose!` runs in Ci::Config. - # The actual validation logic is in lib/gitlab/ci/config/external/rules.rb. validations do validates :config, presence: true validates :config, type: { with: Hash } @@ -24,8 +21,14 @@ class Rules::Rule < ::Gitlab::Config::Entry::Node with_options allow_nil: true do validates :if, expression: true + validates :exists, array_of_strings_or_string: true, allow_blank: true + validates :when, allowed_values: { in: ALLOWED_WHEN } end end + + def value + Feature.enabled?(:ci_refactor_external_rules) ? config.compact : super + end end end end diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 59e666b8bb5dae6cc4b15a43ade3ebd5e57964c9..0e6209460e0c28f098ba652199a50659e6e2de1d 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,15 +5,29 @@ module Ci class Config module External class Rules + # Remove these two constants when FF `ci_refactor_external_rules` is removed ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN InvalidIncludeRulesError = Class.new(Mapper::Error) def initialize(rule_hashes) - validate(rule_hashes) + if Feature.enabled?(:ci_refactor_external_rules) + return unless rule_hashes - @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + # We must compose the include rules entry here because included + # files are expanded before `@root.compose!` runs in Ci::Config. + rules_entry = Entry::Include::Rules.new(rule_hashes) + rules_entry.compose! + + raise InvalidIncludeRulesError, "include:#{rules_entry.errors.first}" unless rules_entry.valid? + + @rule_list = Build::Rules::Rule.fabricate_list(rules_entry.value) + else + validate(rule_hashes) + + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + end end def evaluate(context) @@ -32,6 +46,7 @@ def match_rule(context) @rule_list.find { |rule| rule.matches?(nil, context) } end + # Remove this method when FF `ci_refactor_external_rules` is removed def validate(rule_hashes) return unless rule_hashes.is_a?(Array) @@ -42,6 +57,7 @@ def validate(rule_hashes) end end + # Remove this method when FF `ci_refactor_external_rules` is removed def valid_when?(rule_hash) rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) end diff --git a/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb b/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb index 10c1d92e209bbb01ed713d2881ae7a32417fd107..dd15b049b9b896989fbb4c1a2574d50a5d9df576 100644 --- a/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb @@ -1,117 +1,132 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' # Change this to fast spec helper when FF `ci_refactor_external_rules` is removed require_dependency 'active_model' RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule, feature_category: :pipeline_composition do let(:factory) do - Gitlab::Config::Entry::Factory.new(described_class) - .value(config) + Gitlab::Config::Entry::Factory.new(described_class).value(config) end subject(:entry) { factory.create! } - describe '.new' do - shared_examples 'an invalid config' do |error_message| - it { is_expected.not_to be_valid } + before do + entry.compose! + end + + shared_examples 'a valid config' do + it { is_expected.to be_valid } + + it 'returns the expected value' do + expect(entry.value).to eq(config.compact) + end - it 'has errors' do - expect(entry.errors).to include(error_message) + context 'when FF `ci_refactor_external_rules` is disabled' do + before do + stub_feature_flags(ci_refactor_external_rules: false) end + + it 'returns the expected value' do + expect(entry.value).to eq(config) + end + end + end + + shared_examples 'an invalid config' do |error_message| + it { is_expected.not_to be_valid } + + it 'has errors' do + expect(entry.errors).to include(error_message) end + end - context 'when specifying an if: clause' do - let(:config) { { if: '$THIS || $THAT' } } + context 'when specifying an if: clause' do + let(:config) { { if: '$THIS || $THAT' } } - it { is_expected.to be_valid } + it_behaves_like 'a valid config' - context 'with when:' do - let(:config) { { if: '$THIS || $THAT', when: 'never' } } + context 'with when:' do + let(:config) { { if: '$THIS || $THAT', when: 'never' } } - it { is_expected.to be_valid } - end + it_behaves_like 'a valid config' end - context 'when specifying an exists: clause' do - let(:config) { { exists: './this.md' } } + context 'with when: ' do + let(:config) { { if: '$THIS || $THAT', when: 'on_success' } } - it { is_expected.to be_valid } + it_behaves_like 'an invalid config', /when unknown value: on_success/ end - context 'using a list of multiple expressions' do - let(:config) { { if: ['$MY_VAR == "this"', '$YOUR_VAR == "that"'] } } + context 'with when: null' do + let(:config) { { if: '$THIS || $THAT', when: nil } } - it_behaves_like 'an invalid config', /invalid expression syntax/ + it_behaves_like 'a valid config' end - context 'when specifying an invalid if: clause expression' do - let(:config) { { if: ['$MY_VAR =='] } } + context 'when if: clause is invalid' do + let(:config) { { if: '$MY_VAR ==' } } it_behaves_like 'an invalid config', /invalid expression syntax/ end - context 'when specifying an if: clause expression with an invalid token' do - let(:config) { { if: ['$MY_VAR == 123'] } } + context 'when if: clause has an integer operand' do + let(:config) { { if: '$MY_VAR == 123' } } it_behaves_like 'an invalid config', /invalid expression syntax/ end - context 'when using invalid regex in an if: clause' do - let(:config) { { if: ['$MY_VAR =~ /some ( thing/'] } } + context 'when if: clause has invalid regex' do + let(:config) { { if: '$MY_VAR =~ /some ( thing/' } } it_behaves_like 'an invalid config', /invalid expression syntax/ end - context 'when using an if: clause with lookahead regex character "?"' do + context 'when if: clause has lookahead regex character "?"' do let(:config) { { if: '$CI_COMMIT_REF =~ /^(?!master).+/' } } it_behaves_like 'an invalid config', /invalid expression syntax/ end - context 'when specifying unknown policy' do - let(:config) { { invalid: :something } } + context 'when if: clause has array of expressions' do + let(:config) { { if: ['$MY_VAR == "this"', '$YOUR_VAR == "that"'] } } - it_behaves_like 'an invalid config', /unknown keys: invalid/ + it_behaves_like 'an invalid config', /invalid expression syntax/ end + end + + context 'when specifying an exists: clause' do + let(:config) { { exists: './this.md' } } - context 'when clause is empty' do - let(:config) { {} } + it_behaves_like 'a valid config' - it_behaves_like 'an invalid config', /can't be blank/ + context 'when array' do + let(:config) { { exists: ['./this.md', './that.md'] } } + + it_behaves_like 'a valid config' end - context 'when policy strategy does not match' do - let(:config) { 'string strategy' } + context 'when null' do + let(:config) { { exists: nil } } - it_behaves_like 'an invalid config', /should be a hash/ + it_behaves_like 'a valid config' end end - describe '#value' do - subject(:value) { entry.value } - - context 'when specifying an if: clause' do - let(:config) { { if: '$THIS || $THAT' } } + context 'when specifying an unknown keyword' do + let(:config) { { invalid: :something } } - it 'returns the config' do - expect(subject).to eq(if: '$THIS || $THAT') - end + it_behaves_like 'an invalid config', /unknown keys: invalid/ + end - context 'with when:' do - let(:config) { { if: '$THIS || $THAT', when: 'never' } } + context 'when config is blank' do + let(:config) { {} } - it 'returns the config' do - expect(subject).to eq(if: '$THIS || $THAT', when: 'never') - end - end - end + it_behaves_like 'an invalid config', /can't be blank/ + end - context 'when specifying an exists: clause' do - let(:config) { { exists: './test.md' } } + context 'when config type is invalid' do + let(:config) { 'invalid' } - it 'returns the config' do - expect(subject).to eq(exists: './test.md') - end - end + it_behaves_like 'an invalid config', /should be a hash/ end end diff --git a/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb b/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb index d5988dbbb582ca380c980f7aa73bbed9b8dd3da3..05db81abfc16126a3c6e06511fd1c1dab7912baa 100644 --- a/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' # Change this to fast spec helper when FF `ci_refactor_external_rules` is removed require_dependency 'active_model' -RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules do +RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules, feature_category: :pipeline_composition do let(:factory) do Gitlab::Config::Entry::Factory.new(described_class) .value(config) @@ -77,23 +77,68 @@ describe '#value' do subject(:value) { entry.value } - context 'with an "if"' do - let(:config) do - [{ if: '$THIS == "that"' }] + let(:config) do + [ + { if: '$THIS == "that"' }, + { if: '$SKIP', when: 'never' } + ] + end + + it { is_expected.to eq([]) } + + context 'when composed' do + before do + entry.compose! end - it { is_expected.to eq(config) } + it 'returns the composed entries value' do + expect(entry).to be_valid + is_expected.to eq( + [ + { if: '$THIS == "that"' }, + { if: '$SKIP', when: 'never' } + ] + ) + end + + context 'when invalid' do + let(:config) do + [ + { if: '$THIS == "that"' }, + { if: '$SKIP', invalid: 'invalid' } + ] + end + + it 'returns the invalid config' do + expect(entry).not_to be_valid + is_expected.to eq(config) + end + end end - context 'with a list of two rules' do - let(:config) do - [ - { if: '$THIS == "that"' }, - { if: '$SKIP' } - ] + context 'when FF `ci_refactor_external_rules` is disabled' do + before do + stub_feature_flags(ci_refactor_external_rules: false) + end + + context 'with an "if"' do + let(:config) do + [{ if: '$THIS == "that"' }] + end + + it { is_expected.to eq(config) } end - it { is_expected.to eq(config) } + context 'with a list of two rules' do + let(:config) do + [ + { if: '$THIS == "that"' }, + { if: '$SKIP' } + ] + end + + it { is_expected.to eq(config) } + end end end end diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index c25e3eddc384fa45954491874da3e0e32b14923a..19113ce6a4e80b8cfa95d9d91b3f69cac5dcfce3 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -561,7 +561,17 @@ end it 'raises IncludeError' do - expect { subject }.to raise_error(described_class::IncludeError, /invalid include rule/) + expect { subject }.to raise_error(described_class::IncludeError, /contains unknown keys: changes/) + end + + context 'when FF `ci_refactor_external_rules` is disabled' do + before do + stub_feature_flags(ci_refactor_external_rules: false) + end + + it 'raises IncludeError' do + expect { subject }.to raise_error(described_class::IncludeError, /invalid include rule/) + end end end end diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb index 25b7998ef5ea18c2f597dd921e70c10d8c0743f9..8674af7ab65c7deb8da481f5c002c180db5b8927 100644 --- a/spec/lib/gitlab/ci/config/external/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -76,8 +76,7 @@ let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'on_success' }] } it 'raises an error' do - expect { result }.to raise_error(described_class::InvalidIncludeRulesError, - 'invalid include rule: {:if=>"$MY_VAR == \"hello\"", :when=>"on_success"}') + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, /when unknown value: on_success/) end end @@ -105,8 +104,7 @@ let(:rule_hashes) { [{ exists: 'Dockerfile', when: 'on_success' }] } it 'raises an error' do - expect { result }.to raise_error(described_class::InvalidIncludeRulesError, - 'invalid include rule: {:exists=>"Dockerfile", :when=>"on_success"}') + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, /when unknown value: on_success/) end end @@ -121,8 +119,94 @@ let(:rule_hashes) { [{ changes: ['$MY_VAR'] }] } it 'raises an error' do - expect { result }.to raise_error(described_class::InvalidIncludeRulesError, - 'invalid include rule: {:changes=>["$MY_VAR"]}') + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, /contains unknown keys: changes/) + end + end + + context 'when FF `ci_refactor_external_rules` is disabled' do + before do + stub_feature_flags(ci_refactor_external_rules: false) + end + + context 'when there is no rule' do + let(:rule_hashes) {} + + it { is_expected.to eq(true) } + end + + it_behaves_like 'when there is a rule with if' + + context 'when there is a rule with exists' do + let(:rule_hashes) { [{ exists: 'Dockerfile' }] } + + it_behaves_like 'when there is a rule with exists' + end + + context 'when there is a rule with if and when' do + context 'with when: never' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'never' }] } + + it_behaves_like 'when there is a rule with if', false, false + end + + context 'with when: always' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'always' }] } + + it_behaves_like 'when there is a rule with if' + end + + context 'with when: ' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'on_success' }] } + + it 'raises an error' do + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, + 'invalid include rule: {:if=>"$MY_VAR == \"hello\"", :when=>"on_success"}') + end + end + + context 'with when: null' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: nil }] } + + it_behaves_like 'when there is a rule with if' + end + end + + context 'when there is a rule with exists and when' do + context 'with when: never' do + let(:rule_hashes) { [{ exists: 'Dockerfile', when: 'never' }] } + + it_behaves_like 'when there is a rule with exists', false, false + end + + context 'with when: always' do + let(:rule_hashes) { [{ exists: 'Dockerfile', when: 'always' }] } + + it_behaves_like 'when there is a rule with exists' + end + + context 'with when: ' do + let(:rule_hashes) { [{ exists: 'Dockerfile', when: 'on_success' }] } + + it 'raises an error' do + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, + 'invalid include rule: {:exists=>"Dockerfile", :when=>"on_success"}') + end + end + + context 'with when: null' do + let(:rule_hashes) { [{ exists: 'Dockerfile', when: nil }] } + + it_behaves_like 'when there is a rule with exists' + end + end + + context 'when there is a rule with changes' do + let(:rule_hashes) { [{ changes: ['$MY_VAR'] }] } + + it 'raises an error' do + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, + 'invalid include rule: {:changes=>["$MY_VAR"]}') + end end end end