diff --git a/config/feature_flags/development/ci_rules_variables.yml b/config/feature_flags/development/ci_rules_variables.yml new file mode 100644 index 0000000000000000000000000000000000000000..fdd9de1947287f4cee8b4277a43b3c91433eb786 --- /dev/null +++ b/config/feature_flags/development/ci_rules_variables.yml @@ -0,0 +1,8 @@ +--- +name: ci_rules_variables +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48752 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/289803 +milestone: '13.7' +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 9fdf489eefee5330b00d136142646caf1da03be2..42c779e060eeafcb893e3ca2b45c10017e3a6270 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -989,6 +989,7 @@ The job attributes you can use with `rules` are: - [`when`](#when): If not defined, defaults to `when: on_success`. - If used as `when: delayed`, `start_in` is also required. - [`allow_failure`](#allow_failure): If not defined, defaults to `allow_failure: false`. +- [`variables`](#rulesvariables): If not defined, uses the [variables defined elsewhere](#variables). If a rule evaluates to true, and `when` has any value except `never`, the job is included in the pipeline. @@ -1410,6 +1411,56 @@ job: In this example, if the first rule matches, then the job has `when: manual` and `allow_failure: true`. +#### `rules:variables` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209864) in GitLab 13.7. +> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-rulesvariables). **(CORE ONLY)** + +WARNING: +This feature might not be available to you. Check the **version history** note above for details. + +You can use [`variables`](#variables) in `rules:` to define variables for specific conditions. + +For example: + +```yaml +job: + variables: + DEPLOY_VARIABLE: "default-deploy" + rules: + - if: $CI_COMMIT_REF_NAME =~ /master/ + variables: # Override DEPLOY_VARIABLE defined + DEPLOY_VARIABLE: "deploy-production" # at the job level. + - if: $CI_COMMIT_REF_NAME =~ /feature/ + variables: + IS_A_FEATURE: "true" # Define a new variable. + script: + - echo "Run script with $DEPLOY_VARIABLE as an argument" + - echo "Run another script if $IS_A_FEATURE exists" +``` + +##### Enable or disable rules:variables **(CORE ONLY)** + +rules:variables is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:ci_rules_variables) +``` + +To disable it: + +```ruby +Feature.disable(:ci_rules_variables) +``` + #### Complex rule clauses To conjoin `if`, `changes`, and `exists` clauses with an `AND`, use them in the diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a500a0cc35dd15034ea28fefd36500ca785f7ed9..a39afee194c50d25a03e02bd8084a4ad6d730780 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,18 +6,31 @@ module Build class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure) do - def build_attributes + Result = Struct.new(:when, :start_in, :allow_failure, :variables) do + def build_attributes(seed_attributes = {}) { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure + allow_failure: allow_failure, + yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) }.compact end def pass? self.when != 'never' end + + private + + def yaml_variables(seed_variables) + return unless variables && seed_variables + + indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } + + variables.each_with_object(indexed_seed_variables) do |var, hash| + hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } + end.values + end end def initialize(rule_hashes, default_when:) @@ -32,7 +45,8 @@ def evaluate(pipeline, context) Result.new( matched_rule.attributes[:when] || @default_when, matched_rule.attributes[:start_in], - matched_rule.attributes[:allow_failure] + matched_rule.attributes[:allow_failure], + matched_rule.attributes[:variables] ) else Result.new('never') diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 8ffd49b8a93690b3c43d4ab67e7a76a7452b8769..840f2d6f31a0930e9e1edcbeb2cbc48821c1f23f 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -6,14 +6,18 @@ class Config module Entry class Rules::Rule < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable CLAUSES = %i[if changes exists].freeze - ALLOWED_KEYS = %i[if changes exists when start_in allow_failure].freeze + ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze attributes :if, :changes, :exists, :when, :start_in, :allow_failure + entry :variables, Entry::Variables, + description: 'Environment variables to define for rule conditions.' + validations do validates :config, presence: true validates :config, type: { with: Hash } diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c4dca2988395a0b66a3ee3393d1d519a36adb90d..c989b3112e1a16af5d7b4fdab9663652f6052cb0 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -66,6 +66,10 @@ def self.ci_pipeline_editor_page_enabled?(project) def self.allow_failure_with_exit_codes_enabled? ::Feature.enabled?(:ci_allow_failure_with_exit_codes) end + + def self.rules_variables_enabled?(project) + ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 105221dd6afc7b0a39821fab7fbd0a4354bd6fc4..2271915a72b279de5b8a5d28fa8ecd189e21186a 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -156,10 +156,12 @@ def pipeline_attributes def rules_attributes strong_memoize(:rules_attributes) do - if @using_rules - rules_result.build_attributes + next {} unless @using_rules + + if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) + rules_result.build_attributes(@seed_attributes) else - {} + rules_result.build_attributes end end end diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..faede7a361ff3ed33f98aee0ec58166dbf15d203 --- /dev/null +++ b/spec/lib/gitlab/ci/build/rules/rule/clause_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause do + describe '.fabricate' do + using RSpec::Parameterized::TableSyntax + + let(:value) { 'some value' } + + subject { described_class.fabricate(type, value) } + + context 'when type is valid' do + where(:type, :result) do + 'changes' | Gitlab::Ci::Build::Rules::Rule::Clause::Changes + 'exists' | Gitlab::Ci::Build::Rules::Rule::Clause::Exists + 'if' | Gitlab::Ci::Build::Rules::Rule::Clause::If + end + + with_them do + it { is_expected.to be_instance_of(result) } + end + end + + context 'when type is invalid' do + let(:type) { 'when' } + + it { is_expected.to be_nil } + + context "when type is 'variables'" do + let(:type) { 'variables' } + + it { is_expected.to be_nil } + end + end + end +end diff --git a/spec/lib/gitlab/ci/build/rules_spec.rb b/spec/lib/gitlab/ci/build/rules_spec.rb index ba99650396d943a17411a4218cea98bc88ceaa83..a1af5b75f87d206fc3f204c6a3318983a8141400 100644 --- a/spec/lib/gitlab/ci/build/rules_spec.rb +++ b/spec/lib/gitlab/ci/build/rules_spec.rb @@ -104,7 +104,7 @@ context 'with one rule without any clauses' do let(:rule_list) { [{ when: 'manual', allow_failure: true }] } - it { is_expected.to eq(described_class::Result.new('manual', nil, true)) } + it { is_expected.to eq(described_class::Result.new('manual', nil, true, nil)) } end context 'with one matching rule' do @@ -171,7 +171,7 @@ context 'with matching rule' do let(:rule_list) { [{ if: '$VAR == null', allow_failure: true }] } - it { is_expected.to eq(described_class::Result.new('on_success', nil, true)) } + it { is_expected.to eq(described_class::Result.new('on_success', nil, true, nil)) } end context 'with non-matching rule' do @@ -180,25 +180,61 @@ it { is_expected.to eq(described_class::Result.new('never')) } end end + + context 'with variables' do + context 'with matching rule' do + let(:rule_list) { [{ if: '$VAR == null', variables: { MY_VAR: 'my var' } }] } + + it { is_expected.to eq(described_class::Result.new('on_success', nil, nil, { MY_VAR: 'my var' })) } + end + end end describe 'Gitlab::Ci::Build::Rules::Result' do let(:when_value) { 'on_success' } let(:start_in) { nil } let(:allow_failure) { nil } + let(:variables) { nil } subject(:result) do - Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure) + Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure, variables) end describe '#build_attributes' do + let(:seed_attributes) { {} } + subject(:build_attributes) do - result.build_attributes + result.build_attributes(seed_attributes) end it 'compacts nil values' do is_expected.to eq(options: {}, when: 'on_success') end + + context 'when there are variables in rules' do + let(:variables) { { VAR1: 'new var 1', VAR3: 'var 3' } } + + context 'when there are seed variables' do + let(:seed_attributes) do + { yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true }, + { key: 'VAR2', value: 'var 2', public: true }] } + end + + it 'returns yaml_variables with override' do + is_expected.to include( + yaml_variables: [{ key: 'VAR1', value: 'new var 1', public: true }, + { key: 'VAR2', value: 'var 2', public: true }, + { key: 'VAR3', value: 'var 3', public: true }] + ) + end + end + + context 'when there is not seed variables' do + it 'does not return yaml_variables' do + is_expected.not_to have_key(:yaml_variables) + end + end + end end describe '#pass?' do @@ -206,7 +242,7 @@ let!(:when_value) { 'never' } it 'returns false' do - expect(subject.pass?).to eq(false) + expect(result.pass?).to eq(false) end end @@ -214,7 +250,7 @@ let!(:when_value) { 'on_success' } it 'returns true' do - expect(subject.pass?).to eq(true) + expect(result.pass?).to eq(true) end end end diff --git a/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb b/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb index 4a43e6c9a8619150331c46cc4bcc54c20722e2b2..d1bd22e55737e7412f8e4e084c62b69b7012161c 100644 --- a/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb @@ -339,6 +339,22 @@ end end end + + context 'with an invalid variables' do + let(:config) do + { if: '$THIS == "that"', variables: 'hello' } + end + + before do + subject.compose! + end + + it { is_expected.not_to be_valid } + + it 'returns an error about invalid variables:' do + expect(subject.errors).to include(/variables config should be a hash of key value pairs/) + end + end end context 'allow_failure: validation' do diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 3cbc498c3a47054c42051815b45a63e941dedc11..bc10e94c81d45dd6e95c4bf33be5c27fce693494 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -71,6 +71,33 @@ end end + context 'with job:rules:[variables:]' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true }, + { key: 'VAR2', value: 'var 2', public: true }], + rules: [{ if: '$VAR == null', variables: { VAR1: 'new var 1', VAR3: 'var 3' } }] } + end + + it do + is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'new var 1', public: true }, + { key: 'VAR2', value: 'var 2', public: true }, + { key: 'VAR3', value: 'var 3', public: true }]) + end + + context 'when FF ci_rules_variables is disabled' do + before do + stub_feature_flags(ci_rules_variables: false) + end + + it do + is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true }, + { key: 'VAR2', value: 'var 2', public: true }]) + end + end + end + context 'with cache:key' do let(:attributes) do { diff --git a/spec/services/ci/create_pipeline_service/rules_spec.rb b/spec/services/ci/create_pipeline_service/rules_spec.rb index a18c8f23ee78dcb22e1852c4c359dace9a03d442..ac6c4c188e400748b51fea8d4594f473c9de14a4 100644 --- a/spec/services/ci/create_pipeline_service/rules_spec.rb +++ b/spec/services/ci/create_pipeline_service/rules_spec.rb @@ -160,6 +160,81 @@ def find_job(name) end end end + + context 'if:' do + context 'variables:' do + let(:config) do + <<-EOY + job: + script: "echo job1" + variables: + VAR1: my var 1 + VAR2: my var 2 + rules: + - if: $CI_COMMIT_REF_NAME =~ /master/ + variables: + VAR1: overridden var 1 + - if: $CI_COMMIT_REF_NAME =~ /feature/ + variables: + VAR2: overridden var 2 + VAR3: new var 3 + - when: on_success + EOY + end + + let(:job) { pipeline.builds.find_by(name: 'job') } + + context 'when matching to the first rule' do + let(:ref) { 'refs/heads/master' } + + it 'overrides VAR1' do + variables = job.scoped_variables_hash + + expect(variables['VAR1']).to eq('overridden var 1') + expect(variables['VAR2']).to eq('my var 2') + expect(variables['VAR3']).to be_nil + end + + context 'when FF ci_rules_variables is disabled' do + before do + stub_feature_flags(ci_rules_variables: false) + end + + it 'does not affect variables' do + variables = job.scoped_variables_hash + + expect(variables['VAR1']).to eq('my var 1') + expect(variables['VAR2']).to eq('my var 2') + expect(variables['VAR3']).to be_nil + end + end + end + + context 'when matching to the second rule' do + let(:ref) { 'refs/heads/feature' } + + it 'overrides VAR2 and adds VAR3' do + variables = job.scoped_variables_hash + + expect(variables['VAR1']).to eq('my var 1') + expect(variables['VAR2']).to eq('overridden var 2') + expect(variables['VAR3']).to eq('new var 3') + end + end + + context 'when no match' do + let(:ref) { 'refs/heads/wip' } + + it 'does not affect vars' do + variables = job.scoped_variables_hash + + expect(variables['VAR1']).to eq('my var 1') + expect(variables['VAR2']).to eq('my var 2') + expect(variables['VAR3']).to be_nil + end + end + end + end end context 'when workflow:rules are used' do