From d516170ec6af112913ba4a61f640b970fa021e19 Mon Sep 17 00:00:00 2001 From: Laura Montemayor Date: Thu, 20 Oct 2022 14:49:25 +0200 Subject: [PATCH 1/5] Support variables in rules: exists Add variable expansion to the rules:exists --- .../ci/build/rules/rule/clause/exists.rb | 27 ++++++++----- .../ci/build/rules/rule/clause/exists_spec.rb | 39 +++++++++++++++---- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index aebd81e7b0798c..0532b22021c563 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -9,20 +9,26 @@ class Rules::Rule::Clause::Exists < Rules::Rule::Clause MAX_PATTERN_COMPARISONS = 10_000 def initialize(globs) - globs = Array(globs) - - @top_level_only = globs.all?(&method(:top_level_glob?)) - @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) + @globs = Array(globs) + @top_level_only = @globs.all?(&method(:top_level_glob?)) end def satisfied_by?(_pipeline, context) paths = worktree_paths(context) + expanded_globs = expand_globs(context) + exact_globs, pattern_globs = expanded_globs.partition(&method(:exact_glob?)) - exact_matches?(paths) || pattern_matches?(paths) + exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end private + def expand_globs(context) + @globs.map do |glob| + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) + end + end + def worktree_paths(context) return [] unless context.project @@ -33,13 +39,16 @@ def worktree_paths(context) end end - def exact_matches?(paths) - @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } } + def exact_matches?(paths, exact_globs) + exact_globs.any? do |glob| + paths.bsearch { |path| glob <=> path } + end end - def pattern_matches?(paths) + def pattern_matches?(paths, pattern_globs) comparisons = 0 - @pattern_globs.any? do |glob| + + pattern_globs.any? do |glob| paths.any? do |path| comparisons += 1 comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path) diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb index f9ebab149a5365..aa9299fd13dc89 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb @@ -4,11 +4,36 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do describe '#satisfied_by?' do - shared_examples 'an exists rule with a context' do + shared_examples 'a rules:exists with a context' do it_behaves_like 'a glob matching rule' do let(:project) { create(:project, :custom_repo, files: files) } end + context 'when the rules:exists has a variable' do + let(:globs) { ['$HELM_DIR/**/*'] } + let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + + context 'when the context has the specified variables' do + let(:variables_hash) do + { 'HELM_DIR' => 'helm' } + end + + before do + allow(context).to receive(:variables_hash).and_return(variables_hash) + end + + it { is_expected.to be_truthy } + end + + context 'when variable expansion does not match' do + before do + allow(context).to receive(:variables_hash).and_return({}) + end + + it { is_expected.to be_falsey } + end + end + context 'after pattern comparision limit is reached' do let(:globs) { ['*definitely_not_a_matching_glob*'] } let(:project) { create(:project, :repository) } @@ -24,24 +49,24 @@ subject(:satisfied_by?) { described_class.new(globs).satisfied_by?(nil, context) } - context 'when context is Build::Context::Build' do - it_behaves_like 'an exists rule with a context' do + context 'when the rules are being evaluated at job level' do + it_behaves_like 'a rules:exists with a context' do let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.commit.sha) } let(:context) { Gitlab::Ci::Build::Context::Build.new(pipeline, sha: project.repository.commit.sha) } end end - context 'when context is Build::Context::Global' do - it_behaves_like 'an exists rule with a context' do + context 'when the rules are being evaluated for an entire pipeline' do + it_behaves_like 'a rules:exists with a context' do let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.commit.sha) } let(:context) { Gitlab::Ci::Build::Context::Global.new(pipeline, yaml_variables: {}) } end end - context 'when context is Config::External::Context' do + context 'when rules are being evaluated with `include`' do let(:context) { Gitlab::Ci::Config::External::Context.new(project: project, sha: sha) } - it_behaves_like 'an exists rule with a context' do + it_behaves_like 'a rules:exists with a context' do let(:sha) { project.repository.commit.sha } end -- GitLab From de3da98f2a46b1678c4ad60e7de6b8e1d20cd5a0 Mon Sep 17 00:00:00 2001 From: Laura Montemayor Date: Tue, 1 Nov 2022 15:05:53 +0100 Subject: [PATCH 2/5] Adds ci_variable_rules_exist feature flag Also adds some specs --- .../ci_variable_expansion_in_rules_exists.yml | 8 ++++ .../ci/build/rules/rule/clause/exists.rb | 8 +++- .../ci/build/rules/rule/clause/exists_spec.rb | 39 ++++++++++++++----- 3 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml diff --git a/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml new file mode 100644 index 00000000000000..f8383e777dde0c --- /dev/null +++ b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml @@ -0,0 +1,8 @@ +--- +name: ci_variable_expansion_in_rules_exists +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101639 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338165 +milestone: '15.6' +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 0532b22021c563..92797bcb8a6278 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -15,8 +15,12 @@ def initialize(globs) def satisfied_by?(_pipeline, context) paths = worktree_paths(context) - expanded_globs = expand_globs(context) - exact_globs, pattern_globs = expanded_globs.partition(&method(:exact_glob?)) + if Feature.enabled?(:ci_variable_expansion_in_rules_exists) + expanded_globs = expand_globs(context) + exact_globs, pattern_globs = expanded_globs.partition(&method(:exact_glob?)) + else + exact_globs, pattern_globs = @globs.partition(&method(:exact_glob?)) + end exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb index aa9299fd13dc89..91423e4b875b9a 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb @@ -9,28 +9,47 @@ let(:project) { create(:project, :custom_repo, files: files) } end - context 'when the rules:exists has a variable' do - let(:globs) { ['$HELM_DIR/**/*'] } - let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } - - context 'when the context has the specified variables' do + context 'when the ci_variables_rules_exists FF is disabled' do + context 'when the rules:exists has a variable' do + let(:globs) { ['$HELM_DIR/**/*'] } + let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } let(:variables_hash) do { 'HELM_DIR' => 'helm' } end before do + stub_feature_flags(ci_variable_expansion_in_rules_exists: false) allow(context).to receive(:variables_hash).and_return(variables_hash) end - it { is_expected.to be_truthy } + it { is_expected.to be_falsey } end + end - context 'when variable expansion does not match' do - before do - allow(context).to receive(:variables_hash).and_return({}) + context 'when the ci_variables_rules_exists FF is enabled' do + context 'when the rules:exists has a variable' do + let(:globs) { ['$HELM_DIR/**/*'] } + let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + + context 'when the context has the specified variables' do + let(:variables_hash) do + { 'HELM_DIR' => 'helm' } + end + + before do + allow(context).to receive(:variables_hash).and_return(variables_hash) + end + + it { is_expected.to be_truthy } end - it { is_expected.to be_falsey } + context 'when variable expansion does not match' do + before do + allow(context).to receive(:variables_hash).and_return({}) + end + + it { is_expected.to be_falsey } + end end end -- GitLab From dd9805d83c59b6add905d4bbcec46c8cc93b5a3a Mon Sep 17 00:00:00 2001 From: Laura Montemayor Date: Wed, 2 Nov 2022 17:19:27 +0100 Subject: [PATCH 3/5] Updates with some feedback --- .../ci_variable_expansion_in_rules_exists.yml | 2 +- lib/gitlab/ci/build/rules/rule/clause/exists.rb | 15 ++++++++++----- .../ci/build/rules/rule/clause/exists_spec.rb | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml index f8383e777dde0c..dec187db4ab85b 100644 --- a/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml +++ b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml @@ -1,7 +1,7 @@ --- name: ci_variable_expansion_in_rules_exists introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101639 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338165 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/381046 milestone: '15.6' type: development group: group::pipeline authoring diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 92797bcb8a6278..da2dc1156dcf69 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -15,18 +15,23 @@ def initialize(globs) def satisfied_by?(_pipeline, context) paths = worktree_paths(context) + exact_globs, pattern_globs = separate_globs(context) + + exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) + end + + private + + def separate_globs(context) if Feature.enabled?(:ci_variable_expansion_in_rules_exists) expanded_globs = expand_globs(context) - exact_globs, pattern_globs = expanded_globs.partition(&method(:exact_glob?)) + expanded_globs.partition(&method(:exact_glob?)) else - exact_globs, pattern_globs = @globs.partition(&method(:exact_glob?)) + @globs.partition(&method(:exact_glob?)) end - exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end - private - def expand_globs(context) @globs.map do |glob| ExpandVariables.expand_existing(glob, -> { context.variables_hash }) diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb index 91423e4b875b9a..ab56ba9ec8c550 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb @@ -12,7 +12,7 @@ context 'when the ci_variables_rules_exists FF is disabled' do context 'when the rules:exists has a variable' do let(:globs) { ['$HELM_DIR/**/*'] } - let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } let(:variables_hash) do { 'HELM_DIR' => 'helm' } end @@ -29,7 +29,7 @@ context 'when the ci_variables_rules_exists FF is enabled' do context 'when the rules:exists has a variable' do let(:globs) { ['$HELM_DIR/**/*'] } - let(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } context 'when the context has the specified variables' do let(:variables_hash) do -- GitLab From 60a427452b0ecb1855369d9a1ba7589059d91665 Mon Sep 17 00:00:00 2001 From: Laura Montemayor Date: Wed, 2 Nov 2022 17:42:35 +0100 Subject: [PATCH 4/5] Adds docs --- doc/ci/variables/where_variables_can_be_used.md | 1 + doc/ci/yaml/index.md | 6 +++--- lib/gitlab/ci/build/rules/rule/clause/exists.rb | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index beb35645492406..c8436fc044d176 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -36,6 +36,7 @@ There are two places defined variables can be used. On the: | [`include`](../yaml/index.md#include) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.

See [Use variables with include](../yaml/includes.md#use-variables-with-include) for more information on supported variables. | | [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).
- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).
- [Persisted variables](#persisted-variables). | | [`resource_group`](../yaml/index.md#resource_group) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:
- `CI_ENVIRONMENT_URL`
- [Persisted variables](#persisted-variables). | +| [`rules:exists`](../yaml/index.md#rulesexists) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. | | [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).
- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).
- [Persisted variables](#persisted-variables). | | [`script`](../yaml/index.md#script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | | [`services:name`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index e8dd24cf7e0779..f511a74b24483e 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -3437,7 +3437,8 @@ relative to `refs/heads/branch1` and the pipeline source is a merge request even #### `rules:exists` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. +> - CI/CD variable support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/283881) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_variable_expansion_in_rules_exists`. Disabled by default. Use `exists` to run a job when certain files exist in the repository. @@ -3445,8 +3446,7 @@ Use `exists` to run a job when certain files exist in the repository. **Possible inputs**: -- An array of file paths. Paths are relative to the project directory (`$CI_PROJECT_DIR`) - and can't directly link outside it. File paths can use glob patterns. +- An array of file paths. Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. File paths can use glob patterns and [CI/CD variables](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). **Example of `rules:exists`**: diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index da2dc1156dcf69..25244121f363f0 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -29,7 +29,6 @@ def separate_globs(context) else @globs.partition(&method(:exact_glob?)) end - end def expand_globs(context) -- GitLab From cc57113773b55d84d5547e81718438b9f047dea1 Mon Sep 17 00:00:00 2001 From: Laura Montemayor Date: Fri, 4 Nov 2022 15:30:36 +0100 Subject: [PATCH 5/5] Refactors specs and adds context.project to feature flag --- .../ci/build/rules/rule/clause/exists.rb | 2 +- .../ci/build/rules/rule/clause/exists_spec.rb | 44 +++++++------------ 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 25244121f363f0..5617d153bc88ca 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -23,7 +23,7 @@ def satisfied_by?(_pipeline, context) private def separate_globs(context) - if Feature.enabled?(:ci_variable_expansion_in_rules_exists) + if ::Feature.enabled?(:ci_variable_expansion_in_rules_exists, context.project) expanded_globs = expand_globs(context) expanded_globs.partition(&method(:exact_glob?)) else diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb index ab56ba9ec8c550..c31d1c1fbf4c22 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb @@ -4,49 +4,41 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do describe '#satisfied_by?' do + subject(:satisfied_by?) { described_class.new(globs).satisfied_by?(nil, context) } + shared_examples 'a rules:exists with a context' do it_behaves_like 'a glob matching rule' do let(:project) { create(:project, :custom_repo, files: files) } end - context 'when the ci_variables_rules_exists FF is disabled' do - context 'when the rules:exists has a variable' do - let(:globs) { ['$HELM_DIR/**/*'] } - let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } - let(:variables_hash) do - { 'HELM_DIR' => 'helm' } - end + context 'when the rules:exists has a variable' do + let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + + let(:globs) { ['$HELM_DIR/**/*'] } + let(:variables_hash) do + { 'HELM_DIR' => 'helm' } + end + + before do + allow(context).to receive(:variables_hash).and_return(variables_hash) + end + + context 'when the ci_variables_rules_exists FF is disabled' do before do stub_feature_flags(ci_variable_expansion_in_rules_exists: false) - allow(context).to receive(:variables_hash).and_return(variables_hash) end it { is_expected.to be_falsey } end - end - - context 'when the ci_variables_rules_exists FF is enabled' do - context 'when the rules:exists has a variable' do - let(:globs) { ['$HELM_DIR/**/*'] } - let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + context 'when the ci_variables_rules_exists FF is enabled' do context 'when the context has the specified variables' do - let(:variables_hash) do - { 'HELM_DIR' => 'helm' } - end - - before do - allow(context).to receive(:variables_hash).and_return(variables_hash) - end - it { is_expected.to be_truthy } end context 'when variable expansion does not match' do - before do - allow(context).to receive(:variables_hash).and_return({}) - end + let(:variables_hash) { {} } it { is_expected.to be_falsey } end @@ -66,8 +58,6 @@ end end - subject(:satisfied_by?) { described_class.new(globs).satisfied_by?(nil, context) } - context 'when the rules are being evaluated at job level' do it_behaves_like 'a rules:exists with a context' do let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.commit.sha) } -- GitLab