diff --git a/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml new file mode 100644 index 0000000000000000000000000000000000000000..dec187db4ab85b4170b66409929dc826dbc47621 --- /dev/null +++ b/config/feature_flags/development/ci_variable_expansion_in_rules_exists.yml @@ -0,0 +1,8 @@ +--- +name: ci_variable_expansion_in_rules_exists +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101639 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/381046 +milestone: '15.6' +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index beb35645492406694e2224abf6302ea717d88aab..c8436fc044d176f4a4f1184dabb21632f799d3a7 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -36,6 +36,7 @@ There are two places defined variables can be used. On the: | [`include`](../yaml/index.md#include) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.

See [Use variables with include](../yaml/includes.md#use-variables-with-include) for more information on supported variables. | | [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).
- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).
- [Persisted variables](#persisted-variables). | | [`resource_group`](../yaml/index.md#resource_group) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:
- `CI_ENVIRONMENT_URL`
- [Persisted variables](#persisted-variables). | +| [`rules:exists`](../yaml/index.md#rulesexists) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. | | [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).
- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).
- [Persisted variables](#persisted-variables). | | [`script`](../yaml/index.md#script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | | [`services:name`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index e8dd24cf7e07795532d1c848042c16f71c9c3f3c..f511a74b24483ed5ce469f9a0fd531e4571f6202 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -3437,7 +3437,8 @@ relative to `refs/heads/branch1` and the pipeline source is a merge request even #### `rules:exists` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. +> - CI/CD variable support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/283881) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_variable_expansion_in_rules_exists`. Disabled by default. Use `exists` to run a job when certain files exist in the repository. @@ -3445,8 +3446,7 @@ Use `exists` to run a job when certain files exist in the repository. **Possible inputs**: -- An array of file paths. Paths are relative to the project directory (`$CI_PROJECT_DIR`) - and can't directly link outside it. File paths can use glob patterns. +- An array of file paths. Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. File paths can use glob patterns and [CI/CD variables](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). **Example of `rules:exists`**: diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index aebd81e7b0798c32d08fb5d43893810cf95abbc9..5617d153bc88ca1e300961975db1845d92e700e9 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -9,20 +9,34 @@ class Rules::Rule::Clause::Exists < Rules::Rule::Clause MAX_PATTERN_COMPARISONS = 10_000 def initialize(globs) - globs = Array(globs) - - @top_level_only = globs.all?(&method(:top_level_glob?)) - @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) + @globs = Array(globs) + @top_level_only = @globs.all?(&method(:top_level_glob?)) end def satisfied_by?(_pipeline, context) paths = worktree_paths(context) + exact_globs, pattern_globs = separate_globs(context) - exact_matches?(paths) || pattern_matches?(paths) + exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end private + def separate_globs(context) + if ::Feature.enabled?(:ci_variable_expansion_in_rules_exists, context.project) + expanded_globs = expand_globs(context) + expanded_globs.partition(&method(:exact_glob?)) + else + @globs.partition(&method(:exact_glob?)) + end + end + + def expand_globs(context) + @globs.map do |glob| + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) + end + end + def worktree_paths(context) return [] unless context.project @@ -33,13 +47,16 @@ def worktree_paths(context) end end - def exact_matches?(paths) - @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } } + def exact_matches?(paths, exact_globs) + exact_globs.any? do |glob| + paths.bsearch { |path| glob <=> path } + end end - def pattern_matches?(paths) + def pattern_matches?(paths, pattern_globs) comparisons = 0 - @pattern_globs.any? do |glob| + + pattern_globs.any? do |glob| paths.any? do |path| comparisons += 1 comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path) diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb index f9ebab149a53652888fcf965dd0da55ad207d38c..c31d1c1fbf4c22a8b0ca660766a8c3aa250db21d 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb @@ -4,11 +4,47 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do describe '#satisfied_by?' do - shared_examples 'an exists rule with a context' do + subject(:satisfied_by?) { described_class.new(globs).satisfied_by?(nil, context) } + + shared_examples 'a rules:exists with a context' do it_behaves_like 'a glob matching rule' do let(:project) { create(:project, :custom_repo, files: files) } end + context 'when the rules:exists has a variable' do + let_it_be(:project) { create(:project, :custom_repo, files: { 'helm/helm_file.txt' => '' }) } + + let(:globs) { ['$HELM_DIR/**/*'] } + + let(:variables_hash) do + { 'HELM_DIR' => 'helm' } + end + + before do + allow(context).to receive(:variables_hash).and_return(variables_hash) + end + + context 'when the ci_variables_rules_exists FF is disabled' do + before do + stub_feature_flags(ci_variable_expansion_in_rules_exists: false) + end + + it { is_expected.to be_falsey } + end + + context 'when the ci_variables_rules_exists FF is enabled' do + context 'when the context has the specified variables' do + it { is_expected.to be_truthy } + end + + context 'when variable expansion does not match' do + let(:variables_hash) { {} } + + it { is_expected.to be_falsey } + end + end + end + context 'after pattern comparision limit is reached' do let(:globs) { ['*definitely_not_a_matching_glob*'] } let(:project) { create(:project, :repository) } @@ -22,26 +58,24 @@ end end - subject(:satisfied_by?) { described_class.new(globs).satisfied_by?(nil, context) } - - context 'when context is Build::Context::Build' do - it_behaves_like 'an exists rule with a context' do + context 'when the rules are being evaluated at job level' do + it_behaves_like 'a rules:exists with a context' do let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.commit.sha) } let(:context) { Gitlab::Ci::Build::Context::Build.new(pipeline, sha: project.repository.commit.sha) } end end - context 'when context is Build::Context::Global' do - it_behaves_like 'an exists rule with a context' do + context 'when the rules are being evaluated for an entire pipeline' do + it_behaves_like 'a rules:exists with a context' do let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.commit.sha) } let(:context) { Gitlab::Ci::Build::Context::Global.new(pipeline, yaml_variables: {}) } end end - context 'when context is Config::External::Context' do + context 'when rules are being evaluated with `include`' do let(:context) { Gitlab::Ci::Config::External::Context.new(project: project, sha: sha) } - it_behaves_like 'an exists rule with a context' do + it_behaves_like 'a rules:exists with a context' do let(:sha) { project.repository.commit.sha } end