diff --git a/ee/lib/gitlab/ci/project_config/compliance.rb b/ee/lib/gitlab/ci/project_config/compliance.rb index 4f1881a35d3c7bc519b7026a758bb551e2bdc0a3..ad0594724f14a97b34d45e75d5330769a1310aa2 100644 --- a/ee/lib/gitlab/ci/project_config/compliance.rb +++ b/ee/lib/gitlab/ci/project_config/compliance.rb @@ -16,6 +16,10 @@ def content end end + def contains_internal_include? + true + end + def source :compliance_source end diff --git a/ee/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/ee/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb index 778ae68f996e0c9e590017fa2b4790e98cc4b5ff..558dc6a8b5caf6a9bd10178b29a25e7f55f2afc8 100644 --- a/ee/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb +++ b/ee/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Config::Content do +RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :continuous_integration do let(:ci_config_path) { nil } let(:pipeline) { build(:ci_pipeline, project: project) } let(:content) { nil } @@ -36,6 +36,7 @@ expect(pipeline.config_source).to eq 'compliance_source' expect(pipeline.pipeline_config.content).to eq(content_result) expect(command.config_content).to eq(content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 585e671ce426be3b81f27e631d153889eb5a544d..f6c4e94a064463a21f482973a9b43efac363b120 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -21,14 +21,15 @@ class Config attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + # rubocop: disable Metrics/ParameterLists + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path @project = project @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline, pipeline_config: pipeline_config) end @context.set_deadline(TIMEOUT_SECONDS) @@ -49,6 +50,7 @@ def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_ rescue *rescue_errors => e raise Config::ConfigError, e.message end + # rubocop: enable Metrics/ParameterLists def valid? @root.valid? @@ -157,13 +159,14 @@ def find_sha(project) end end - def build_context(project:, pipeline:, sha:, user:, parent_pipeline:) + def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_config:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: build_variables(pipeline: pipeline), + pipeline_config: pipeline_config, logger: logger) end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 6eef279d3de843396efe9d3e6399babb1bea6f5f..ab0477e4d44f0063f23d6d5ede2142438ca53dca 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -14,20 +14,21 @@ class Context include ::Gitlab::Utils::StrongMemoize - attr_reader :project, :sha, :user, :parent_pipeline, :variables + attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger def initialize( project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, - logger: nil + pipeline_config: nil, logger: nil ) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline @variables = variables || Ci::Variables::Collection.new + @pipeline_config = pipeline_config @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) @@ -91,6 +92,12 @@ def includes expandset.map(&:metadata) end + # Some ProjectConfig sources inject an `include` into the config content. We use this + # method to exclude that `include` from the calculation of the total included files. + def contains_internal_include? + !!pipeline_config&.contains_internal_include? + end + protected attr_writer :expandset, :execution_deadline, :logger, :max_includes diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d2dc712e366a6b985680ed836a9aa974eb0fad8e..4bc2f6c7be70cb38c09743d49bdaf1a9de30e8f2 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -13,7 +13,8 @@ module Chain :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed, + :pipeline_config ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index d41213ef6dde3ebc38ac2ce52b6a01d34235cc16..779aac7d5209eb4e600423dab4a14371b11bba8f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -14,6 +14,7 @@ def perform! @pipeline.build_pipeline_config(content: pipeline_config.content) @command.config_content = pipeline_config.content @pipeline.config_source = pipeline_config.source + @command.pipeline_config = pipeline_config else error('Missing CI config file') end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index ad6b2fd341172f4e0d55b8bc73719a5fbacab37c..4976e075727b3edcbb81056bd9b690edbb07209f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -20,6 +20,7 @@ def perform! source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline, + pipeline_config: @command.pipeline_config, logger: logger } ) diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb index ded6877ef2915bb70187c240b9e3c1ff47097498..e69efb85a937f63d443c7307e12cb29dc7c7a641 100644 --- a/lib/gitlab/ci/project_config.rb +++ b/lib/gitlab/ci/project_config.rb @@ -26,6 +26,7 @@ def initialize(project:, sha:, custom_content: nil, pipeline_source: nil, pipeli end delegate :content, :source, to: :@config, allow_nil: true + delegate :contains_internal_include?, to: :@config def exists? !!@config&.exists? diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb index c6905f480a2a61fbf4596aa0d46c3132aa980104..70f8c7b9bb3f2385c6bf2dc13f3c8325e66ed6cc 100644 --- a/lib/gitlab/ci/project_config/auto_devops.rb +++ b/lib/gitlab/ci/project_config/auto_devops.rb @@ -13,6 +13,10 @@ def content end end + def contains_internal_include? + true + end + def source :auto_devops_source end diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb index 0ed5d6fa2268b8052191090ad45248c8a815a5e7..b29825514585cbd4e85ce36e84ce60c52a2000f9 100644 --- a/lib/gitlab/ci/project_config/external_project.rb +++ b/lib/gitlab/ci/project_config/external_project.rb @@ -17,6 +17,10 @@ def content end end + def contains_internal_include? + true + end + def source :external_project_source end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb index cf1292706d23720c64ccd39335e15e6abc4bcc65..17d5a9ee68fd9618c5dd74c7d053a8f72cd2d67c 100644 --- a/lib/gitlab/ci/project_config/remote.rb +++ b/lib/gitlab/ci/project_config/remote.rb @@ -12,6 +12,10 @@ def content end end + def contains_internal_include? + true + end + def source :remote_source end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 435ad4d42fe6c4ede68fb92a3db85e8dc339f818..c975023d35e27a1a7471f3c63a9030e9719330df 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -12,6 +12,10 @@ def content end end + def contains_internal_include? + true + end + def source :repository_source end diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index ebe5728163b007cd61d7cbe764773a36509594da..30edf1b552a5bb47ef58633c820c79ce2c35c8ac 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -24,6 +24,11 @@ def content raise NotImplementedError end + # Indicates if we are internally injecting an 'include' into the content + def contains_internal_include? + false + end + def source raise NotImplementedError end diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb index ee28137e54f2831e6be462892a6f62ad2f1ca3ec..bae898e13ebdfb37a74b2027f62d2b03566afca0 100644 --- a/spec/lib/gitlab/ci/config/external/context_spec.rb +++ b/spec/lib/gitlab/ci/config/external/context_spec.rb @@ -170,4 +170,29 @@ describe '#sentry_payload' do it { expect(subject.sentry_payload).to match(a_hash_including(:project, :user)) } end + + describe '#contains_internal_include?' do + context 'when pipeline_config is provided' do + let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) } + let(:attributes) do + { project: project, user: user, sha: sha, variables: variables, pipeline_config: pipeline_config } + end + + where(:value) { [true, false] } + + with_them do + it 'returns the value of .contains_internal_include?' do + allow(pipeline_config).to receive(:contains_internal_include?).and_return(value) + + expect(subject.contains_internal_include?).to eq(value) + end + end + end + + context 'when pipeline_config is not provided' do + it 'returns false' do + expect(subject.contains_internal_include?).to eq(false) + end + end + end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb index e0d656f456eddb15a8fe2896873aeea15e10d8e4..434acfb52742e11a705b2c0548f4edb6553fce48 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do +RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :continuous_integration do let(:project) { create(:project, ci_config_path: ci_config_path) } let(:pipeline) { build(:ci_pipeline, project: project) } let(:content) { nil } @@ -26,6 +26,7 @@ expect(pipeline.config_source).to eq 'bridge_source' expect(command.config_content).to eq 'the-yaml' + expect(command.pipeline_config.contains_internal_include?).to eq(false) end end @@ -52,6 +53,7 @@ expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end @@ -71,6 +73,7 @@ expect(pipeline.config_source).to eq 'remote_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end @@ -91,6 +94,7 @@ expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end context 'when path specifies a refname' do @@ -111,6 +115,7 @@ expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end end @@ -138,6 +143,7 @@ expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end @@ -161,6 +167,7 @@ expect(pipeline.config_source).to eq 'auto_devops_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.contains_internal_include?).to eq(true) end end @@ -181,6 +188,7 @@ expect(pipeline.config_source).to eq 'parameter_source' expect(pipeline.pipeline_config.content).to eq(content) expect(command.config_content).to eq(content) + expect(command.pipeline_config.contains_internal_include?).to eq(false) end end @@ -197,6 +205,7 @@ expect(pipeline.config_source).to eq('unknown_source') expect(pipeline.pipeline_config).to be_nil expect(command.config_content).to be_nil + expect(command.pipeline_config).to be_nil expect(pipeline.errors.full_messages).to include('Missing CI config file') end end diff --git a/spec/lib/gitlab/ci/project_config/repository_spec.rb b/spec/lib/gitlab/ci/project_config/repository_spec.rb index 2105b691d9ee356221f7fe2b98314c663f7fa557..b31a909934879bf981053c4007d925141b1b68f8 100644 --- a/spec/lib/gitlab/ci/project_config/repository_spec.rb +++ b/spec/lib/gitlab/ci/project_config/repository_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Repository do +RSpec.describe Gitlab::Ci::ProjectConfig::Repository, feature_category: :continuous_integration do let(:project) { create(:project, :custom_repo, files: files) } let(:sha) { project.repository.head_commit.sha } let(:files) { { 'README.md' => 'hello' } } @@ -44,4 +44,10 @@ it { is_expected.to eq(:repository_source) } end + + describe '#contains_internal_include?' do + subject { config.contains_internal_include? } + + it { is_expected.to eq(true) } + end end diff --git a/spec/lib/gitlab/ci/project_config/source_spec.rb b/spec/lib/gitlab/ci/project_config/source_spec.rb index dda5c7cdce83120d6e4ebe37f0be0786d05175b9..5248cf080e8df019aa8f3b0f24d060268eda104c 100644 --- a/spec/lib/gitlab/ci/project_config/source_spec.rb +++ b/spec/lib/gitlab/ci/project_config/source_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Source do +RSpec.describe Gitlab::Ci::ProjectConfig::Source, feature_category: :continuous_integration do let_it_be(:custom_config_class) { Class.new(described_class) } let_it_be(:project) { build_stubbed(:project) } let_it_be(:sha) { '123456' } @@ -20,4 +20,10 @@ it { expect { source }.to raise_error(NotImplementedError) } end + + describe '#contains_internal_include?' do + subject(:contains_internal_include) { custom_config.contains_internal_include? } + + it { expect(contains_internal_include).to eq(false) } + end end