diff --git a/.rubocop_todo/rspec/spec_file_path_format.yml b/.rubocop_todo/rspec/spec_file_path_format.yml index 59aefa2ee33c78ac3033f3819e97b601882aeede..9a5af7eb6cd84a1580c6abb5535f3db1962e2c94 100644 --- a/.rubocop_todo/rspec/spec_file_path_format.yml +++ b/.rubocop_todo/rspec/spec_file_path_format.yml @@ -30,31 +30,4 @@ RSpec/SpecFilePathFormat: - 'spec/requests/api/issues/post_projects_issues_spec.rb' - 'spec/requests/api/issues/put_projects_issues_spec.rb' - 'spec/requests/api/pages/pages_spec.rb' - - 'spec/services/ci/create_pipeline_service/artifacts_spec.rb' - - 'spec/services/ci/create_pipeline_service/cache_spec.rb' - - 'spec/services/ci/create_pipeline_service/creation_errors_and_warnings_spec.rb' - - 'spec/services/ci/create_pipeline_service/cross_project_pipeline_spec.rb' - - 'spec/services/ci/create_pipeline_service/custom_config_content_spec.rb' - - 'spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb' - - 'spec/services/ci/create_pipeline_service/dry_run_spec.rb' - - 'spec/services/ci/create_pipeline_service/environment_spec.rb' - - 'spec/services/ci/create_pipeline_service/evaluate_runner_tags_spec.rb' - - 'spec/services/ci/create_pipeline_service/include_spec.rb' - - 'spec/services/ci/create_pipeline_service/inputs_spec.rb' - - 'spec/services/ci/create_pipeline_service/limit_active_jobs_spec.rb' - - 'spec/services/ci/create_pipeline_service/logger_spec.rb' - - 'spec/services/ci/create_pipeline_service/merge_requests_spec.rb' - - 'spec/services/ci/create_pipeline_service/needs_spec.rb' - - 'spec/services/ci/create_pipeline_service/parallel_spec.rb' - - 'spec/services/ci/create_pipeline_service/parameter_content_spec.rb' - - 'spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb' - - 'spec/services/ci/create_pipeline_service/partitioning_spec.rb' - - 'spec/services/ci/create_pipeline_service/pre_post_stages_spec.rb' - - 'spec/services/ci/create_pipeline_service/rate_limit_spec.rb' - - 'spec/services/ci/create_pipeline_service/rules_spec.rb' - - 'spec/services/ci/create_pipeline_service/run_spec.rb' - - 'spec/services/ci/create_pipeline_service/scripts_spec.rb' - - 'spec/services/ci/create_pipeline_service/stop_linting_spec.rb' - - 'spec/services/ci/create_pipeline_service/tags_spec.rb' - - 'spec/services/ci/create_pipeline_service/variables_spec.rb' - - 'spec/services/ci/create_pipeline_service/workflow_auto_cancel_spec.rb' + - 'spec/services/ci/create_pipeline_service/*.rb' diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 3765e9ec317ae32e8d0f65d60a48113cadc767c0..dd5c84a90495a700d02c37031cfd117fe30c66e5 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -169,7 +169,7 @@ def expand_config(config, inputs) def build_config(config, inputs) initial_config = logger.instrument(:config_yaml_load, once: true) do - Config::Yaml.load!(config, inputs, @context.variables) + Config::Yaml.load!(config, inputs, @context.variables, yaml_context) end initial_config = logger.instrument(:config_external_process, once: true) do @@ -212,6 +212,13 @@ def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_c logger: logger) end + def yaml_context + Config::Yaml::Context.new( + project: context.project, + pipeline: context.pipeline + ) + end + def build_variables(pipeline:) logger.instrument(:config_build_variables, once: true) do pipeline diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index d3f6faee4b86e9faca993e1dfd15f1369095a65e..e531ea29e45d0ef66b83a929e92d764361a296cc 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -13,7 +13,7 @@ class Context attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config, :parallel_requests, :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes, - :pipeline_policy_context + :pipeline_policy_context, :component attr_accessor :total_file_size_in_bytes @@ -25,7 +25,7 @@ class Context # rubocop:disable Metrics/ParameterLists -- all arguments needed def initialize( project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, - pipeline_config: nil, logger: nil, pipeline_policy_context: nil + pipeline_config: nil, logger: nil, pipeline_policy_context: nil, component: nil ) @project = project @pipeline = pipeline @@ -35,6 +35,7 @@ def initialize( @variables = variables || Ci::Variables::Collection.new @pipeline_config = pipeline_config @pipeline_policy_context = pipeline_policy_context + @component = component || {} @expandset = [] @parallel_requests = [] @execution_deadline = 0 @@ -86,6 +87,7 @@ def mutate(attrs = {}) ctx.max_includes = max_includes ctx.max_total_yaml_size_bytes = max_total_yaml_size_bytes ctx.parallel_requests = parallel_requests + ctx.component = component end end @@ -138,7 +140,7 @@ def internal_include? protected attr_writer :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes, - :parallel_requests + :parallel_requests, :component private diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index d2e7020e261bedfd357d11347a704da3a619e2c2..dd7a1ce30cec7e5d24ba2cdb7acef0c906fca383 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -117,12 +117,24 @@ def content_inputs def content_result context.logger.instrument(:config_file_fetch_content_hash) do ::Gitlab::Ci::Config::Yaml::Loader.new( - content, inputs: content_inputs, variables: context.variables + content, inputs: content_inputs, variables: context.variables, context: yaml_context ).load end end strong_memoize_attr :content_result + def yaml_context + Config::Yaml::Context.new(**yaml_context_attributes) + end + + def yaml_context_attributes + { + project: context.project, + pipeline: context.pipeline, + component: context.component + } + end + def expanded_content_hash return if content_result.content.blank? diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index b34c303785b7b0a37d4fa0664cb3dd27025a4f08..6c4426750136bf6a6d7b8688b5efd97192233139 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -83,10 +83,16 @@ def expand_context_attrs project: component_payload.fetch(:project), sha: component_payload.fetch(:sha), user: context.user, - variables: context.variables + variables: context.variables, + component: component_yaml_context } end + override :yaml_context_attributes + def yaml_context_attributes + super.merge(component: component_yaml_context) + end + def masked_blob return unless component_payload @@ -111,9 +117,14 @@ def component_attrs { project: component_payload.fetch(:project), sha: component_payload.fetch(:sha), - name: component_payload.fetch(:name) + name: component_payload.fetch(:name), + version: 'v1.2.3' # TODO } end + + def component_yaml_context + component_attrs.slice(:name, :sha, :version) + end end end end diff --git a/lib/gitlab/ci/config/header/context.rb b/lib/gitlab/ci/config/header/context.rb new file mode 100644 index 0000000000000000000000000000000000000000..32c329f13e39e2b03d54743c78f58d8653b42e48 --- /dev/null +++ b/lib/gitlab/ci/config/header/context.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + ## + # Context configuration used for interpolation with the CI configuration. + # + # This class defines the available context information that can be used + # in CI configuration interpolation, such as component, pipeline, and project details. + # + class Context < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_VALUES = %w[project pipeline component].freeze + + validations do + validates :config, type: Array, array_of_strings: true, allowed_array_values: { in: ALLOWED_VALUES } + end + + def value + return [] unless config.is_a?(Array) + + config.uniq + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/header/root.rb b/lib/gitlab/ci/config/header/root.rb index 251682d13b40107e00e3db546c2b92868b77972f..f08ca9eed025234b282e954ab42aec3cd98267d4 100644 --- a/lib/gitlab/ci/config/header/root.rb +++ b/lib/gitlab/ci/config/header/root.rb @@ -29,6 +29,10 @@ class Root < ::Gitlab::Config::Entry::Node def inputs_value spec_entry.inputs_value end + + def context_value + spec_entry.context_value + end end end end diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb index 4753c1eb4412b158d61fa694051f8ab57031bb15..14ea065231c3d232e1fdd2e19bdd0cd5fdeaf6d3 100644 --- a/lib/gitlab/ci/config/header/spec.rb +++ b/lib/gitlab/ci/config/header/spec.rb @@ -7,7 +7,7 @@ module Header class Spec < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[inputs].freeze + ALLOWED_KEYS = %i[inputs context].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -17,6 +17,11 @@ class Spec < ::Gitlab::Config::Entry::Node description: 'Allowed input parameters used for interpolation.', inherit: false, metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input } + + entry :context, Header::Context, + description: 'The available context used for interpolation.', + inherit: false, + default: [] end end end diff --git a/lib/gitlab/ci/config/interpolation/access.rb b/lib/gitlab/ci/config/interpolation/access.rb index 604caa8ab1f67f56819bbe744df7094685cc6b4d..1e601d02bb186d137f473061fded1a94ad0dde33 100644 --- a/lib/gitlab/ci/config/interpolation/access.rb +++ b/lib/gitlab/ci/config/interpolation/access.rb @@ -48,7 +48,7 @@ def evaluate! @value ||= objects.inject(@ctx) do |memo, value| key = value.to_sym - break @errors.push("unknown input name provided: `#{key}`") unless memo.key?(key) + break @errors.push("unknown interpolation provided: `#{key}`") unless memo.key?(key) memo.fetch(key) end diff --git a/lib/gitlab/ci/config/interpolation/context.rb b/lib/gitlab/ci/config/interpolation/context.rb index 19ea619f7da3758558c79f1015b1e22dfec9f85b..00e924cc6227ded25400052bdd9adff4a3da10d1 100644 --- a/lib/gitlab/ci/config/interpolation/context.rb +++ b/lib/gitlab/ci/config/interpolation/context.rb @@ -16,7 +16,7 @@ class Context attr_reader :variables - def initialize(data, variables: []) + def initialize(data, variables:) @data = data @variables = Ci::Variables::Collection.fabricate(variables) diff --git a/lib/gitlab/ci/config/interpolation/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb index 406234cc5fe6c7927510fc8633fd0326bc06b9e7..80b5e26240a77283b49ad358b777cb73208a8299 100644 --- a/lib/gitlab/ci/config/interpolation/interpolator.rb +++ b/lib/gitlab/ci/config/interpolation/interpolator.rb @@ -8,12 +8,13 @@ module Interpolation # Performs CI config file interpolation, and surfaces all possible interpolation errors. # class Interpolator - attr_reader :config, :args, :variables, :errors + attr_reader :config, :args, :variables, :yaml_context, :errors - def initialize(config, args, variables) + def initialize(config, args, variables, yaml_context) @config = config @args = args.nil? ? {} : args @variables = variables + @yaml_context = yaml_context @errors = [] @interpolated = false end @@ -88,12 +89,18 @@ def inputs end def context - @context ||= Context.new({ inputs: inputs.to_hash }, variables: variables) + @context ||= Context.new({ inputs: inputs.to_hash }.merge(yaml_context_hash), variables: variables) end def template @template ||= Template.new(content, context) end + + def yaml_context_hash + return {} unless yaml_context + + yaml_context.to_hash(header.context_value) + end end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index da5f3db209ca2d77dec10f25706635aee7062474..eb7308e8877daa07483256e28249dae04aa179f6 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -7,8 +7,8 @@ module Yaml LoadError = Class.new(StandardError) class << self - def load!(content, inputs = {}, variables = []) - Loader.new(content, inputs: inputs, variables: variables).load.then do |result| + def load!(content, inputs = {}, variables = [], context = nil) + Loader.new(content, inputs: inputs, variables: variables, context: context).load.then do |result| raise result.error_class, result.error if !result.valid? && result.error_class.present? raise LoadError, result.error unless result.valid? diff --git a/lib/gitlab/ci/config/yaml/context.rb b/lib/gitlab/ci/config/yaml/context.rb new file mode 100644 index 0000000000000000000000000000000000000000..895c486695e3970731a5965cc36ba04332233589 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/context.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Context + def initialize(project:, pipeline:, component: {}) + @project = project + @pipeline = pipeline + @component = component + end + + def to_hash(header_context) + { context: selected_data(header_context) } + end + + private + + attr_reader :project, :pipeline, :component + + def selected_data(header_context) + all_data.select do |key, _value| + header_context.include?(key.to_s) + end + end + + def all_data + { + project: { + id: project&.id, + full_path: project&.full_path + }.compact, + pipeline: { + ref: pipeline&.ref + }.compact, + component: { + name: component[:name], + sha: component[:sha], + version: component[:version] + }.compact + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index 0168def5f127fb74c998e066b028142cd3412b94..9a37c6492cc69edb238a6b1acf73c94b64e40b3b 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -10,10 +10,11 @@ class Loader AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, inputs: {}, variables: []) + def initialize(content, inputs: {}, variables: [], context: nil) @content = content @inputs = inputs @variables = variables + @context = context end def load @@ -21,7 +22,7 @@ def load return yaml_result unless yaml_result.valid? - interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables) + interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables, context) interpolator.interpolate! @@ -42,7 +43,7 @@ def load_uninterpolated_yaml private - attr_reader :content, :inputs, :variables + attr_reader :content, :inputs, :variables, :context def load_yaml! ensure_custom_tags diff --git a/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb b/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb index f716a8dcd75e03a96e2916c46bcfbe877a8924de..f06356cc29b97294e7e80aff214235e24e337e16 100644 --- a/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb +++ b/spec/lib/gitlab/ci/config/interpolation/interpolator_spec.rb @@ -7,7 +7,7 @@ let(:result) { ::Gitlab::Ci::Config::Yaml::Result.new(config: [header, content]) } - subject { described_class.new(result, arguments, []) } + subject { described_class.new(result, arguments, [], nil) } context 'when input data is valid' do let(:header) do diff --git a/spec/lib/gitlab/ci/config/yaml/fixtures/inputs-with-context-ci.yml b/spec/lib/gitlab/ci/config/yaml/fixtures/inputs-with-context-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..c3db8902524e5eb8b2839ff7f281dfe63c1b33e9 --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml/fixtures/inputs-with-context-ci.yml @@ -0,0 +1,16 @@ +spec: + context: [project, pipeline] + inputs: + compiler: + default: gcc + optimization_level: + type: number + default: 2 + +--- + +test: + script: + - echo "Building with $[[ inputs.compiler ]] and optimization level $[[ inputs.optimization_level ]]" + - echo "Project $[[ context.project.id ]] / $[[ context.project.full_path ]]" + - echo "Pipeline ref $[[ context.pipeline.ref ]]" diff --git a/spec/services/ci/create_pipeline_service/context_interpolation_spec.rb b/spec/services/ci/create_pipeline_service/context_interpolation_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..ccefd061319cce62e9d47743788ac46f73cd21d5 --- /dev/null +++ b/spec/services/ci/create_pipeline_service/context_interpolation_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::CreatePipelineService, feature_category: :pipeline_composition do + include RepoHelpers + + context 'for spec:context' do + let_it_be(:project) { create(:project, :small_repo) } + let_it_be(:user) { project.first_owner } + + let(:ref) { 'refs/heads/master' } + let(:source) { :push } + let(:content) { nil } + let(:inputs) { {} } + + let_it_be(:spec_context_example_path) { 'spec/lib/gitlab/ci/config/yaml/fixtures/inputs-with-context-ci.yml' } + let_it_be(:spec_context_example_yaml) { File.read(Rails.root.join(spec_context_example_path)) } + + let(:service) { described_class.new(project, user, { ref: ref }) } + + subject(:execute) { service.execute(source, content: content, inputs: inputs) } + + around do |example| + create_and_delete_files(project, { '.gitlab-ci.yml' => spec_context_example_yaml }) do + example.run + end + end + + it 'creates a pipeline with correct jobs' do + response = execute + pipeline = response.payload + + expect(response).to be_success + expect(pipeline).to be_created_successfully + + expect(pipeline.builds.map(&:name)).to contain_exactly('test') + + test_job = pipeline.builds.find { |build| build.name == 'test' } + expect(test_job.options[:script]).to eq([ + 'echo "Building with gcc and optimization level 2"', + "echo \"Project #{project.id} / #{project.full_path}\"", + 'echo "Pipeline ref master"' + ]) + end + end +end diff --git a/spec/services/ci/create_pipeline_service/including_ci_components_spec.rb b/spec/services/ci/create_pipeline_service/including_ci_components_spec.rb index b4f931b11af166ff8f52b78a49f66e020cd707d3..19d9a43d4f21e7e98029a047d8a00d1f61022ce5 100644 --- a/spec/services/ci/create_pipeline_service/including_ci_components_spec.rb +++ b/spec/services/ci/create_pipeline_service/including_ci_components_spec.rb @@ -112,7 +112,7 @@ expect(pipeline).to be_persisted expect(pipeline.error_messages[0].content) - .to include 'unknown input name provided: `suite`' + .to include 'unknown interpolation provided: `suite`' end end