diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ca9a350ea79b3c851c939adaa7b5725fc16c4230..35d14b6e2973d7e2e64455f472a48287a731ab8c 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -305,6 +305,10 @@ def stage_seeds @stage_seeds ||= config_processor.stage_seeds(self) end + def has_kubernetes_active? + project.kubernetes_service&.active? + end + def has_stage_seeds? stage_seeds.any? end diff --git a/app/views/ci/lints/_create.html.haml b/app/views/ci/lints/_create.html.haml index c91602fcff77b10c6e0b6c20be090bdc8aa29db7..30bf1384b229309f8f48e6eb16098ee873ee062e 100644 --- a/app/views/ci/lints/_create.html.haml +++ b/app/views/ci/lints/_create.html.haml @@ -22,10 +22,10 @@ %b Tag list: = build[:tag_list].to_a.join(", ") %br - %b Refs only: + %b Only policy: = @jobs[build[:name].to_sym][:only].to_a.join(", ") %br - %b Refs except: + %b Except policy: = @jobs[build[:name].to_sym][:except].to_a.join(", ") %br %b Environment: diff --git a/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..00c38a0c671f0509b87850eb09077979094b517c --- /dev/null +++ b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Add CI/CD active kubernetes job policy +merge_request: 13849 +author: +type: added diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index abf4ec7dbf8825afd4c45f22dece0e8fee7faa60..cacfd2ed2540535c5396908316427ca068489ec2 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -427,16 +427,16 @@ a "key: value" pair. Be careful when using special characters: are executed in `parallel`. For more info about the use of `stage` please check [stages](#stages). -### only and except +### only and except (simplified) -`only` and `except` are two parameters that set a refs policy to limit when -jobs are built: +`only` and `except` are two parameters that set a job policy to limit when +jobs are created: 1. `only` defines the names of branches and tags for which the job will run. 2. `except` defines the names of branches and tags for which the job will **not** run. -There are a few rules that apply to the usage of refs policy: +There are a few rules that apply to the usage of job policy: * `only` and `except` are inclusive. If both `only` and `except` are defined in a job specification, the ref is filtered by `only` and `except`. @@ -497,6 +497,36 @@ job: The above example will run `job` for all branches on `gitlab-org/gitlab-ce`, except master. +### only and except (complex) + +> Introduced in GitLab 10.0 + +> This an _alpha_ feature, and it it subject to change at any time without + prior notice! + +Since GitLab 10.0 it is possible to define a more elaborate only/except job +policy configuration. + +GitLab now supports both, simple and complex strategies, so it is possible to +use an array and a hash configuration scheme. + +Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to +simplified only/except configuration, whereas kubernetes strategy accepts only +`active` keyword. + +See the example below. Job is going to be created only when pipeline has been +scheduled or runs for a `master` branch, and only if kubernetes service is +active in the project. + +```yaml +job: + only: + refs: + - master + - schedules + kubernetes: active +``` + ### Job variables It is possible to define job variables using a `variables` keyword on a job diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0f955172e9885ab1ca2d75036b8dab..62b44389b152b6037bebdb854d15fc6b32e9db0e 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,24 +20,6 @@ def initialize(config, path = nil) raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) @@ -52,8 +34,7 @@ def builds def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end @@ -101,6 +82,34 @@ def self.validation_message(content) private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 3cdae1cee4f300fe37c37ed8476e9b831a092588..0027e9ec8c5d89a2e62c776406a84a26bc141450 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -7,6 +7,7 @@ module Entry # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } class RefsPolicy < Entry::Node include Entry::Validatable @@ -14,6 +15,27 @@ class RefsPolicy < Entry::Node validations do validates :config, array_of_strings_or_regexps: true end + + def value + { refs: @config } + end + end + + class ComplexPolicy < Entry::Node + include Entry::Validatable + include Entry::Attributable + + attributes :refs, :kubernetes + + validations do + validates :config, presence: true + validates :config, allowed_keys: %i[refs kubernetes] + + with_options allow_nil: true do + validates :refs, array_of_strings_or_regexps: true + validates :kubernetes, allowed_values: %w[active] + end + end end class UnknownStrategy < Entry::Node diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4024008d4db90bd2393e1a9461e256..0159179f0a983d5035876b965648e92c5d0e1920 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ def validate_each(record, attribute, value) end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index c70a4cb55fe19e675bcb4f82e55690474e92e66b..1efd3113a434ef5dcb5d0c45b3b7de21161b13e8 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -164,9 +164,46 @@ module Ci expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' end end + + context 'when kubernetes policy is specified' do + let(:pipeline) { create(:ci_empty_pipeline) } + + let(:config) do + YAML.dump( + spinach: { stage: 'test', script: 'spinach' }, + production: { + stage: 'deploy', + script: 'cap', + only: { kubernetes: 'active' } + } + ) + end + + context 'when kubernetes is active' do + let(:project) { create(:kubernetes_project) } + let(:pipeline) { create(:ci_empty_pipeline, project: project) } + + it 'returns seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 2 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + expect(seeds.second.builds.dig(0, :name)).to eq 'production' + end + end + + context 'when kubernetes is not active' do + it 'does not return seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 1 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + end + end + end end - describe "#builds_for_ref" do + describe "#builds_for_stage_and_ref" do let(:type) { 'test' } it "returns builds if no branch specified" do diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index 36a84da4a526d3775c048335e9c5042103b2495e..5e83abf645b87f1532aa0fdc5009876030ae4037 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -16,8 +16,8 @@ end describe '#value' do - it 'returns key value' do - expect(entry.value).to eq config + it 'returns refs hash' do + expect(entry.value).to eq(refs: config) end end end @@ -56,6 +56,50 @@ end end + context 'when using complex policy' do + context 'when specifiying refs policy' do + let(:config) { { refs: ['master'] } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(refs: %w[master]) + end + end + + context 'when specifying kubernetes policy' do + let(:config) { { kubernetes: 'active' } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(kubernetes: 'active') + end + end + + context 'when specifying invalid kubernetes policy' do + let(:config) { { kubernetes: 'something' } } + + it 'reports an error about invalid policy' do + expect(entry.errors).to include /unknown value: something/ + end + end + + context 'when specifying unknown policy' do + let(:config) { { refs: ['master'], invalid: :something } } + + it 'returns error about invalid key' do + expect(entry.errors).to include /unknown keys: invalid/ + end + end + + context 'when policy is empty' do + let(:config) { {} } + + it 'is not a valid configuration' do + expect(entry.errors).to include /can't be blank/ + end + end + end + context 'when policy strategy does not match' do let(:config) { 'string strategy' } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index b84e3ff18e8e2594b0cca48269d7eacf8ad091b6..84656ffe0b9b4c03a14eefbd5a536b54485f448a 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -546,6 +546,22 @@ def create_build(name, *traits, queued_at: current, started_from: 0, **opts) end end + describe '#has_kubernetes_active?' do + context 'when kubernetes is active' do + let(:project) { create(:kubernetes_project) } + + it 'returns true' do + expect(pipeline).to have_kubernetes_active + end + end + + context 'when kubernetes is not active' do + it 'returns false' do + expect(pipeline).not_to have_kubernetes_active + end + end + end + describe '#has_stage_seeds?' do context 'when pipeline has stage seeds' do subject { build(:ci_pipeline_with_one_job) } diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index b1743cd608e7ebbd01b7228d31afcf6adc356eb0..537cdadd528f4ec62a09303684bab4536450e78f 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -203,18 +203,13 @@ describe '#predefined_variables' do let(:kubeconfig) do - config = - YAML.load(File.read(expand_fixture_path('config/kubeconfig.yml'))) - - config.dig('users', 0, 'user')['token'] = - 'token' - + config_file = expand_fixture_path('config/kubeconfig.yml') + config = YAML.load(File.read(config_file)) + config.dig('users', 0, 'user')['token'] = 'token' + config.dig('contexts', 0, 'context')['namespace'] = namespace config.dig('clusters', 0, 'cluster')['certificate-authority-data'] = Base64.encode64('CA PEM DATA') - config.dig('contexts', 0, 'context')['namespace'] = - namespace - YAML.dump(config) end diff --git a/spec/views/ci/lints/show.html.haml_spec.rb b/spec/views/ci/lints/show.html.haml_spec.rb index 3390ae247ff2305d939b5569e21ff8aa00dd0ca9..f2c19c7642a8944f46eb14ea027d8c4f51e24b33 100644 --- a/spec/views/ci/lints/show.html.haml_spec.rb +++ b/spec/views/ci/lints/show.html.haml_spec.rb @@ -73,8 +73,8 @@ render expect(rendered).to have_content('Tag list: dotnet') - expect(rendered).to have_content('Refs only: test@dude/repo') - expect(rendered).to have_content('Refs except: deploy') + expect(rendered).to have_content('Only policy: refs, test@dude/repo') + expect(rendered).to have_content('Except policy: refs, deploy') expect(rendered).to have_content('Environment: testing') expect(rendered).to have_content('When: on_success') end