diff --git a/app/models/ci/workflow_event.rb b/app/models/ci/workflow_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..0980c9ca89b77828a91cd3a7999d972580f2d741 --- /dev/null +++ b/app/models/ci/workflow_event.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Ci + class WorkflowEvent < Ci::ApplicationRecord + belongs_to :project + belongs_to :user + + def self.upsert_workflow!(workflow, sha:, project:, user:) + entry = self.new( + sha: sha, + user: user, + project: project, + name: workflow.name, + uuid: workflow.uuid, + # TODO make it more robust and resilient + event_scopes: [project.id], + event_types: workflow.event_types, + event_groups: workflow.event_groups + # TODO: not supported yet + # event_ids: workflow.event_ids + ) + + entry.validate! + + self.upsert(entry.attributes.compact, returning: %w[uuid], unique_by: [:project_id, :name]) + end + + def self.find_matching(project, hook) + event_group_id = ::Gitlab::Ci::Workflows::Event.system_hook_to_index(hook) + + ::Ci::WorkflowEvent.where( + "event_scopes && '{?}' AND event_types && '{?}' AND event_groups && '{?}'", project.id, 1, event_group_id + ) + end + + def matches_event?(event) + name == event.workflow.to_s && + event_types.include?(event.type_id) && + event_groups.include?(event.group_id) + + ## event_ids.include?(event.event_id) TODO not supported yet + end + end +end diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb index 8ed6c54441bf16e332d222305096ffb7a07cba8f..471155ad6e39f37a2a117db8df60128ee73a5dfb 100644 --- a/app/models/concerns/enums/ci/pipeline.rb +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -39,7 +39,8 @@ def self.sources parent_pipeline: 12, ondemand_dast_scan: 13, ondemand_dast_validation: 14, - security_orchestration_policy: 15 + security_orchestration_policy: 15, + workflow: 16 } end diff --git a/app/models/concerns/triggerable_hooks.rb b/app/models/concerns/triggerable_hooks.rb index 8fe34632430b9de2df51469f18395f3529eaae11..97eae56837d8fda5eca289803d49cd99a8ed2e16 100644 --- a/app/models/concerns/triggerable_hooks.rb +++ b/app/models/concerns/triggerable_hooks.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true module TriggerableHooks + ## TODO + # CI Workflow depend on the order of the values defined in the hash below. Add new ones at the end. + # + # > Hashes enumerate their values in the order that the corresponding keys were inserted. + # + # https://ruby-doc.org/core-2.7.0/Hash.html + # AVAILABLE_TRIGGERS = { repository_update_hooks: :repository_update_events, push_hooks: :push_events, diff --git a/app/models/project.rb b/app/models/project.rb index 110f400d3ece3ec1bdb31406b6f39c5ef08c775a..badb549ae802bd64fba78433a8512a5084ab4282 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1612,6 +1612,13 @@ def execute_hooks(data, hooks_scope = :push_hooks) run_after_commit_or_now do triggered_hooks(hooks_scope, data).execute SystemHooksService.new.execute_hooks(data, hooks_scope) + ## + # TODO Ensure proper mult-database support + LFKs. + # TODO Ensure performance with project.has_workflows? or caching. + # TODO For a limited availability we can make this an opt-in feature and + # check it with project.workflow_events_feature_available? + # + Ci::DispatchWorkflowService.new(self).execute(data, hooks_scope) end end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 02f25a82307d6364178da2eb74069c24c1484e3e..1bb6d768a7cb51f11b366bf8cc221269c2aa46ee 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -21,6 +21,8 @@ class CreatePipelineService < BaseService Gitlab::Ci::Pipeline::Chain::Config::Process, Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig, Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs, + Gitlab::Ci::Pipeline::Chain::Workflows::Create, + Gitlab::Ci::Pipeline::Chain::Workflows::Select, Gitlab::Ci::Pipeline::Chain::SeedBlock, Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules, Gitlab::Ci::Pipeline::Chain::Seed, diff --git a/app/services/ci/dispatch_workflow_service.rb b/app/services/ci/dispatch_workflow_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..32a8e041f042db0976a8712b362bed9c060223e0 --- /dev/null +++ b/app/services/ci/dispatch_workflow_service.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Ci + class DispatchWorkflowService < ::BaseService + def execute(data, hook) + # rubocop:disable CodeReuse/ActiveRecord + + return unless ::Ci::WorkflowEvent.where(project: project).any? + + # rubocop:enable CodeReuse/ActiveRecord + + workflows = ::Ci::WorkflowEvent.find_matching(project, hook) + + ## TODO + # + # Populate pipeline with a file-type variable with `data`. + # + + workflows.each do |workflow| + params = { ignore_skip_ci: true, save_on_errors: true, workflow_id: workflow.id } + + ## TODO + # + # Pipeline should run for the ref that the workflow has been defined for! + # + # Use something like `workflow.ref` instead of `project.default_branch` below. + # + ::CreatePipelineWorker.perform_async( + project.id, + workflow.user.id, + project.default_branch, + :workflow, + **params + ) + end + end + end +end diff --git a/db/migrate/20220628100300_add_ci_workflow_events.rb b/db/migrate/20220628100300_add_ci_workflow_events.rb new file mode 100644 index 0000000000000000000000000000000000000000..2922d371b9e4e553b0d5ede92658ef561ac6853a --- /dev/null +++ b/db/migrate/20220628100300_add_ci_workflow_events.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddCiWorkflowEvents < Gitlab::Database::Migration[2.0] + def change + create_table :ci_workflow_events do |t| + t.timestamps_with_timezone null: false, default: -> { 'NOW()' } + t.references :project, null: false, index: true, foreign_key: false # TODO LFKs + t.references :user, null: false, foreign_key: false # TODO LFKs + t.bigint :event_scopes, null: false, array: true, default: [] + t.integer :event_types, null: false, array: true, default: [] + t.integer :event_groups, null: false, array: true, default: [] + t.integer :event_ids, null: false, array: true, default: [] + t.text :name, limit: 255, null: false + t.text :sha, limit: 40, null: false + t.uuid :uuid, null: false + + t.index [:project_id, :name], unique: true, + name: 'ci_workflow_events_project_id_name_index' + t.index [:event_scopes, :event_types, :event_groups, :event_ids], + using: 'GIN', name: 'ci_workflow_events_gin_index' + end + end +end diff --git a/db/schema_migrations/20220628100300 b/db/schema_migrations/20220628100300 new file mode 100644 index 0000000000000000000000000000000000000000..35f724c056b616a7c2b9ed6bb9ea52e06e76a570 --- /dev/null +++ b/db/schema_migrations/20220628100300 @@ -0,0 +1 @@ +2d4653fe2a054d5542af513f82cb54ef683037558f66a5201d4ece611e611b88 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7a966564e06dacf1902815b2121e1f099448a746..ae06e678f2add5003103491aa0fa10a7baa04474 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13314,6 +13314,32 @@ CREATE SEQUENCE ci_variables_id_seq ALTER SEQUENCE ci_variables_id_seq OWNED BY ci_variables.id; +CREATE TABLE ci_workflow_events ( + id bigint NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + project_id bigint NOT NULL, + user_id bigint NOT NULL, + event_scopes bigint[] DEFAULT '{}'::bigint[] NOT NULL, + event_types integer[] DEFAULT '{}'::integer[] NOT NULL, + event_groups integer[] DEFAULT '{}'::integer[] NOT NULL, + event_ids integer[] DEFAULT '{}'::integer[] NOT NULL, + name text NOT NULL, + sha text NOT NULL, + uuid uuid NOT NULL, + CONSTRAINT check_57c0faffdd CHECK ((char_length(name) <= 255)), + CONSTRAINT check_d454861a71 CHECK ((char_length(sha) <= 40)) +); + +CREATE SEQUENCE ci_workflow_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_workflow_events_id_seq OWNED BY ci_workflow_events.id; + CREATE TABLE cluster_agent_tokens ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -22771,6 +22797,8 @@ ALTER TABLE ONLY ci_unit_tests ALTER COLUMN id SET DEFAULT nextval('ci_unit_test ALTER TABLE ONLY ci_variables ALTER COLUMN id SET DEFAULT nextval('ci_variables_id_seq'::regclass); +ALTER TABLE ONLY ci_workflow_events ALTER COLUMN id SET DEFAULT nextval('ci_workflow_events_id_seq'::regclass); + ALTER TABLE ONLY cluster_agent_tokens ALTER COLUMN id SET DEFAULT nextval('cluster_agent_tokens_id_seq'::regclass); ALTER TABLE ONLY cluster_agents ALTER COLUMN id SET DEFAULT nextval('cluster_agents_id_seq'::regclass); @@ -24510,6 +24538,9 @@ ALTER TABLE ONLY ci_unit_tests ALTER TABLE ONLY ci_variables ADD CONSTRAINT ci_variables_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_workflow_events + ADD CONSTRAINT ci_workflow_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY cluster_agent_tokens ADD CONSTRAINT cluster_agent_tokens_pkey PRIMARY KEY (id); @@ -26798,6 +26829,10 @@ CREATE INDEX cadence_create_iterations_automation ON iterations_cadences USING b CREATE INDEX ci_builds_gitlab_monitor_metrics ON ci_builds USING btree (status, created_at, project_id) WHERE ((type)::text = 'Ci::Build'::text); +CREATE INDEX ci_workflow_events_gin_index ON ci_workflow_events USING gin (event_scopes, event_types, event_groups, event_ids); + +CREATE UNIQUE INDEX ci_workflow_events_project_id_name_index ON ci_workflow_events USING btree (project_id, name); + CREATE INDEX code_owner_approval_required ON protected_branches USING btree (project_id, code_owner_approval_required) WHERE (code_owner_approval_required = true); CREATE UNIQUE INDEX commit_user_mentions_on_commit_id_and_note_id_unique_index ON commit_user_mentions USING btree (commit_id, note_id); @@ -27598,6 +27633,10 @@ CREATE INDEX index_ci_variables_on_key ON ci_variables USING btree (key); CREATE UNIQUE INDEX index_ci_variables_on_project_id_and_key_and_environment_scope ON ci_variables USING btree (project_id, key, environment_scope); +CREATE INDEX index_ci_workflow_events_on_project_id ON ci_workflow_events USING btree (project_id); + +CREATE INDEX index_ci_workflow_events_on_user_id ON ci_workflow_events USING btree (user_id); + CREATE INDEX index_cicd_settings_on_namespace_id_where_stale_pruning_enabled ON namespace_ci_cd_settings USING btree (namespace_id) WHERE (allow_stale_runner_pruning = true); CREATE INDEX index_cluster_agent_tokens_on_agent_id_status_last_used_at ON cluster_agent_tokens USING btree (agent_id, status, last_used_at DESC NULLS LAST); diff --git a/lib/gitlab/ci/config/entry/events.rb b/lib/gitlab/ci/config/entry/events.rb new file mode 100644 index 0000000000000000000000000000000000000000..bbb7ab5ea565df3263f5701a797def51f3362ad0 --- /dev/null +++ b/lib/gitlab/ci/config/entry/events.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Events < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, array_of_strings_or_string: true + + ## TODO + # + # In YAML processor we will also need to check if proper webhooks are being used. This should be done on the + # logical-validaton level in YAML processor, not here, on the entry-level. + # + # We can also check if the provided event path conforms with the regexp defined in Workflows::Event class. + # + # Initially we also do not support pipeline-related events to avoid circular workflows. + validate do + Array(config).each do |event| + unless event.include?('/webhooks/') + errors.add(:config, 'only webhooks events are supported') + end + + if event.match?(%r{webhooks/job|webhooks/pipeline|webhooks/deployment}) + errors.add(:config, 'pipeline related events are not supported yet') + end + end + end + end + + # TODO remove duplication, return a hierarchical hash + def value + Array(@config) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7513936a18a55eb6395e894f33484524b44ca1c2..b9423d925b6cb0291bcd0f020e3abc10b7936e69 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ class Job < ::Gitlab::Config::Entry::Node ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - release].freeze + release on].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -100,6 +100,10 @@ class Job < ::Gitlab::Config::Entry::Node description: 'Environment configuration for this job.', inherit: false + entry :on, Entry::Events, + description: 'Events a job will be triggered by.', + inherit: false + entry :coverage, Entry::Coverage, description: 'Coverage configuration for this job.', inherit: false @@ -143,6 +147,7 @@ def value cache: cache_value, tags: tags_value, when: self.when, + on: on_value, start_in: self.start_in, dependencies: dependencies, environment: environment_defined? ? environment_value : nil, diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index de833619c8df2469f7223cedd790caf4b5332c87..561d724688b8495a07e41300e435a99cc8dd4d52 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -10,6 +10,20 @@ class << self def load!(content) ensure_custom_tags + ## + # TODO + # + # A hack to solve `on:` YAML serialization to `true`: + # + # pry > YAML.load('abc: something') + # => {"abc"=>"something"} + # pry > YAML.load('on: something') + # => {true=>"something"} + # + # This needs to be changed!!! + # + content = content.gsub(/\s\son:/, ' "on":') + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 0a6f6fd740c2435d6c9a905f22ee1696c08eaeb9..1af9a5a3e15baecd65c30947abc6a74c4bd61b61 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Chain :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, :workflow_id, # These attributes are set by Chains during processing: :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do @@ -76,6 +76,12 @@ def ambiguous_ref? end end + def workflow + strong_memoize(:workflow) do + ::Ci::WorkflowEvent.find(workflow_id) if workflow_id.present? + end + end + def parent_pipeline bridge&.parent_pipeline end diff --git a/lib/gitlab/ci/pipeline/chain/workflows/create.rb b/lib/gitlab/ci/pipeline/chain/workflows/create.rb new file mode 100644 index 0000000000000000000000000000000000000000..ca069c3ff5cc80af1c03d259eac612c1a372582e --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/workflows/create.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Workflows + ## + # This pipeline chain class is responsible for creating workflows events defined in a pipeline. + # + class Create < Chain::Base + def perform! + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result + + events = [] + + @command.yaml_processor_result.jobs.each do |name, config| + events.concat(config.dig(:on)) if config.key?(:on) + end + + events.map! do |event| + ::Gitlab::Ci::Workflows::Event.new(event) + end + + workflows = ::Gitlab::Ci::Workflows::Workflow.fabricate(events) + + ensure_workflows!(workflows) if workflows.any? + + ## + # TODO remove workflows when these get removed from `.gitlab-ci.yml`. + # + end + + def break? + false + end + + private + + def ensure_workflows!(workflows) + ## + # TODO There are a few ways to make it better, we can for example, check if there are any changes in UUIDs + # to only upsert if we know that we need to. + # + # TODO how we create and update workflows needs more tests, benchmarks and making sure that it works well. + # Here it is just PoC, we would need to improve this area to "productionize" this code. + # + # TODO in this PoC creating a workflow defines ownership over pipelines that will be triggered by a + # workflow. It means that if user A defines a workflow/webhooks/issues/created, and user B creates an + # issue that will trigger it, it will be the user A who will be an owner of a pipeline and its builds. + # + workflows.each do |workflow| + attributes = { sha: pipeline.sha, project: pipeline.project, user: pipeline.user } + + ## TODO add `ref` to workflow event! + # + ::Ci::WorkflowEvent.upsert_workflow!(workflow, **attributes) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/workflows/select.rb b/lib/gitlab/ci/pipeline/chain/workflows/select.rb new file mode 100644 index 0000000000000000000000000000000000000000..9ac1ba25a752da6c9c8588ec5c8a5eab44196713 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/workflows/select.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + ## + # TODO we can have only a single workflow per pipeline, so we can safely rename this module to `Workflow`. + # + module Workflows + ## + # This pipeline chain class is responsible for selecting jobs to run based on defined workflow. + # + class Select < Chain::Base + def perform! + ## + # + # Select builds with on: that matches the workflow along with their `needs:` builds in subsequent stages. + # + # TODO + # + # We are doing it here in a similar way this is done in Chain::RemoveUnwantedChatJobs. Both ways are + # wrong, and we need to refactor both into the new mechanism that will integrate better with pipeline + # seeds. + + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result + + selected = [] + + unless pipeline.workflow? + @command.yaml_processor_result.jobs.reject! do |name, config| + selected.push(name) if config.key?(:on) + end + + @command.yaml_processor_result.jobs.reject! do |_, config| + needs = config.dig(:needs, :job).to_a + .map { |hash| hash[:name].to_sym } + + (needs & selected).present? if needs.any? + end + + return + end + + @command.yaml_processor_result.jobs.select! do |name, config| + if config.key?(:on) + config.dig(:on).any? do |on| + event = ::Gitlab::Ci::Workflows::Event.new(on) + + if @command.workflow.matches_event?(event) + selected.push(name) + end + end + else + needs = config.dig(:needs, :job).to_a + .map { |hash| hash[:name].to_sym } + + (needs & selected).present? if needs.any? + end + end + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/workflows/event.rb b/lib/gitlab/ci/workflows/event.rb new file mode 100644 index 0000000000000000000000000000000000000000..adcc62785c6dc5592184c3144974d87b9cef76ff --- /dev/null +++ b/lib/gitlab/ci/workflows/event.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Workflows + class Event + include Comparable + + # TODO + # + # Rename this to 'hooks' as there is no "web" component in the implementation. + # + EVENT_GROUPS = { + webhooks: 1 # we can probably extract this definion to a better place + }.freeze + + # TODO event named group contains leading slash as it is optional + # rubocop:disable Lint/MixedRegexpCaptureTypes TODO + PATH_REGEXP = %r{(?[\w-]+)/webhooks/(?[\w-]+)(?[/\w-]+)?}.freeze + # rubocop:enable Lint/MixedRegexpCaptureTypes + + attr_reader :path + + def initialize(path) + @path = path + @matches = path.match(PATH_REGEXP) + end + + def valid? + @matches.present? && self.class.webhooks.include?(@matches[:group]) + end + + def workflow + @matches[:name] + end + + def type_id + EVENT_GROUPS[:webhooks] + end + + def group_id + raise ArgumentError unless valid? + + @index ||= self.class.webhook_index(@matches[:group]) + end + + ## TODO + # + # Webhooks are not granular enough to expose concrete events. We would need to extend them to support events + # like issue/created, issue/updated. This information is available in a payload, but we need additional code to + # connect it with workflow events. + # + # TODO: Make it possible to parse a payload before we create a pipeline. We will need to use pipeline + # expressions combined with a JSON payload representation even before we actually start a pipeline for better + # efficency. This can be done in a next iteration. + # + def event_id + raise NotImplementedError + end + + def <=>(other) + self.path <=> other.path + end + + def self.webhooks + @hooks ||= ::TriggerableHooks::AVAILABLE_TRIGGERS.values.map do |group| + group.to_s.sub('_events', '') + end + end + + ## + # TODO memoize methods below, refactor them, extract to a better place + # + def self.webhook_index(hook_name) + webhooks.index(hook_name.to_s) + 1 + end + + def self.hook_to_event(hook_name) + ::TriggerableHooks::AVAILABLE_TRIGGERS[hook_name.to_sym].to_s.gsub('_events', '') + end + + def self.system_hook_to_index(hook_name) + webhook_index(hook_to_event(hook_name)) + end + end + end + end +end diff --git a/lib/gitlab/ci/workflows/workflow.rb b/lib/gitlab/ci/workflows/workflow.rb new file mode 100644 index 0000000000000000000000000000000000000000..522e63cf359fa4491fdacfc6750990ad92cc71fe --- /dev/null +++ b/lib/gitlab/ci/workflows/workflow.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Workflows + class Workflow + include Enumerable + + attr_reader :events, :name + + def initialize(events) + @events = events + + names = events.map(&:workflow).uniq + + raise ArgumentError if names.size > 1 + + @name = names.first + end + + def valid? + events.all?(&:valid?) + end + + def each(&block) + events.each(&block) + end + + def uuid + sha256 = events.sort.uniq.reduce(0) do |sha, event| + ::OpenSSL::Digest::SHA256.hexdigest(event.path + sha.to_s) + end + + sha256[0..31] + end + + def event_types + events.map(&:type_id).uniq.sort + end + + def event_groups + events.map(&:group_id).uniq.sort + end + + def event_ids + events.map(&:event_id).uniq.sort + end + + def self.fabricate(events) + workflows = events.flatten.each_with_object({}) do |event, hash| + (hash[event.workflow] ||= []).push(event) + end + + workflows.keys.map do |key| + self.new(workflows.dig(key)) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 15ebd50605531b3e5881f15fc40196ba8a8ef564..8d58ad010563b93f04768681051af2a3a253c4d5 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -49,6 +49,11 @@ def run_logical_validations! end def validate_job!(name, job) + ## TODO add logical validation for CI workflows + # + # We initially don't want to support on: events for builds / pipelines webhooks to avoid possible circular + # workflows. + # validate_job_stage!(name, job) validate_job_dependencies!(name, job) validate_job_needs!(name, job) diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 576fb509d476cf3e3b57048bc1d46fc14a946b83..1c9bb0acf143f38fb58712e77bf015515bc153cd 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -99,7 +99,8 @@ def build_attributes(name) start_in: job[:start_in], trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, - release: release(job) + release: release(job), + on: job[:on] }.compact }.compact end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 0085b578102ce47e35f983b4f897674a9cc27f5b..0b993fd2ce986933e859265d6a2eb1120bc0bd93 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -120,6 +120,7 @@ ci_triggers: :gitlab_ci ci_unit_test_failures: :gitlab_ci ci_unit_tests: :gitlab_ci ci_variables: :gitlab_ci +ci_workflow_events: :gitlab_ci cluster_agents: :gitlab_main cluster_agent_tokens: :gitlab_main cluster_enabled_grants: :gitlab_main diff --git a/spec/lib/gitlab/ci/workflows/event_spec.rb b/spec/lib/gitlab/ci/workflows/event_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..af324b87efb8a1b9619f6aa84112ae055c05a901 --- /dev/null +++ b/spec/lib/gitlab/ci/workflows/event_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'app/models/concerns/triggerable_hooks' + +RSpec.describe Gitlab::Ci::Workflows::Event do + describe '.webhooks' do + it 'returns available system hooks' do + expect(described_class.webhooks).to include('issues') + end + end + + describe 'valid?' do + context 'when event is not webhooks event' do + it 'is not valid' do + event = described_class.new('workflow/cloud/event') + + expect(event).not_to be_valid + end + end + + context 'when webhook subgroup is not known' do + it 'is not valid' do + event = described_class.new('workflow/webhooks/unknown/event') + + expect(event).not_to be_valid + end + end + + context 'when event matches defined schema' do + it 'is not valid' do + event = described_class.new('workflow/webhooks/issues/created') + + expect(event).to be_valid + end + end + + context 'when event is using simplified schema' do + it 'is not valid' do + event = described_class.new('workflow/webhooks/issues') + + expect(event).to be_valid + end + end + end + + context 'when a recognized event path is used' do + subject { described_class.new('workflow/webhooks/issues') } + + describe '#type_id' do + it 'returns 1' do + expect(subject.type_id).to eq 1 + end + end + + describe '#group_id' do + it 'returns an id of the issues webhooks group' do + expect(subject.group_id).to eq 4 + end + end + end + + describe 'last known system hook identifier' do + it "generates an expected id for the latest known system hook" do + latest = TriggerableHooks::AVAILABLE_TRIGGERS.values.last + event = described_class.new('workflow/webhooks/subgroup') + + expect(latest).to eq :subgroup_events + expect(event.group_id).to eq 16 + end + end + + describe '#workflow' do + it 'properly exposes a name' do + event = described_class.new('my-workflow/webhooks/issues') + + expect(event.workflow).to eq 'my-workflow' + end + end +end diff --git a/spec/lib/gitlab/ci/workflows/workflow_spec.rb b/spec/lib/gitlab/ci/workflows/workflow_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1534768a82d08151134d7c7b82e0f4b5b296c86e --- /dev/null +++ b/spec/lib/gitlab/ci/workflows/workflow_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'app/models/concerns/triggerable_hooks' + +RSpec.describe Gitlab::Ci::Workflows::Workflow do + let(:events) do + [Gitlab::Ci::Workflows::Event.new('workflow/webhooks/merge_requests/updated'), + Gitlab::Ci::Workflows::Event.new('workflow/webhooks/issues/created')] + end + + subject { described_class.new(events) } + + describe '#valid?' do + it 'is valid' do + expect(subject).to be_valid + end + end + + describe '#uuid' do + it 'properly calculates workflow unique identifier' do + expect(subject.uuid).to eq 'a2e2051124fc2e751272b314830add49' + end + + it 'does not change uuid when an order is changed' do + workflow = described_class.new(events.reverse) + + expect(subject.uuid).to eq workflow.uuid + end + + it 'does not change uuid when an event is duplicated' do + workflow = described_class.new(events + [events.last]) + + expect(subject.uuid).to eq workflow.uuid + end + end + + describe '#event_types' do + it 'returns ids of event types' do + expect(subject.event_types).to eq [1] + end + end + + describe '#event_groups' do + it 'returns sorted ids of event groups' do + expect(subject.event_groups).to eq [4, 8] + end + + it 'returns the same set of ids when an event is duplicated' do + workflow = described_class.new(events + [events.last]) + + expect(subject.event_groups).to eq workflow.event_groups + end + end + + describe '.fabricate' do + let(:event) do + Gitlab::Ci::Workflows::Event.new('another/webhooks/issues/created') + end + + it 'creates multiple workflow objects correctly' do + workflows = described_class.fabricate(events + [event]) + + expect(workflows).to all(be_valid) + expect(workflows.count).to eq 2 + expect(workflows.first.uuid).to eq 'a2e2051124fc2e751272b314830add49' + expect(workflows.second.uuid).to eq 'fc345747a1c44dadc3937b3c1265f134' + end + end +end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 15567b3667330a3e6ea598806454de5c76fdfd20..31cce9755c467ef75b6c2cbe20d715eaa63595f9 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -326,6 +326,27 @@ module Ci end end + describe 'a build with events' do + subject { described_class.new(config).execute } + + let(:config) do + YAML.dump( + annotate: { + on: 'workflow/webhooks/issues', + script: 'run annotate issue' + } + ) + end + + it 'has events attribute' do + expect(subject.errors).to be_empty + + options = subject.build_attributes('annotate').dig(:options) + + expect(options).to match(a_hash_including(on: ['workflow/webhooks/issues'])) + end + end + describe 'bridge job' do let(:config) do YAML.dump(rspec: { diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index aac059f2104437e3fce23bae6362c1b8cea18a35..85848f3112e03058d8978581c777d518ad8b7602 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -876,6 +876,26 @@ def previous_commit_sha_from_ref(ref) end end + context 'when pipeline workflow is used' do + before do + config = YAML.dump( + annotate: { + on: 'workflow-1/webhooks/issues/created', + script: 'run my-scripts' + } + ) + + stub_ci_pipeline_yaml_file(config) + end + + it 'creates a new workflow' do + result = execute_service.payload + + expect(Ci::WorkflowEvent.all.count).to eq 1 + expect(Ci::WorkflowEvent.first.user).to eq result.user + end + end + context 'with resource group' do context 'when resource group is defined' do before do diff --git a/spec/services/ci/dispatch_workflow_service_spec.rb b/spec/services/ci/dispatch_workflow_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..79ad89f469f350ffe980384b475fa8e1a747945d --- /dev/null +++ b/spec/services/ci/dispatch_workflow_service_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::DispatchWorkflowService, '#execute' do + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository) } + + let(:event_group_id) do + ::Gitlab::Ci::Workflows::Event.webhook_index('issues') + end + + let(:data) { spy('payload') } # rubocop:disable RSpec/VerifiedDoubles + + subject { described_class.new(project) } + + before do + project.add_developer(user) + + # TODO FactoryBot + Ci::WorkflowEvent.create!( + name: 'my-workflow', + sha: '1234asdf', + uuid: '12345678123456781234567812345678', + project: project, + user: user, + event_scopes: [project.id], + event_types: [1], + event_groups: [event_group_id] + ) + + Ci::WorkflowEvent.create!( + name: 'workflow-2', + sha: '1234asdf', + uuid: '12345678123456781234567812345678', + project: project, + user: user, + event_scopes: [project.id], + event_types: [1], + event_groups: [event_group_id] + ) + + config = <<~CONFIG + stages: + - first + - second + - third + + first: + stage: first + script: irrelevant + + annotate: + on: my-workflow/webhooks/issues/created + stage: second + script: ./run something + + summary: + stage: third + needs: + - annotate + script: append-to-summary + + annotate-2: + on: workflow-2/webhooks/issues/updated + stage: third + script: ./run something + + fourth: + stage: third + script: irrelevant + CONFIG + + stub_ci_pipeline_yaml_file(config) + end + + it 'creates new pipelines', :sidekiq_inline do + subject.execute(data, 'issue_hooks') + + expect(Ci::Pipeline.all.count).to eq 2 + end + + it 'properly selects builds to run', :sidekiq_inline do + subject.execute(data, 'issue_hooks') + + first = Ci::Pipeline.first + second = Ci::Pipeline.second + + expect(first.yaml_errors).to be_blank + expect(second.yaml_errors).to be_blank + + expect(first.statuses.count).to eq 2 + expect(second.statuses.count).to eq 1 + + expect(first.statuses.map(&:name)). to match_array(%w[annotate summary]) + expect(second.statuses.map(&:name)). to match_array(%w[annotate-2]) + end +end diff --git a/spec/services/issues/create_service_spec.rb b/spec/services/issues/create_service_spec.rb index 5c1544d8ebcf361f77372c6127267fb8678f1d03..45f1539a9b310eb7ea6a8c3a76608a45c7dc97cc 100644 --- a/spec/services/issues/create_service_spec.rb +++ b/spec/services/issues/create_service_spec.rb @@ -69,6 +69,14 @@ expect(issue.issue_customer_relations_contacts).to be_empty end + it 'invokes a CI workflow dispatcher' do + allow(Ci::DispatchWorkflowService) + .to receive_message_chain(:new, :execute) + .with(a_hash_including(event_type: 'issue'), :issue_hooks) + + expect(issue).to be_persisted + end + context 'when a build_service is provided' do let(:issue) { described_class.new(project: project, current_user: user, params: opts, spam_params: spam_params, build_service: build_service).execute }