diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 5031a6684d379e42edb8617e00f51680510fa98c..81b7531ca39ec7f0fc71fdeb23b2ad155564e21d 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -103,6 +103,7 @@ - [elastic_indexer, 1] - [elastic_commit_indexer, 1] - [export_csv, 1] + - [incident_management, 2] # Deprecated queues: Remove after 10.7 - geo_base_scheduler diff --git a/ee/app/models/incident_management/project_incident_management_setting.rb b/ee/app/models/incident_management/project_incident_management_setting.rb index 1e9f70df9a72ff51eb4188a244883453d4b22a5f..bf57c5b883f24c86447ff4958f5e07544b7e6d86 100644 --- a/ee/app/models/incident_management/project_incident_management_setting.rb +++ b/ee/app/models/incident_management/project_incident_management_setting.rb @@ -2,6 +2,8 @@ module IncidentManagement class ProjectIncidentManagementSetting < ApplicationRecord + include Gitlab::Utils::StrongMemoize + belongs_to :project validate :issue_template_exists, if: :create_issue? @@ -10,14 +12,23 @@ def available_issue_templates Gitlab::Template::IssueTemplate.all(project) end + def issue_template_content + strong_memoize(:issue_template_content) do + issue_template&.content if issue_template_key.present? + end + end + private def issue_template_exists return unless issue_template_key.present? + errors.add(:issue_template_key, 'not found') unless issue_template + end + + def issue_template Gitlab::Template::IssueTemplate.find(issue_template_key, project) rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - errors.add(:issue_template_key, 'not found') end end end diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..f30629d3cdfb6200b5a8a12a861d3053c9ab0b08 --- /dev/null +++ b/ee/app/services/incident_management/create_issue_service.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module IncidentManagement + class CreateIssueService < BaseService + include Gitlab::Utils::StrongMemoize + + def execute + return error_with('setting disabled') unless incident_management_setting.create_issue? + return error_with('invalid alert') unless alert.valid? + + success(issue: create_issue) + end + + private + + def create_issue + Issues::CreateService.new( + project, + issue_author, + title: issue_title, + description: issue_description + ).execute + end + + def issue_author + strong_memoize(:issue_author) do + # This is a temporary solution before we've implemented User.alert_bot + # https://gitlab.com/gitlab-org/gitlab-ee/issues/10159 + User.ghost + end + end + + def issue_title + alert.title + end + + def issue_description + return alert_summary unless issue_template_content + + horizontal_line = "\n---\n\n" + + alert_summary + horizontal_line + issue_template_content + end + + def alert_summary + <<~MARKDOWN + ## Summary + + #{annotation_list} + MARKDOWN + end + + def annotation_list + strong_memoize(:annotation_list) do + alert.annotations + .map { |annotation| "* #{annotation.label}: #{annotation.value}" } + .join("\n") + end + end + + def alert + strong_memoize(:alert) do + Gitlab::Alerting::Alert.new(project: project, payload: params) + end + end + + def issue_template_content + incident_management_setting.issue_template_content + end + + def incident_management_setting + strong_memoize(:incident_management_setting) do + project.incident_management_setting || + project.build_incident_management_setting + end + end + + def error_with(message) + log_error(%{Cannot create incident issue for "#{project.full_name}": #{message}}) + + error(message) + end + end +end diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb index 81b979e6ef2da2416475f13978c959fd42d1a226..527d73f1e81047b77329d29901abd18e26f520fc 100644 --- a/ee/app/services/projects/prometheus/alerts/notify_service.rb +++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb @@ -4,12 +4,15 @@ module Projects module Prometheus module Alerts class NotifyService < BaseService + include Gitlab::Utils::StrongMemoize + def execute(token) return false unless valid_version? return false unless valid_alert_manager_token?(token) send_alert_email if send_email? - persist_events(project, params) + process_incident_issues if create_issue? + persist_events true end @@ -24,13 +27,28 @@ def incident_management_feature_enabled? Feature.enabled?(:incident_management) end + def incident_management_available? + has_incident_management_license? && incident_management_feature_enabled? + end + + def incident_management_setting + strong_memoize(:incident_management_setting) do + project.incident_management_setting || + project.build_incident_management_setting + end + end + def send_email? - return firings.any? unless incident_management_feature_enabled? && - has_incident_management_license? + return firings.any? unless incident_management_available? - setting = project.incident_management_setting || project.build_incident_management_setting + incident_management_setting.send_email && firings.any? + end + + def create_issue? + return unless firings.any? + return unless incident_management_available? - setting.send_email && firings.any? + incident_management_setting.create_issue? end def firings @@ -112,7 +130,14 @@ def send_alert_email .prometheus_alerts_fired(project, firings) end - def persist_events(project, params) + def process_incident_issues + firings.each do |alert| + IncidentManagement::ProcessAlertWorker + .perform_async(project.id, alert) + end + end + + def persist_events CreateEventsService.new(project, nil, params).execute end end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 5334359baa6f56eb5b040462a9dc1c6a29552df9..37514017ee96bb93da7b131852fcdecee3908428 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -45,6 +45,8 @@ - pipeline_default:store_security_reports - pipeline_default:ci_create_cross_project_pipeline +- incident_management:incident_management_process_alert + - admin_emails - create_github_webhook - elastic_batch_project_indexer diff --git a/ee/app/workers/incident_management/process_alert_worker.rb b/ee/app/workers/incident_management/process_alert_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..fc6e96b4a7fa6e5d1906fc49cab3bd1f576171e3 --- /dev/null +++ b/ee/app/workers/incident_management/process_alert_worker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module IncidentManagement + class ProcessAlertWorker + include ApplicationWorker + + queue_namespace :incident_management + + def perform(project_id, alert) + project = find_project(project_id) + return unless project + + create_issue(project, alert) + end + + private + + def find_project(project_id) + Project.find_by_id(project_id) + end + + def create_issue(project, alert) + IncidentManagement::CreateIssueService + .new(project, nil, alert) + .execute + end + end +end diff --git a/ee/lib/gitlab/alerting/alert.rb b/ee/lib/gitlab/alerting/alert.rb index 963fae9217a357c3883299d56defe83772f427b8..ae32b0ba714b08ec3a0f4fe2a768a958c50dd868 100644 --- a/ee/lib/gitlab/alerting/alert.rb +++ b/ee/lib/gitlab/alerting/alert.rb @@ -31,6 +31,12 @@ def environment gitlab_alert&.environment end + def annotations + strong_memoize(:annotations) do + parse_annotations_from_payload || [] + end + end + def valid? project && title end @@ -60,6 +66,12 @@ def parse_title_from_payload def parse_description_from_payload payload&.dig('annotations', 'description') end + + def parse_annotations_from_payload + payload&.dig('annotations')&.map do |label, value| + Alerting::AlertAnnotation.new(label: label, value: value) + end + end end end end diff --git a/ee/lib/gitlab/alerting/alert_annotation.rb b/ee/lib/gitlab/alerting/alert_annotation.rb new file mode 100644 index 0000000000000000000000000000000000000000..a4b3a97b08c0df71c22c01581e9788eafbc6803d --- /dev/null +++ b/ee/lib/gitlab/alerting/alert_annotation.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Alerting + class AlertAnnotation + include ActiveModel::Model + + attr_accessor :label, :value + end + end +end diff --git a/ee/spec/lib/gitlab/alerting/alert_spec.rb b/ee/spec/lib/gitlab/alerting/alert_spec.rb index 99073e85e2324500b4cbde2fce2bb1ab9c798a8e..05ee801dec8bd7eb94e402d098cedc0ffdd8a6ec 100644 --- a/ee/spec/lib/gitlab/alerting/alert_spec.rb +++ b/ee/spec/lib/gitlab/alerting/alert_spec.rb @@ -52,6 +52,27 @@ end end + context 'with annotations' do + before do + payload['annotations'] = { + 'label' => 'value', + 'another' => 'value2' + } + end + + it 'parses annotations' do + expect(alert.annotations.size).to eq(2) + expect(alert.annotations.map(&:label)).to eq(%w(label another)) + expect(alert.annotations.map(&:value)).to eq(%w(value value2)) + end + end + + context 'without annotations' do + it 'has no annotations' do + expect(alert.annotations).to be_empty + end + end + context 'with empty payload' do it 'cannot load gitlab_alert' do expect(alert.gitlab_alert).to be_nil diff --git a/ee/spec/models/incident_management/project_incident_management_setting_spec.rb b/ee/spec/models/incident_management/project_incident_management_setting_spec.rb index 701bd294ad648a87c2b3c613498b7e7c6f5d2752..b2b0a645d6e1a24aa754f1b44cf20cf0c76f3c85 100644 --- a/ee/spec/models/incident_management/project_incident_management_setting_spec.rb +++ b/ee/spec/models/incident_management/project_incident_management_setting_spec.rb @@ -72,4 +72,40 @@ end end end + + describe '#issue_template_content' do + subject { build(:project_incident_management_setting, project: project) } + + shared_examples 'no content' do + it 'returns no content' do + expect(subject.issue_template_content).to be_nil + end + end + + context 'with valid issue_template_key' do + before do + subject.issue_template_key = 'bug' + end + + it 'returns issue content' do + expect(subject.issue_template_content).to eq('something valid') + end + end + + context 'with unknown issue_template_key' do + before do + subject.issue_template_key = 'unknown' + end + + it_behaves_like 'no content' + end + + context 'without issue_template_key' do + before do + subject.issue_template_key = nil + end + + it_behaves_like 'no content' + end + end end diff --git a/ee/spec/services/incident_management/create_issue_service_spec.rb b/ee/spec/services/incident_management/create_issue_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f0841dae9ea898d9015665358f35726ad4774474 --- /dev/null +++ b/ee/spec/services/incident_management/create_issue_service_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe IncidentManagement::CreateIssueService do + set(:project) { create(:project, :repository, create_templates: :issue) } + let(:service) { described_class.new(project, nil, alert_payload) } + let(:alert_title) { 'TITLE' } + let(:alert_payload) do + build_alert_payload(annotations: { title: alert_title }) + end + + let!(:setting) do + create(:project_incident_management_setting, project: project) + end + + subject { service.execute } + + context 'when create_issue enabled' do + let(:user) { create(:user) } + + before do + setting.update!(create_issue: true) + end + + context 'without issue_template_content' do + it 'creates an issue with alert summary only' do + expect(subject).to include(status: :success) + + issue = subject[:issue] + expect(issue.author).to eq(User.ghost) + expect(issue.title).to eq(alert_title) + expect(issue.description).to include('Summary') + expect(issue.description).to include(alert_title) + expect(issue.description).not_to include("---\n\n") + end + end + + context 'with issue_template_content' do + before do + setting.update!(issue_template_key: 'bug') + end + + it 'creates an issue appending issue template' do + expect(subject).to include(status: :success) + + issue = subject[:issue] + expect(issue.description).to include("---\n\n") + expect(issue.description).to include(setting.issue_template_content) + end + end + + context 'with an invalid alert payload' do + let(:alert_payload) { build_alert_payload(annotations: {}) } + + it 'does not create an issue' do + expect(service) + .to receive(:log_error) + .with(error_message('invalid alert')) + + expect(subject).to eq(status: :error, message: 'invalid alert') + end + end + end + + context 'when create_issue disabled' do + before do + setting.update!(create_issue: false) + end + + it 'returns an error' do + expect(service) + .to receive(:log_error) + .with(error_message('setting disabled')) + + expect(subject).to eq(status: :error, message: 'setting disabled') + end + end + + private + + def build_alert_payload(annotations: {}) + { 'annotations' => annotations.stringify_keys } + end + + def error_message(message) + %{Cannot create incident issue for "#{project.full_name}": #{message}} + end +end diff --git a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb index 29b48135c57f107420e9bfe5df50a8a4d883d0f5..898b2b4472b5fbf2c0732e6857fd640e5ddb28d9 100644 --- a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb +++ b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb @@ -29,6 +29,28 @@ end end + shared_examples 'processes incident issues' do |amount| + let(:create_incident_service) { spy } + + it 'processes issues' do + expect(IncidentManagement::ProcessAlertWorker) + .to receive(:perform_async) + .with(project.id, anything) + .exactly(amount).times + + expect(subject).to eq(true) + end + end + + shared_examples 'does not process incident issues' do + it 'does not process issues' do + expect(IncidentManagement::ProcessAlertWorker) + .not_to receive(:perform_async) + + expect(subject).to eq(true) + end + end + shared_examples 'persists events' do let(:create_events_service) { spy } @@ -250,6 +272,59 @@ end end end + + context 'process incident issues' do + let!(:setting) do + create( + :project_incident_management_setting, + project: project, + create_issue: true + ) + end + + before do + create(:prometheus_service, project: project) + create(:project_alerting_setting, project: project, token: token) + end + + context 'with license' do + before do + stub_licensed_features(incident_management: true) + end + + context 'with create_issue setting enabled' do + before do + setting.update!(create_issue: true) + end + + it_behaves_like 'processes incident issues', 1 + + context 'without firing alerts' do + let(:payload_raw) do + payload_for(firing: [], resolved: [alert_resolved]) + end + + it_behaves_like 'does not process incident issues' + end + end + + context 'with create_issue setting disabled' do + before do + setting.update!(create_issue: false) + end + + it_behaves_like 'does not process incident issues' + end + end + + context 'without license' do + before do + stub_licensed_features(incident_management: false) + end + + it_behaves_like 'does not process incident issues' + end + end end context 'with invalid payload' do diff --git a/ee/spec/workers/incident_management/process_alert_worker_spec.rb b/ee/spec/workers/incident_management/process_alert_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..b23a9c6b2dc53987b36e12d19c228b88d0d08056 --- /dev/null +++ b/ee/spec/workers/incident_management/process_alert_worker_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe IncidentManagement::ProcessAlertWorker do + set(:project) { create(:project) } + + describe '#perform' do + let(:alert) { :alert } + let(:create_issue_service) { spy(:create_issue_service) } + + subject { described_class.new.perform(project.id, alert) } + + it 'calls create issue service' do + expect(Project).to receive(:find_by_id).and_call_original + + expect(IncidentManagement::CreateIssueService) + .to receive(:new).with(project, nil, :alert) + .and_return(create_issue_service) + + expect(create_issue_service).to receive(:execute) + + subject + end + + context 'with invalid project' do + let(:invalid_project_id) { 0 } + + subject { described_class.new.perform(invalid_project_id, alert) } + + it 'does not create issues' do + expect(Project).to receive(:find_by_id).and_call_original + expect(IncidentManagement::CreateIssueService).not_to receive(:new) + + subject + end + end + end +end