From 0596d4157dd2821b58eb6009483900b2f6ec5fa6 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Thu, 28 Feb 2019 15:55:28 +0100
Subject: [PATCH 01/16] Expose issue template's content

---
 .../project_incident_management_setting.rb    | 15 ++++++++
 ...roject_incident_management_setting_spec.rb | 36 +++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/ee/app/models/incident_management/project_incident_management_setting.rb b/ee/app/models/incident_management/project_incident_management_setting.rb
index 1e9f70df9a72ff..d896f4056a95be 100644
--- a/ee/app/models/incident_management/project_incident_management_setting.rb
+++ b/ee/app/models/incident_management/project_incident_management_setting.rb
@@ -2,6 +2,8 @@
 
 module IncidentManagement
   class ProjectIncidentManagementSetting < ApplicationRecord
+    include Gitlab::Utils::StrongMemoize
+
     belongs_to :project
 
     validate :issue_template_exists, if: :create_issue?
@@ -10,6 +12,12 @@ def available_issue_templates
       Gitlab::Template::IssueTemplate.all(project)
     end
 
+    def issue_template_content
+      strong_memoize(:issue_template_content) do
+        load_issue_template_content
+      end
+    end
+
     private
 
     def issue_template_exists
@@ -19,5 +27,12 @@ def issue_template_exists
     rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
       errors.add(:issue_template_key, 'not found')
     end
+
+    def load_issue_template_content
+      return unless issue_template_key.present?
+
+      Gitlab::Template::IssueTemplate.find(issue_template_key, project).content
+    rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
+    end
   end
 end
diff --git a/ee/spec/models/incident_management/project_incident_management_setting_spec.rb b/ee/spec/models/incident_management/project_incident_management_setting_spec.rb
index 701bd294ad648a..b2b0a645d6e1a2 100644
--- a/ee/spec/models/incident_management/project_incident_management_setting_spec.rb
+++ b/ee/spec/models/incident_management/project_incident_management_setting_spec.rb
@@ -72,4 +72,40 @@
       end
     end
   end
+
+  describe '#issue_template_content' do
+    subject { build(:project_incident_management_setting, project: project) }
+
+    shared_examples 'no content' do
+      it 'returns no content' do
+        expect(subject.issue_template_content).to be_nil
+      end
+    end
+
+    context 'with valid issue_template_key' do
+      before do
+        subject.issue_template_key = 'bug'
+      end
+
+      it 'returns issue content' do
+        expect(subject.issue_template_content).to eq('something valid')
+      end
+    end
+
+    context 'with unknown issue_template_key' do
+      before do
+        subject.issue_template_key = 'unknown'
+      end
+
+      it_behaves_like 'no content'
+    end
+
+    context 'without issue_template_key' do
+      before do
+        subject.issue_template_key = nil
+      end
+
+      it_behaves_like 'no content'
+    end
+  end
 end
-- 
GitLab


From 8f7a6b843bc7a03c5a915e2e1af9e5710ae39aa5 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Thu, 28 Feb 2019 16:14:05 +0100
Subject: [PATCH 02/16] Unify finding issue template

---
 .../project_incident_management_setting.rb         | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ee/app/models/incident_management/project_incident_management_setting.rb b/ee/app/models/incident_management/project_incident_management_setting.rb
index d896f4056a95be..8a825fe9965216 100644
--- a/ee/app/models/incident_management/project_incident_management_setting.rb
+++ b/ee/app/models/incident_management/project_incident_management_setting.rb
@@ -13,9 +13,7 @@ def available_issue_templates
     end
 
     def issue_template_content
-      strong_memoize(:issue_template_content) do
-        load_issue_template_content
-      end
+      strong_memoize(:issue_template_content) { load_issue_template_content }
     end
 
     private
@@ -23,15 +21,17 @@ def issue_template_content
     def issue_template_exists
       return unless issue_template_key.present?
 
-      Gitlab::Template::IssueTemplate.find(issue_template_key, project)
-    rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
-      errors.add(:issue_template_key, 'not found')
+      errors.add(:issue_template_key, 'not found') unless find_issue_template
     end
 
     def load_issue_template_content
       return unless issue_template_key.present?
 
-      Gitlab::Template::IssueTemplate.find(issue_template_key, project).content
+      find_issue_template&.content
+    end
+
+    def find_issue_template
+      Gitlab::Template::IssueTemplate.find(issue_template_key, project)
     rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
     end
   end
-- 
GitLab


From 57a4af39aeb291a962e23d71adbb3874f12c8954 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Mon, 4 Mar 2019 11:57:10 +0100
Subject: [PATCH 03/16] Expose Alert's annotations

---
 ee/lib/gitlab/alerting/alert.rb            | 12 ++++++++++++
 ee/lib/gitlab/alerting/alert_annotation.rb | 11 +++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 ee/lib/gitlab/alerting/alert_annotation.rb

diff --git a/ee/lib/gitlab/alerting/alert.rb b/ee/lib/gitlab/alerting/alert.rb
index 963fae9217a357..ae32b0ba714b08 100644
--- a/ee/lib/gitlab/alerting/alert.rb
+++ b/ee/lib/gitlab/alerting/alert.rb
@@ -31,6 +31,12 @@ def environment
         gitlab_alert&.environment
       end
 
+      def annotations
+        strong_memoize(:annotations) do
+          parse_annotations_from_payload || []
+        end
+      end
+
       def valid?
         project && title
       end
@@ -60,6 +66,12 @@ def parse_title_from_payload
       def parse_description_from_payload
         payload&.dig('annotations', 'description')
       end
+
+      def parse_annotations_from_payload
+        payload&.dig('annotations')&.map do |label, value|
+          Alerting::AlertAnnotation.new(label: label, value: value)
+        end
+      end
     end
   end
 end
diff --git a/ee/lib/gitlab/alerting/alert_annotation.rb b/ee/lib/gitlab/alerting/alert_annotation.rb
new file mode 100644
index 00000000000000..a4b3a97b08c0df
--- /dev/null
+++ b/ee/lib/gitlab/alerting/alert_annotation.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Alerting
+    class AlertAnnotation
+      include ActiveModel::Model
+
+      attr_accessor :label, :value
+    end
+  end
+end
-- 
GitLab


From c56b636ac4912a13524313b0dff8f0e576933b5e Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Mon, 4 Mar 2019 12:06:27 +0100
Subject: [PATCH 04/16] Create an incident issue from an alert

---
 .../create_issue_service.rb                   | 80 +++++++++++++++++++
 .../prometheus/alerts/notify_service.rb       | 16 +++-
 ee/spec/lib/gitlab/alerting/alert_spec.rb     | 21 +++++
 .../create_issue_service_spec.rb              | 77 ++++++++++++++++++
 .../prometheus/alerts/notify_service_spec.rb  | 15 ++++
 5 files changed, 207 insertions(+), 2 deletions(-)
 create mode 100644 ee/app/services/incident_management/create_issue_service.rb
 create mode 100644 ee/spec/services/incident_management/create_issue_service_spec.rb

diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb
new file mode 100644
index 00000000000000..7e0f0e658f55fc
--- /dev/null
+++ b/ee/app/services/incident_management/create_issue_service.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module IncidentManagement
+  class CreateIssueService < BaseService
+    include Gitlab::Utils::StrongMemoize
+
+    def execute
+      return error('disabled') unless incident_management_setting.create_issue?
+      return error('alert invalid') unless alert.valid?
+
+      issue = create_issue(alert)
+
+      success(issue: issue)
+    end
+
+    private
+
+    def create_issue(alert)
+      Issues::CreateService.new(
+        project,
+        issue_author,
+        title: issue_title,
+        description: issue_description
+      ).execute
+    end
+
+    def issue_author
+      strong_memoize(:issue_author) do
+        # This is a temporary solution before we've implemented User.alert_bot
+        # https://gitlab.com/gitlab-org/gitlab-ee/issues/10159
+        User.ghost
+      end
+    end
+
+    def issue_title
+      alert.title
+    end
+
+    def issue_description
+      return alert_summary unless issue_template_content
+
+      horizontal_line = "\n---\n\n"
+
+      alert_summary + horizontal_line + issue_template_content
+    end
+
+    def alert_summary
+      annotation_list = annotations_as_list(alert.annotations)
+
+      <<~MARKDOWN
+        ## Summary
+
+        #{annotation_list}
+      MARKDOWN
+    end
+
+    def annotations_as_list(annotations)
+      annotations
+        .map { |annotation| "* #{annotation.label}: #{annotation.value}" }
+        .join("\n")
+    end
+
+    def alert
+      strong_memoize(:alert) do
+        Gitlab::Alerting::Alert.new(project: project, payload: params)
+      end
+    end
+
+    def issue_template_content
+      incident_management_setting.issue_template_content
+    end
+
+    def incident_management_setting
+      strong_memoize(:incident_management_setting) do
+        project.incident_management_setting ||
+          project.build_incident_management_setting
+      end
+    end
+  end
+end
diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb
index 81b979e6ef2da2..0e14b38b879a4f 100644
--- a/ee/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb
@@ -9,7 +9,8 @@ def execute(token)
           return false unless valid_alert_manager_token?(token)
 
           send_alert_email if send_email?
-          persist_events(project, params)
+          open_incident_issue
+          persist_events
 
           true
         end
@@ -112,7 +113,18 @@ def send_alert_email
             .prometheus_alerts_fired(project, firings)
         end
 
-        def persist_events(project, params)
+        def open_incident_issue
+          return unless firings.any?
+
+          # TODO async?
+          firings.each do |firing_alert|
+            IncidentManagement::CreateIssueService
+              .new(project, nil, firing_alert)
+              .execute
+          end
+        end
+
+        def persist_events
           CreateEventsService.new(project, nil, params).execute
         end
       end
diff --git a/ee/spec/lib/gitlab/alerting/alert_spec.rb b/ee/spec/lib/gitlab/alerting/alert_spec.rb
index 99073e85e23245..05ee801dec8bd7 100644
--- a/ee/spec/lib/gitlab/alerting/alert_spec.rb
+++ b/ee/spec/lib/gitlab/alerting/alert_spec.rb
@@ -52,6 +52,27 @@
     end
   end
 
+  context 'with annotations' do
+    before do
+      payload['annotations'] = {
+        'label' => 'value',
+        'another' => 'value2'
+      }
+    end
+
+    it 'parses annotations' do
+      expect(alert.annotations.size).to eq(2)
+      expect(alert.annotations.map(&:label)).to eq(%w(label another))
+      expect(alert.annotations.map(&:value)).to eq(%w(value value2))
+    end
+  end
+
+  context 'without annotations' do
+    it 'has no annotations' do
+      expect(alert.annotations).to be_empty
+    end
+  end
+
   context 'with empty payload' do
     it 'cannot load gitlab_alert' do
       expect(alert.gitlab_alert).to be_nil
diff --git a/ee/spec/services/incident_management/create_issue_service_spec.rb b/ee/spec/services/incident_management/create_issue_service_spec.rb
new file mode 100644
index 00000000000000..b1cc7521fc0706
--- /dev/null
+++ b/ee/spec/services/incident_management/create_issue_service_spec.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe IncidentManagement::CreateIssueService do
+  set(:project) { create(:project, :repository, create_templates: :issue) }
+  let(:service) { described_class.new(project, nil, alert_payload) }
+  let(:alert_title) { 'TITLE' }
+  let(:alert_payload) do
+    build_alert_payload(annotations: { title: alert_title })
+  end
+
+  let!(:setting) do
+    create(:project_incident_management_setting, project: project)
+  end
+
+  subject { service.execute }
+
+  context 'when create_issue enabled' do
+    let(:user) { create(:user) }
+
+    before do
+      setting.update!(create_issue: true)
+    end
+
+    context 'without issue_template_content' do
+      it 'creates an issue with alert summary only' do
+        expect(subject).to include(status: :success)
+
+        issue = subject[:issue]
+        expect(issue.author).to eq(User.ghost)
+        expect(issue.title).to eq(alert_title)
+        expect(issue.description).to include('Summary')
+        expect(issue.description).to include(alert_title)
+        expect(issue.description).not_to include("---\n\n")
+      end
+    end
+
+    context 'with issue_template_content' do
+      before do
+        setting.update!(issue_template_key: 'bug')
+      end
+
+      it 'creates an issue appending issue template' do
+        expect(subject).to include(status: :success)
+
+        issue = subject[:issue]
+        expect(issue.description).to include("---\n\n")
+        expect(issue.description).to include(setting.issue_template_content)
+      end
+    end
+
+    context 'with an invalid alert payload' do
+      let(:alert_payload) { build_alert_payload(annotations: {}) }
+
+      it 'does not create an issue' do
+        expect(subject).to eq(status: :error, message: 'alert invalid')
+      end
+    end
+  end
+
+  context 'when create_issue disabled' do
+    before do
+      setting.update!(create_issue: false)
+    end
+
+    it 'returns an error' do
+      expect(subject).to eq(status: :error, message: 'disabled')
+    end
+  end
+
+  private
+
+  def build_alert_payload(annotations: {})
+    { 'annotations' => annotations.stringify_keys }
+  end
+end
diff --git a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
index 29b48135c57f10..06bce64546f0c5 100644
--- a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
+++ b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
@@ -29,6 +29,21 @@
     end
   end
 
+  shared_examples 'creates incident issue' do
+    let(:create_incident_service) { spy }
+
+    it 'creates an incident issue' do
+      expect(IncidentManagement::CreateIssueService)
+        .to receive(:new)
+        .and_return(create_incident_service)
+
+      expect(create_incident_service)
+        .to receive(:execute)
+
+      expect(subject).to eq(true)
+    end
+  end
+
   shared_examples 'persists events' do
     let(:create_events_service) { spy }
 
-- 
GitLab


From a1a80b7bb853bd35d02711cde84931658db957fe Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 8 Mar 2019 13:56:35 +0100
Subject: [PATCH 05/16] Implement a worker processing alerts

Currently, the worker calls a service to create an issue
based on the incoming alert
---
 ee/app/workers/all_queues.yml                 |  2 +
 .../process_alerts_worker.rb                  | 32 +++++++++++
 .../process_alerts_worker_spec.rb             | 54 +++++++++++++++++++
 3 files changed, 88 insertions(+)
 create mode 100644 ee/app/workers/incident_management/process_alerts_worker.rb
 create mode 100644 ee/spec/workers/incident_management/process_alerts_worker_spec.rb

diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml
index 5334359baa6f56..af4dc0de084df1 100644
--- a/ee/app/workers/all_queues.yml
+++ b/ee/app/workers/all_queues.yml
@@ -45,6 +45,8 @@
 - pipeline_default:store_security_reports
 - pipeline_default:ci_create_cross_project_pipeline
 
+- incident_management:process_alerts
+
 - admin_emails
 - create_github_webhook
 - elastic_batch_project_indexer
diff --git a/ee/app/workers/incident_management/process_alerts_worker.rb b/ee/app/workers/incident_management/process_alerts_worker.rb
new file mode 100644
index 00000000000000..468f2d55b67699
--- /dev/null
+++ b/ee/app/workers/incident_management/process_alerts_worker.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module IncidentManagement
+  class ProcessAlertsWorker
+    include ApplicationWorker
+
+    queue_namespace :incident_management
+
+    def perform(project_id, alerts)
+      return unless alerts.any?
+
+      project = find_project(project_id)
+      return unless project
+
+      alerts.each do |alert|
+        create_issue(project, alert)
+      end
+    end
+
+    private
+
+    def find_project(project_id)
+      Project.find_by_id(project_id)
+    end
+
+    def create_issue(project, alert)
+      IncidentManagement::CreateIssueService
+        .new(project, nil, alert)
+        .execute
+    end
+  end
+end
diff --git a/ee/spec/workers/incident_management/process_alerts_worker_spec.rb b/ee/spec/workers/incident_management/process_alerts_worker_spec.rb
new file mode 100644
index 00000000000000..85a0b4fe6a39b4
--- /dev/null
+++ b/ee/spec/workers/incident_management/process_alerts_worker_spec.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe IncidentManagement::ProcessAlertsWorker do
+  set(:project) { create(:project) }
+
+  subject { described_class.new.perform(project.id, alerts) }
+
+  context 'with alerts' do
+    let(:alerts) { [:one, :two] }
+    let(:create_issue_service) { spy(:create_issue_service) }
+
+    it 'calls create issue service' do
+      expect(Project).to receive(:find_by_id).and_call_original
+
+      expect(IncidentManagement::CreateIssueService)
+        .to receive(:new).with(project, nil, :one)
+        .and_return(create_issue_service)
+
+      expect(IncidentManagement::CreateIssueService)
+        .to receive(:new).with(project, nil, :two)
+        .and_return(create_issue_service)
+
+      expect(create_issue_service).to receive(:execute).twice
+
+      subject
+    end
+
+    context 'with invalid project' do
+      let(:invalid_project_id) { 0 }
+
+      subject { described_class.new.perform(invalid_project_id, alerts) }
+
+      it 'does not create issues' do
+        expect(Project).to receive(:find_by_id).and_call_original
+        expect(IncidentManagement::CreateIssueService).not_to receive(:new)
+
+        subject
+      end
+    end
+  end
+
+  context 'without alerts' do
+    let(:alerts) { [] }
+
+    it 'does nothing' do
+      expect(Project).not_to receive(:find_by_id)
+      expect(IncidentManagement::CreateIssueService).not_to receive(:new)
+
+      subject
+    end
+  end
+end
-- 
GitLab


From 398cc557ee3e8e4ca40a8649a3132055fb6df765 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 8 Mar 2019 14:39:41 +0100
Subject: [PATCH 06/16] Use worker to process issues from notify service

---
 .../prometheus/alerts/notify_service.rb       | 31 +++++---
 ...erts_worker.rb => process_alert_worker.rb} | 10 +--
 .../prometheus/alerts/notify_service_spec.rb  | 74 +++++++++++++++++--
 ...r_spec.rb => process_alert_worker_spec.rb} | 31 ++------
 4 files changed, 100 insertions(+), 46 deletions(-)
 rename ee/app/workers/incident_management/{process_alerts_worker.rb => process_alert_worker.rb} (72%)
 rename ee/spec/workers/incident_management/{process_alerts_worker_spec.rb => process_alert_worker_spec.rb} (52%)

diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb
index 0e14b38b879a4f..801e1aa9a97a43 100644
--- a/ee/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb
@@ -4,12 +4,14 @@ module Projects
   module Prometheus
     module Alerts
       class NotifyService < BaseService
+        include Gitlab::Utils::StrongMemoize
+
         def execute(token)
           return false unless valid_version?
           return false unless valid_alert_manager_token?(token)
 
           send_alert_email if send_email?
-          open_incident_issue
+          process_incident_issues if create_issue?
           persist_events
 
           true
@@ -25,6 +27,14 @@ def incident_management_feature_enabled?
           Feature.enabled?(:incident_management)
         end
 
+        # TODO re-use in send_email? once that code is in `master`
+        def incident_management_setting
+          strong_memoize(:incident_management_setting) do
+            project.incident_management_setting ||
+              project.build_incident_management_setting
+          end
+        end
+
         def send_email?
           return firings.any? unless incident_management_feature_enabled? &&
               has_incident_management_license?
@@ -34,6 +44,13 @@ def send_email?
           setting.send_email && firings.any?
         end
 
+        def create_issue?
+          return unless firings.any?
+          return unless has_incident_management_license?
+
+          incident_management_setting.create_issue?
+        end
+
         def firings
           @firings ||= alerts_by_status('firing')
         end
@@ -113,14 +130,10 @@ def send_alert_email
             .prometheus_alerts_fired(project, firings)
         end
 
-        def open_incident_issue
-          return unless firings.any?
-
-          # TODO async?
-          firings.each do |firing_alert|
-            IncidentManagement::CreateIssueService
-              .new(project, nil, firing_alert)
-              .execute
+        def process_incident_issues
+          firings.each do |alert|
+            IncidentManagement::ProcessAlertWorker
+              .perform_async(project.id, alert)
           end
         end
 
diff --git a/ee/app/workers/incident_management/process_alerts_worker.rb b/ee/app/workers/incident_management/process_alert_worker.rb
similarity index 72%
rename from ee/app/workers/incident_management/process_alerts_worker.rb
rename to ee/app/workers/incident_management/process_alert_worker.rb
index 468f2d55b67699..fc6e96b4a7fa6e 100644
--- a/ee/app/workers/incident_management/process_alerts_worker.rb
+++ b/ee/app/workers/incident_management/process_alert_worker.rb
@@ -1,20 +1,16 @@
 # frozen_string_literal: true
 
 module IncidentManagement
-  class ProcessAlertsWorker
+  class ProcessAlertWorker
     include ApplicationWorker
 
     queue_namespace :incident_management
 
-    def perform(project_id, alerts)
-      return unless alerts.any?
-
+    def perform(project_id, alert)
       project = find_project(project_id)
       return unless project
 
-      alerts.each do |alert|
-        create_issue(project, alert)
-      end
+      create_issue(project, alert)
     end
 
     private
diff --git a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
index 06bce64546f0c5..898b2b4472b5fb 100644
--- a/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
+++ b/ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
@@ -29,16 +29,23 @@
     end
   end
 
-  shared_examples 'creates incident issue' do
+  shared_examples 'processes incident issues' do |amount|
     let(:create_incident_service) { spy }
 
-    it 'creates an incident issue' do
-      expect(IncidentManagement::CreateIssueService)
-        .to receive(:new)
-        .and_return(create_incident_service)
+    it 'processes issues' do
+      expect(IncidentManagement::ProcessAlertWorker)
+        .to receive(:perform_async)
+        .with(project.id, anything)
+        .exactly(amount).times
 
-      expect(create_incident_service)
-        .to receive(:execute)
+      expect(subject).to eq(true)
+    end
+  end
+
+  shared_examples 'does not process incident issues' do
+    it 'does not process issues' do
+      expect(IncidentManagement::ProcessAlertWorker)
+        .not_to receive(:perform_async)
 
       expect(subject).to eq(true)
     end
@@ -265,6 +272,59 @@
         end
       end
     end
+
+    context 'process incident issues' do
+      let!(:setting) do
+        create(
+          :project_incident_management_setting,
+          project: project,
+          create_issue: true
+        )
+      end
+
+      before do
+        create(:prometheus_service, project: project)
+        create(:project_alerting_setting, project: project, token: token)
+      end
+
+      context 'with license' do
+        before do
+          stub_licensed_features(incident_management: true)
+        end
+
+        context 'with create_issue setting enabled' do
+          before do
+            setting.update!(create_issue: true)
+          end
+
+          it_behaves_like 'processes incident issues', 1
+
+          context 'without firing alerts' do
+            let(:payload_raw) do
+              payload_for(firing: [], resolved: [alert_resolved])
+            end
+
+            it_behaves_like 'does not process incident issues'
+          end
+        end
+
+        context 'with create_issue setting disabled' do
+          before do
+            setting.update!(create_issue: false)
+          end
+
+          it_behaves_like 'does not process incident issues'
+        end
+      end
+
+      context 'without license' do
+        before do
+          stub_licensed_features(incident_management: false)
+        end
+
+        it_behaves_like 'does not process incident issues'
+      end
+    end
   end
 
   context 'with invalid payload' do
diff --git a/ee/spec/workers/incident_management/process_alerts_worker_spec.rb b/ee/spec/workers/incident_management/process_alert_worker_spec.rb
similarity index 52%
rename from ee/spec/workers/incident_management/process_alerts_worker_spec.rb
rename to ee/spec/workers/incident_management/process_alert_worker_spec.rb
index 85a0b4fe6a39b4..b23a9c6b2dc539 100644
--- a/ee/spec/workers/incident_management/process_alerts_worker_spec.rb
+++ b/ee/spec/workers/incident_management/process_alert_worker_spec.rb
@@ -2,27 +2,23 @@
 
 require 'spec_helper'
 
-describe IncidentManagement::ProcessAlertsWorker do
+describe IncidentManagement::ProcessAlertWorker do
   set(:project) { create(:project) }
 
-  subject { described_class.new.perform(project.id, alerts) }
-
-  context 'with alerts' do
-    let(:alerts) { [:one, :two] }
+  describe '#perform' do
+    let(:alert) { :alert }
     let(:create_issue_service) { spy(:create_issue_service) }
 
+    subject { described_class.new.perform(project.id, alert) }
+
     it 'calls create issue service' do
       expect(Project).to receive(:find_by_id).and_call_original
 
       expect(IncidentManagement::CreateIssueService)
-        .to receive(:new).with(project, nil, :one)
-        .and_return(create_issue_service)
-
-      expect(IncidentManagement::CreateIssueService)
-        .to receive(:new).with(project, nil, :two)
+        .to receive(:new).with(project, nil, :alert)
         .and_return(create_issue_service)
 
-      expect(create_issue_service).to receive(:execute).twice
+      expect(create_issue_service).to receive(:execute)
 
       subject
     end
@@ -30,7 +26,7 @@
     context 'with invalid project' do
       let(:invalid_project_id) { 0 }
 
-      subject { described_class.new.perform(invalid_project_id, alerts) }
+      subject { described_class.new.perform(invalid_project_id, alert) }
 
       it 'does not create issues' do
         expect(Project).to receive(:find_by_id).and_call_original
@@ -40,15 +36,4 @@
       end
     end
   end
-
-  context 'without alerts' do
-    let(:alerts) { [] }
-
-    it 'does nothing' do
-      expect(Project).not_to receive(:find_by_id)
-      expect(IncidentManagement::CreateIssueService).not_to receive(:new)
-
-      subject
-    end
-  end
 end
-- 
GitLab


From 144739fbeca680505d3106a6f16a89f26ecf74bb Mon Sep 17 00:00:00 2001
From: rpereira2 <rpereira@gitlab.com>
Date: Tue, 12 Mar 2019 17:15:20 +0530
Subject: [PATCH 07/16] Fix failing specs

- Add incident_management queue priority settings to sidekiq_queues.yml.
- Correct queue name in all_queues.yml.
---
 config/sidekiq_queues.yml     | 1 +
 ee/app/workers/all_queues.yml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 5031a6684d379e..81b7531ca39ec7 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -103,6 +103,7 @@
   - [elastic_indexer, 1]
   - [elastic_commit_indexer, 1]
   - [export_csv, 1]
+  - [incident_management, 2]
 
   # Deprecated queues: Remove after 10.7
   - geo_base_scheduler
diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml
index af4dc0de084df1..37514017ee96bb 100644
--- a/ee/app/workers/all_queues.yml
+++ b/ee/app/workers/all_queues.yml
@@ -45,7 +45,7 @@
 - pipeline_default:store_security_reports
 - pipeline_default:ci_create_cross_project_pipeline
 
-- incident_management:process_alerts
+- incident_management:incident_management_process_alert
 
 - admin_emails
 - create_github_webhook
-- 
GitLab


From d40742663b85db3d298fb132721e1b27600da92c Mon Sep 17 00:00:00 2001
From: rpereira2 <rpereira@gitlab.com>
Date: Tue, 12 Mar 2019 17:22:16 +0530
Subject: [PATCH 08/16] Reuse incident_management_setting in send_email

---
 ee/app/services/projects/prometheus/alerts/notify_service.rb | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb
index 801e1aa9a97a43..9440442c0ddb37 100644
--- a/ee/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb
@@ -27,7 +27,6 @@ def incident_management_feature_enabled?
           Feature.enabled?(:incident_management)
         end
 
-        # TODO re-use in send_email? once that code is in `master`
         def incident_management_setting
           strong_memoize(:incident_management_setting) do
             project.incident_management_setting ||
@@ -39,9 +38,7 @@ def send_email?
           return firings.any? unless incident_management_feature_enabled? &&
               has_incident_management_license?
 
-          setting = project.incident_management_setting || project.build_incident_management_setting
-
-          setting.send_email && firings.any?
+          incident_management_setting.send_email && firings.any?
         end
 
         def create_issue?
-- 
GitLab


From c759b624079746d72b0daa42141f196a9b0a94d9 Mon Sep 17 00:00:00 2001
From: rpereira2 <rpereira@gitlab.com>
Date: Tue, 12 Mar 2019 19:42:10 +0530
Subject: [PATCH 09/16] Add check for feature flag before creating issue

This is so that the feature flag defaults to not enabled. Implicit
feature flags default to enabled, which is not what we want.
---
 .../projects/prometheus/alerts/notify_service.rb         | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb
index 9440442c0ddb37..19638b0c130bd0 100644
--- a/ee/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb
@@ -27,6 +27,10 @@ def incident_management_feature_enabled?
           Feature.enabled?(:incident_management)
         end
 
+        def incident_management_available?
+          incident_management_feature_enabled? && has_incident_management_license?
+        end
+
         def incident_management_setting
           strong_memoize(:incident_management_setting) do
             project.incident_management_setting ||
@@ -35,15 +39,14 @@ def incident_management_setting
         end
 
         def send_email?
-          return firings.any? unless incident_management_feature_enabled? &&
-              has_incident_management_license?
+          return firings.any? unless incident_management_available?
 
           incident_management_setting.send_email && firings.any?
         end
 
         def create_issue?
           return unless firings.any?
-          return unless has_incident_management_license?
+          return unless incident_management_available?
 
           incident_management_setting.create_issue?
         end
-- 
GitLab


From 85fff85c222c76a679e62d540c202b8f0280c2bb Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:10:25 +0100
Subject: [PATCH 10/16] Inline loading template content

---
 .../project_incident_management_setting.rb             | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/ee/app/models/incident_management/project_incident_management_setting.rb b/ee/app/models/incident_management/project_incident_management_setting.rb
index 8a825fe9965216..346d9290ec65db 100644
--- a/ee/app/models/incident_management/project_incident_management_setting.rb
+++ b/ee/app/models/incident_management/project_incident_management_setting.rb
@@ -13,7 +13,9 @@ def available_issue_templates
     end
 
     def issue_template_content
-      strong_memoize(:issue_template_content) { load_issue_template_content }
+      strong_memoize(:issue_template_content) do
+        find_issue_template&.content if issue_template_key.present?
+      end
     end
 
     private
@@ -24,12 +26,6 @@ def issue_template_exists
       errors.add(:issue_template_key, 'not found') unless find_issue_template
     end
 
-    def load_issue_template_content
-      return unless issue_template_key.present?
-
-      find_issue_template&.content
-    end
-
     def find_issue_template
       Gitlab::Template::IssueTemplate.find(issue_template_key, project)
     rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
-- 
GitLab


From eb6abbc057e58193a66421befccc827d63adb1a2 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:12:22 +0100
Subject: [PATCH 11/16] Rename `find_issue_template` to `issue_template`

---
 .../project_incident_management_setting.rb                  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ee/app/models/incident_management/project_incident_management_setting.rb b/ee/app/models/incident_management/project_incident_management_setting.rb
index 346d9290ec65db..bf57c5b883f24c 100644
--- a/ee/app/models/incident_management/project_incident_management_setting.rb
+++ b/ee/app/models/incident_management/project_incident_management_setting.rb
@@ -14,7 +14,7 @@ def available_issue_templates
 
     def issue_template_content
       strong_memoize(:issue_template_content) do
-        find_issue_template&.content if issue_template_key.present?
+        issue_template&.content if issue_template_key.present?
       end
     end
 
@@ -23,10 +23,10 @@ def issue_template_content
     def issue_template_exists
       return unless issue_template_key.present?
 
-      errors.add(:issue_template_key, 'not found') unless find_issue_template
+      errors.add(:issue_template_key, 'not found') unless issue_template
     end
 
-    def find_issue_template
+    def issue_template
       Gitlab::Template::IssueTemplate.find(issue_template_key, project)
     rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
     end
-- 
GitLab


From 165191ac810d3660aa61baa426c0f7417d100fa4 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:36:44 +0100
Subject: [PATCH 12/16] Re-use memoized alert

---
 ee/app/services/incident_management/create_issue_service.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb
index 7e0f0e658f55fc..c930cee1dbfc1b 100644
--- a/ee/app/services/incident_management/create_issue_service.rb
+++ b/ee/app/services/incident_management/create_issue_service.rb
@@ -8,14 +8,14 @@ def execute
       return error('disabled') unless incident_management_setting.create_issue?
       return error('alert invalid') unless alert.valid?
 
-      issue = create_issue(alert)
+      issue = create_issue
 
       success(issue: issue)
     end
 
     private
 
-    def create_issue(alert)
+    def create_issue
       Issues::CreateService.new(
         project,
         issue_author,
-- 
GitLab


From b0126236c752781e93f64323d68e894bc502b137 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:37:06 +0100
Subject: [PATCH 13/16] Re-order logic check

---
 ee/app/services/projects/prometheus/alerts/notify_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ee/app/services/projects/prometheus/alerts/notify_service.rb b/ee/app/services/projects/prometheus/alerts/notify_service.rb
index 19638b0c130bd0..527d73f1e81047 100644
--- a/ee/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/ee/app/services/projects/prometheus/alerts/notify_service.rb
@@ -28,7 +28,7 @@ def incident_management_feature_enabled?
         end
 
         def incident_management_available?
-          incident_management_feature_enabled? && has_incident_management_license?
+          has_incident_management_license? && incident_management_feature_enabled?
         end
 
         def incident_management_setting
-- 
GitLab


From 92369178f4c727998bac12c7eb2fb6b4196b2b37 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:39:51 +0100
Subject: [PATCH 14/16] Inline issue into success

---
 ee/app/services/incident_management/create_issue_service.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb
index c930cee1dbfc1b..69f518c6fb7e3d 100644
--- a/ee/app/services/incident_management/create_issue_service.rb
+++ b/ee/app/services/incident_management/create_issue_service.rb
@@ -8,9 +8,7 @@ def execute
       return error('disabled') unless incident_management_setting.create_issue?
       return error('alert invalid') unless alert.valid?
 
-      issue = create_issue
-
-      success(issue: issue)
+      success(issue: create_issue)
     end
 
     private
-- 
GitLab


From 6933ce842b6671c4b3719dae24448cd58563c88c Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 11:42:10 +0100
Subject: [PATCH 15/16] Memoize and streamline alert annotation list

---
 .../incident_management/create_issue_service.rb      | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb
index 69f518c6fb7e3d..02dd6e0a7aa286 100644
--- a/ee/app/services/incident_management/create_issue_service.rb
+++ b/ee/app/services/incident_management/create_issue_service.rb
@@ -43,8 +43,6 @@ def issue_description
     end
 
     def alert_summary
-      annotation_list = annotations_as_list(alert.annotations)
-
       <<~MARKDOWN
         ## Summary
 
@@ -52,10 +50,12 @@ def alert_summary
       MARKDOWN
     end
 
-    def annotations_as_list(annotations)
-      annotations
-        .map { |annotation| "* #{annotation.label}: #{annotation.value}" }
-        .join("\n")
+    def annotation_list
+      strong_memoize(:annotation_list) do
+        alert.annotations
+          .map { |annotation| "* #{annotation.label}: #{annotation.value}" }
+          .join("\n")
+      end
     end
 
     def alert
-- 
GitLab


From 5c4b33176ecbb23c09eeedd2baf70b4e2415e102 Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Fri, 15 Mar 2019 12:21:48 +0100
Subject: [PATCH 16/16] Log possible errors during incident creation

---
 .../incident_management/create_issue_service.rb  | 10 ++++++++--
 .../create_issue_service_spec.rb                 | 16 ++++++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/ee/app/services/incident_management/create_issue_service.rb b/ee/app/services/incident_management/create_issue_service.rb
index 02dd6e0a7aa286..f30629d3cdfb62 100644
--- a/ee/app/services/incident_management/create_issue_service.rb
+++ b/ee/app/services/incident_management/create_issue_service.rb
@@ -5,8 +5,8 @@ class CreateIssueService < BaseService
     include Gitlab::Utils::StrongMemoize
 
     def execute
-      return error('disabled') unless incident_management_setting.create_issue?
-      return error('alert invalid') unless alert.valid?
+      return error_with('setting disabled') unless incident_management_setting.create_issue?
+      return error_with('invalid alert') unless alert.valid?
 
       success(issue: create_issue)
     end
@@ -74,5 +74,11 @@ def incident_management_setting
           project.build_incident_management_setting
       end
     end
+
+    def error_with(message)
+      log_error(%{Cannot create incident issue for "#{project.full_name}": #{message}})
+
+      error(message)
+    end
   end
 end
diff --git a/ee/spec/services/incident_management/create_issue_service_spec.rb b/ee/spec/services/incident_management/create_issue_service_spec.rb
index b1cc7521fc0706..f0841dae9ea898 100644
--- a/ee/spec/services/incident_management/create_issue_service_spec.rb
+++ b/ee/spec/services/incident_management/create_issue_service_spec.rb
@@ -54,7 +54,11 @@
       let(:alert_payload) { build_alert_payload(annotations: {}) }
 
       it 'does not create an issue' do
-        expect(subject).to eq(status: :error, message: 'alert invalid')
+        expect(service)
+          .to receive(:log_error)
+          .with(error_message('invalid alert'))
+
+        expect(subject).to eq(status: :error, message: 'invalid alert')
       end
     end
   end
@@ -65,7 +69,11 @@
     end
 
     it 'returns an error' do
-      expect(subject).to eq(status: :error, message: 'disabled')
+      expect(service)
+        .to receive(:log_error)
+        .with(error_message('setting disabled'))
+
+      expect(subject).to eq(status: :error, message: 'setting disabled')
     end
   end
 
@@ -74,4 +82,8 @@
   def build_alert_payload(annotations: {})
     { 'annotations' => annotations.stringify_keys }
   end
+
+  def error_message(message)
+    %{Cannot create incident issue for "#{project.full_name}": #{message}}
+  end
 end
-- 
GitLab