From 4c7d01e373d712499c895887cc58170a04cefead Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 19 Oct 2016 14:49:07 +0200
Subject: [PATCH 01/21] Add initial version of pipeline unlock worker

---
 app/services/ci/process_pipeline_service.rb | 22 ++++---
 app/workers/pipeline_unlock_worker.rb       | 11 ++++
 spec/workers/pipeline_unlock_worker_spec.rb | 67 +++++++++++++++++++++
 3 files changed, 91 insertions(+), 9 deletions(-)
 create mode 100644 app/workers/pipeline_unlock_worker.rb
 create mode 100644 spec/workers/pipeline_unlock_worker_spec.rb

diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 79eb97b7b551..720e0a1c3b0f 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -7,18 +7,26 @@ def execute(pipeline)
 
       ensure_created_builds! # TODO, remove me in 9.0
 
-      new_builds =
-        stage_indexes_of_created_builds.map do |index|
-          process_stage(index)
-        end
+      new_builds = enqueue_builds!
 
-      @pipeline.update_status
+      pipeline.update_status
+      pipeline.touch
 
       new_builds.flatten.any?
     end
 
     private
 
+    def enqueue_builds!
+      stage_indexes_of_created_builds.map do |index|
+        process_stage(index)
+      end
+    end
+
+    def stage_indexes_of_created_builds
+      created_builds.order(:stage_idx).pluck('distinct stage_idx')
+    end
+
     def process_stage(index)
       current_status = status_for_prior_stages(index)
 
@@ -58,10 +66,6 @@ def status_for_prior_stages(index)
       pipeline.builds.where('stage_idx < ?', index).latest.status || 'success'
     end
 
-    def stage_indexes_of_created_builds
-      created_builds.order(:stage_idx).pluck('distinct stage_idx')
-    end
-
     def created_builds_in_stage(index)
       created_builds.where(stage_idx: index)
     end
diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
new file mode 100644
index 000000000000..18bb59437337
--- /dev/null
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -0,0 +1,11 @@
+class PipelineUnlockWorker
+  include Sidekiq::Worker
+
+  def perform
+    Ci::Pipeline.running_or_pending
+      .where('updated_at < ?', 6.hours.ago)
+      .find_each do |pipeline|
+        PipelineProcessWorker.new.perform(pipeline.id)
+      end
+  end
+end
diff --git a/spec/workers/pipeline_unlock_worker_spec.rb b/spec/workers/pipeline_unlock_worker_spec.rb
new file mode 100644
index 000000000000..9b7953c0c999
--- /dev/null
+++ b/spec/workers/pipeline_unlock_worker_spec.rb
@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe PipelineUnlockWorker do
+  describe '#perform' do
+    let(:worker) { described_class.new }
+
+    context 'when locked pipelines are present' do
+      let(:pipeline) do
+        create(:ci_pipeline, status: :running,
+                             updated_at: 10.hours.ago)
+      end
+
+      context 'when pipeline is locked in the last stage' do
+        before do
+          create_build(status: :success, stage: 'test')
+        end
+
+        it 'updates pipeline status to finished' do
+          expect(pipeline.reload.status).to eq 'running'
+
+          worker.perform
+
+          expect(pipeline.reload.status).to eq 'success'
+        end
+      end
+
+      context 'when locked pipeline has multiple stages ahead' do
+        before do
+          create_build(status: :success, stage: 'build', stage_idx: 1)
+          create_build(status: :created, stage: 'test', stage_idx: 2)
+          create_build(status: :created, stage: 'deploy', stage_idx: 3)
+        end
+
+        it 'retriggers pipeline processing' do
+          expect(pipeline.reload.status).to eq 'running'
+
+          worker.perform
+
+          expect(pipeline.reload.status).to eq 'running'
+          expect(pipeline.builds.find_by(stage: 'test').status)
+            .to eq 'pending'
+          expect(pipeline.builds.find_by(stage: 'deploy').status)
+            .to eq 'created'
+        end
+
+        it 'updates pipeline' do
+          expect { worker.perform }
+            .to change { pipeline.reload.updated_at }
+        end
+      end
+    end
+
+    context 'when locked pipelines are not present' do
+      context 'when there are fresh running pipelines' do
+      end
+
+      context 'when there are no pipelines at all' do
+        it 'does nothing' do
+        end
+      end
+    end
+  end
+
+  def create_build(opts)
+    create(:ci_build, opts.merge(pipeline: pipeline))
+  end
+end
-- 
GitLab


From 8cb45eba37c6e8ee7eb5225dba4ac4848b03314d Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 19 Oct 2016 15:13:41 +0200
Subject: [PATCH 02/21] Extend specs for pipeline unlock worker

---
 spec/workers/pipeline_unlock_worker_spec.rb | 26 ++++++++++++++-------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/spec/workers/pipeline_unlock_worker_spec.rb b/spec/workers/pipeline_unlock_worker_spec.rb
index 9b7953c0c999..012081b9c035 100644
--- a/spec/workers/pipeline_unlock_worker_spec.rb
+++ b/spec/workers/pipeline_unlock_worker_spec.rb
@@ -16,11 +16,11 @@
         end
 
         it 'updates pipeline status to finished' do
-          expect(pipeline.reload.status).to eq 'running'
+          expect(pipeline.reload).to be_running
 
           worker.perform
 
-          expect(pipeline.reload.status).to eq 'success'
+          expect(pipeline.reload).to be_success
         end
       end
 
@@ -32,15 +32,13 @@
         end
 
         it 'retriggers pipeline processing' do
-          expect(pipeline.reload.status).to eq 'running'
+          expect(pipeline.reload).to be_running
 
           worker.perform
 
-          expect(pipeline.reload.status).to eq 'running'
-          expect(pipeline.builds.find_by(stage: 'test').status)
-            .to eq 'pending'
-          expect(pipeline.builds.find_by(stage: 'deploy').status)
-            .to eq 'created'
+          expect(pipeline.reload).to be_running
+          expect(find_build(stage: 'test')).to be_pending
+          expect(find_build(stage: 'deploy')).to be_created
         end
 
         it 'updates pipeline' do
@@ -52,10 +50,18 @@
 
     context 'when locked pipelines are not present' do
       context 'when there are fresh running pipelines' do
+        before { create(:ci_pipeline, status: :running) }
+
+        it 'does not trigger update' do
+          expect_any_instance_of(PipelineProcessWorker)
+            .not_to receive(:perform)
+        end
       end
 
       context 'when there are no pipelines at all' do
         it 'does nothing' do
+          expect_any_instance_of(PipelineProcessWorker)
+            .not_to receive(:perform)
         end
       end
     end
@@ -64,4 +70,8 @@
   def create_build(opts)
     create(:ci_build, opts.merge(pipeline: pipeline))
   end
+
+  def find_build(opts)
+    pipeline.builds.find_by(opts)
+  end
 end
-- 
GitLab


From 454392c1ac8da4a4408515a078d8a4d41d96feaf Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 21 Oct 2016 12:18:16 +0200
Subject: [PATCH 03/21] Configure pipeline unlock worker hourly cron job

---
 config/initializers/1_settings.rb        |  4 +++-
 spec/config/cron/pipeline_unlock_spec.rb | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 spec/config/cron/pipeline_unlock_spec.rb

diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index ee97b4e42b95..41fb3a970b5c 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -311,13 +311,15 @@ def host(url)
 Settings.cron_jobs['prune_old_events_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['prune_old_events_worker']['cron'] ||= '0 */6 * * *'
 Settings.cron_jobs['prune_old_events_worker']['job_class'] = 'PruneOldEventsWorker'
-
 Settings.cron_jobs['trending_projects_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['trending_projects_worker']['cron'] = '0 1 * * *'
 Settings.cron_jobs['trending_projects_worker']['job_class'] = 'TrendingProjectsWorker'
 Settings.cron_jobs['remove_unreferenced_lfs_objects_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['cron'] ||= '20 0 * * *'
 Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'RemoveUnreferencedLfsObjectsWorker'
+Settings.cron_jobs['pipeline_unlock_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pipeline_unlock_worker']['cron'] = '40 * * * *'
+Settings.cron_jobs['pipeline_unlock_worker']['job_class'] = 'PipelineUnlockWorker'
 
 #
 # GitLab Shell
diff --git a/spec/config/cron/pipeline_unlock_spec.rb b/spec/config/cron/pipeline_unlock_spec.rb
new file mode 100644
index 000000000000..7eda716873d0
--- /dev/null
+++ b/spec/config/cron/pipeline_unlock_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Settings do
+  describe 'cron jobs' do
+    describe 'pipeline unlock worker' do
+      subject do
+        described_class.cron_jobs[:pipeline_unlock_worker]
+      end
+
+      it 'is scheduled hourly' do
+        expect(subject.cron).to eq '40 * * * *'
+      end
+
+      it 'is tied to proper class' do
+        expect(subject.job_class).to eq 'PipelineUnlockWorker'
+      end
+    end
+  end
+end
-- 
GitLab


From 47e8ab9e09f1ce1e27dc2a08ae9f55a50ebdb426 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 24 Oct 2016 10:44:11 +0200
Subject: [PATCH 04/21] Place pipeline unlock worker in pipeline queue

---
 app/workers/pipeline_unlock_worker.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index 18bb59437337..af62a6d90b31 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -1,5 +1,6 @@
 class PipelineUnlockWorker
   include Sidekiq::Worker
+  include CronjobQueue
 
   def perform
     Ci::Pipeline.running_or_pending
-- 
GitLab


From afae0db17449e876e5a32fb5b6bff908bb5c1afa Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 15:01:20 +0100
Subject: [PATCH 05/21] Extend pipeline scopes, add `unfinished` scope

Also add missing `skipped` status to `finished` scope.
---
 app/models/concerns/has_status.rb       |  8 ++--
 spec/models/ci/pipeline_spec.rb         | 61 +++++++++++++++++++++----
 spec/models/concerns/has_status_spec.rb |  5 +-
 3 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 90432fc4050a..f13e5e7e5651 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -73,12 +73,10 @@ def all_state_names
     scope :canceled, -> { where(status: 'canceled')  }
     scope :skipped, -> { where(status: 'skipped')  }
     scope :running_or_pending, -> { where(status: [:running, :pending]) }
-    scope :finished, -> { where(status: [:success, :failed, :canceled]) }
     scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) }
-
-    scope :cancelable, -> do
-      where(status: [:running, :pending, :created])
-    end
+    scope :cancelable, -> { where(status: [:running, :pending, :created]) }
+    scope :finished, -> { where(status: [:success, :failed, :canceled, :skipped]) }
+    scope :unfinished, -> { where(status: [:created, :running, :pending]) }
   end
 
   def started?
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index cebaa157ef33..d463a652bea2 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -1,10 +1,13 @@
 require 'spec_helper'
 
-describe Ci::Pipeline, models: true do
+describe Ci::Pipeline, :models do
   include EmailHelpers
 
-  let(:project) { FactoryGirl.create :empty_project }
-  let(:pipeline) { FactoryGirl.create :ci_empty_pipeline, status: 'created', project: project }
+  let(:project) { create :empty_project }
+
+  let(:pipeline) do
+    create(:ci_empty_pipeline, status: :created, project: project)
+  end
 
   it { is_expected.to belong_to(:project) }
   it { is_expected.to belong_to(:user) }
@@ -865,24 +868,38 @@ def create_build(name, stage_idx)
   end
 
   describe "#merge_requests" do
-    let(:project) { FactoryGirl.create :project }
-    let(:pipeline) { FactoryGirl.create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master', sha: project.repository.commit('master').id) }
+    let(:project) { create(:project) }
+
+    let(:commit_id) do
+      project.repository.commit('master').id
+    end
+
+    let(:pipeline) do
+      create(:ci_empty_pipeline, status: 'created',
+                                 project: project,
+                                 ref: 'master',
+                                 sha: commit_id)
+    end
 
     it "returns merge requests whose `diff_head_sha` matches the pipeline's SHA" do
-      merge_request = create(:merge_request, source_project: project, source_branch: pipeline.ref)
+      merge_request = create(:merge_request, source_project: project,
+                                             source_branch: pipeline.ref)
 
       expect(pipeline.merge_requests).to eq([merge_request])
     end
 
     it "doesn't return merge requests whose source branch doesn't match the pipeline's ref" do
-      create(:merge_request, source_project: project, source_branch: 'feature', target_branch: 'master')
+      create(:merge_request, source_project: project,
+                             source_branch: 'feature',
+                             target_branch: 'master')
 
       expect(pipeline.merge_requests).to be_empty
     end
 
     it "doesn't return merge requests whose `diff_head_sha` doesn't match the pipeline's SHA" do
       create(:merge_request, source_project: project, source_branch: pipeline.ref)
-      allow_any_instance_of(MergeRequest).to receive(:diff_head_sha) { '97de212e80737a608d939f648d959671fb0a0142b' }
+      allow_any_instance_of(MergeRequest)
+        .to receive(:diff_head_sha) { '97de212e80737a608d939f648d959671fb0a0142b' }
 
       expect(pipeline.merge_requests).to be_empty
     end
@@ -964,4 +981,32 @@ def create_build(name, stage_idx)
       it_behaves_like 'not sending any notification'
     end
   end
+
+  describe 'scopes' do
+    before do
+      create(:ci_empty_pipeline, status: :created)
+      create(:ci_empty_pipeline, status: :running)
+      create(:ci_empty_pipeline, status: :pending)
+      create(:ci_empty_pipeline, status: :success)
+      create(:ci_empty_pipeline, status: :skipped)
+    end
+
+    describe '.running_or_pending' do
+      it 'returns pipelines running or pending' do
+        expect(described_class.running_or_pending.count).to eq 2
+      end
+    end
+
+    describe '.finished' do
+      it 'returns finished pipelines' do
+        expect(described_class.finished.count).to eq 2
+      end
+    end
+
+    describe '.unfinished' do
+      it 'returns unfinished pipelines' do
+        expect(described_class.unfinished.count).to eq 3
+      end
+    end
+  end
 end
diff --git a/spec/models/concerns/has_status_spec.rb b/spec/models/concerns/has_status_spec.rb
index 4d0f51fe82a3..7e1b2a9ba31c 100644
--- a/spec/models/concerns/has_status_spec.rb
+++ b/spec/models/concerns/has_status_spec.rb
@@ -6,7 +6,10 @@
 
     shared_examples 'build status summary' do
       context 'all successful' do
-        let!(:statuses) { Array.new(2) { create(type, status: :success) } }
+        let!(:statuses) do
+          create_list(type, 2, status: :success)
+        end
+
         it { is_expected.to eq 'success' }
       end
 
-- 
GitLab


From 65ceed642ae6eeea2888bbf9f1b91589274d2abd Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 15:12:14 +0100
Subject: [PATCH 06/21] Make PipelineUnlockWorker unlock created pipelines

---
 app/workers/pipeline_unlock_worker.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index af62a6d90b31..477615600ba4 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -3,7 +3,7 @@ class PipelineUnlockWorker
   include CronjobQueue
 
   def perform
-    Ci::Pipeline.running_or_pending
+    Ci::Pipeline.unfinished
       .where('updated_at < ?', 6.hours.ago)
       .find_each do |pipeline|
         PipelineProcessWorker.new.perform(pipeline.id)
-- 
GitLab


From ce3e790a024ee6e8c8565292bb7a47f27b294985 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 15:33:12 +0100
Subject: [PATCH 07/21] Extend tests for pipeline unlock worker

---
 spec/workers/pipeline_unlock_worker_spec.rb | 55 ++++++++++++++-------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/spec/workers/pipeline_unlock_worker_spec.rb b/spec/workers/pipeline_unlock_worker_spec.rb
index 012081b9c035..98fd2fa1e7e4 100644
--- a/spec/workers/pipeline_unlock_worker_spec.rb
+++ b/spec/workers/pipeline_unlock_worker_spec.rb
@@ -5,30 +5,51 @@
     let(:worker) { described_class.new }
 
     context 'when locked pipelines are present' do
-      let(:pipeline) do
-        create(:ci_pipeline, status: :running,
-                             updated_at: 10.hours.ago)
-      end
+      context 'when multiple pipelines are locked in the last stage' do
+        let(:created_pipeline) do
+          create(:ci_pipeline, status: :created,
+                               updated_at: 10.hours.ago)
+        end
+
+        let(:pending_pipeline) do
+          create(:ci_pipeline, status: :pending,
+                               updated_at: 10.hours.ago)
+        end
+
+        let(:running_pipeline) do
+          create(:ci_pipeline, status: :running,
+                               updated_at: 10.hours.ago)
+        end
 
-      context 'when pipeline is locked in the last stage' do
         before do
-          create_build(status: :success, stage: 'test')
+          create_build(created_pipeline, :pending, stage: 'test')
+          create_build(pending_pipeline, :success, stage: 'test')
+          create_build(running_pipeline, :failed, stage: 'test')
         end
 
-        it 'updates pipeline status to finished' do
-          expect(pipeline.reload).to be_running
+        it 'updates each pipeline status' do
+          expect(created_pipeline.reload).to be_created
+          expect(pending_pipeline.reload).to be_pending
+          expect(running_pipeline.reload).to be_running
 
           worker.perform
 
-          expect(pipeline.reload).to be_success
+          expect(created_pipeline.reload).to be_pending
+          expect(pending_pipeline.reload).to be_success
+          expect(running_pipeline.reload).to be_failed
         end
       end
 
       context 'when locked pipeline has multiple stages ahead' do
+        let(:pipeline) do
+          create(:ci_pipeline, status: :running,
+                               updated_at: 10.hours.ago)
+        end
+
         before do
-          create_build(status: :success, stage: 'build', stage_idx: 1)
-          create_build(status: :created, stage: 'test', stage_idx: 2)
-          create_build(status: :created, stage: 'deploy', stage_idx: 3)
+          create_build(pipeline, :success, stage: 'build', stage_idx: 1)
+          create_build(pipeline, :created, stage: 'test', stage_idx: 2)
+          create_build(pipeline, :created, stage: 'deploy', stage_idx: 3)
         end
 
         it 'retriggers pipeline processing' do
@@ -37,8 +58,8 @@
           worker.perform
 
           expect(pipeline.reload).to be_running
-          expect(find_build(stage: 'test')).to be_pending
-          expect(find_build(stage: 'deploy')).to be_created
+          expect(find_build(pipeline, stage: 'test')).to be_pending
+          expect(find_build(pipeline, stage: 'deploy')).to be_created
         end
 
         it 'updates pipeline' do
@@ -67,11 +88,11 @@
     end
   end
 
-  def create_build(opts)
-    create(:ci_build, opts.merge(pipeline: pipeline))
+  def create_build(pipeline, status, opts = {})
+    create(:ci_build, opts.merge(pipeline: pipeline, status: status))
   end
 
-  def find_build(opts)
+  def find_build(pipeline, opts)
     pipeline.builds.find_by(opts)
   end
 end
-- 
GitLab


From 7e07df65d6c7d66c0070825235f1772777acddb2 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 15:37:41 +0100
Subject: [PATCH 08/21] Add Changelog entry for unlock pipeline worker

---
 changelogs/unreleased/feature-add-pipeline-unlock-worker.yml | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 changelogs/unreleased/feature-add-pipeline-unlock-worker.yml

diff --git a/changelogs/unreleased/feature-add-pipeline-unlock-worker.yml b/changelogs/unreleased/feature-add-pipeline-unlock-worker.yml
new file mode 100644
index 000000000000..982211ffc639
--- /dev/null
+++ b/changelogs/unreleased/feature-add-pipeline-unlock-worker.yml
@@ -0,0 +1,4 @@
+---
+title: Add worker that is meant to unlock stuck pipelines to cron
+merge_request: 6988
+author: 
-- 
GitLab


From 6222a92aaad78850fec05eb99cd000e9878b612b Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 15:55:00 +0100
Subject: [PATCH 09/21] Touch only pipelines processed by unlock worker

---
 app/services/ci/process_pipeline_service.rb | 9 +++------
 app/workers/pipeline_unlock_worker.rb       | 1 +
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 720e0a1c3b0f..2d97b40789d7 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -7,12 +7,9 @@ def execute(pipeline)
 
       ensure_created_builds! # TODO, remove me in 9.0
 
-      new_builds = enqueue_builds!
-
-      pipeline.update_status
-      pipeline.touch
-
-      new_builds.flatten.any?
+      enqueue_builds!.flatten.any?.tap do
+        pipeline.update_status
+      end
     end
 
     private
diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index 477615600ba4..a6888e9b1885 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -7,6 +7,7 @@ def perform
       .where('updated_at < ?', 6.hours.ago)
       .find_each do |pipeline|
         PipelineProcessWorker.new.perform(pipeline.id)
+        pipeline.touch
       end
   end
 end
-- 
GitLab


From 82ce79bf8ab3ce6ea121fe342f3b2917744964bb Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 7 Nov 2016 16:10:27 +0100
Subject: [PATCH 10/21] Use locking to touch pipeline in unlock worker

---
 app/workers/pipeline_unlock_worker.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index a6888e9b1885..f1a9e96e6ba3 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -7,7 +7,10 @@ def perform
       .where('updated_at < ?', 6.hours.ago)
       .find_each do |pipeline|
         PipelineProcessWorker.new.perform(pipeline.id)
-        pipeline.touch
+
+        Gitlab::OptimisticLocking.retry_lock(pipeline) do |pipeline|
+          pipeline.touch
+        end
       end
   end
 end
-- 
GitLab


From f960cca54b44d7384dd74086c0898c5d5a1ff228 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Tue, 8 Nov 2016 11:23:25 +0100
Subject: [PATCH 11/21] Unfold new builds block in pipeline create service

---
 app/services/ci/process_pipeline_service.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 2d97b40789d7..6097ee4e110a 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -7,9 +7,9 @@ def execute(pipeline)
 
       ensure_created_builds! # TODO, remove me in 9.0
 
-      enqueue_builds!.flatten.any?.tap do
-        pipeline.update_status
-      end
+      new_builds = enqueue_builds!
+      pipeline.update_status
+      new_builds.flatten.any?
     end
 
     private
-- 
GitLab


From 194bc72996236297a3ccbc42bb2728e3817c1c06 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 11:15:56 +0100
Subject: [PATCH 12/21] Add pipelines scope to select pipelines with builds

---
 app/models/ci/pipeline.rb                     |  4 ++
 .../ci/create_pipeline_builds_service.rb      | 14 +++----
 spec/models/ci/pipeline_spec.rb               | 39 +++++++++++++++++++
 3 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index abbbddaa4f67..d69a9dabcfc4 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -21,6 +21,10 @@ class Pipeline < ActiveRecord::Base
 
     after_create :keep_around_commits, unless: :importing?
 
+    scope :with_builds, -> do
+      joins(:builds).merge(Ci::Build.all)
+    end
+
     state_machine :status, initial: :created do
       event :enqueue do
         transition created: :pending
diff --git a/app/services/ci/create_pipeline_builds_service.rb b/app/services/ci/create_pipeline_builds_service.rb
index b7da3f8e7eb0..2617d99e8214 100644
--- a/app/services/ci/create_pipeline_builds_service.rb
+++ b/app/services/ci/create_pipeline_builds_service.rb
@@ -25,14 +25,14 @@ def create_build(build_attributes)
         user: current_user,
         trigger_request: trigger_request
       )
-      build = pipeline.builds.create(build_attributes)
 
-      # Create the environment before the build starts. This sets its slug and
-      # makes it available as an environment variable
-      project.environments.find_or_create_by(name: build.expanded_environment_name) if
-        build.has_environment?
-
-      build
+      pipeline.builds.create(build_attributes).tap do |build|
+        # Create the environment before the build starts. This sets its slug and
+        # makes it available as an environment variable
+        if build.has_environment?
+          project.environments.find_or_create_by(name: build.expanded_environment_name)
+        end
+      end
     end
 
     def new_builds
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index d463a652bea2..ea5602ca84f6 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -1008,5 +1008,44 @@ def create_build(name, stage_idx)
         expect(described_class.unfinished.count).to eq 3
       end
     end
+
+    describe '.with_builds' do
+      context 'when pipeline has builds' do
+        before do
+          create(:ci_build, pipeline: pipeline)
+        end
+
+        it 'finds the pipeline with builds' do
+          expect(described_class.with_builds).to include pipeline
+        end
+      end
+
+      context 'when pipeline has only external statuses' do
+        before do
+          create(:generic_commit_status, pipeline: pipeline)
+        end
+
+        it 'does not find without builds' do
+          expect(described_class.with_builds).to be_empty
+        end
+      end
+
+      context 'when pipeline has builds and external statuses' do
+        before do
+          create(:ci_build, pipeline: pipeline)
+          create(:generic_commit_status, pipeline: pipeline)
+        end
+
+        it 'finds pipeline with build' do
+          expect(described_class.with_builds).to include pipeline
+        end
+      end
+
+      context 'when pipeline is empty' do
+        it 'does not find it' do
+          expect(described_class.with_builds).to be_empty
+        end
+      end
+    end
   end
 end
-- 
GitLab


From 190d73c3c942ba5614d60bba4e47ffd9634cedd3 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 12:53:41 +0100
Subject: [PATCH 13/21] Improve readability in pipeline model class

---
 app/models/ci/pipeline.rb | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index d69a9dabcfc4..e4de223e0f79 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -312,6 +312,12 @@ def process!
       Ci::ProcessPipelineService.new(project, user).execute(self)
     end
 
+    def predefined_variables
+      [
+        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
+      ]
+    end
+
     def update_status
       Gitlab::OptimisticLocking.retry_lock(self) do
         case latest_builds_status
@@ -325,10 +331,10 @@ def update_status
       end
     end
 
-    def predefined_variables
-      [
-        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
-      ]
+    def update_duration
+      return unless started_at
+
+      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
     end
 
     def queued_duration
@@ -338,12 +344,6 @@ def queued_duration
       seconds unless seconds.zero?
     end
 
-    def update_duration
-      return unless started_at
-
-      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
-    end
-
     def execute_hooks
       data = pipeline_data
       project.execute_hooks(data, :pipeline_hooks)
@@ -371,7 +371,7 @@ def pipeline_data
     end
 
     def latest_builds_status
-      return 'failed' unless yaml_errors.blank?
+      return 'failed' if yaml_errors.present?
 
       statuses.latest.status || 'skipped'
     end
-- 
GitLab


From b2d3a653350704c9c851c575c241f3bc20d07071 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 13:07:19 +0100
Subject: [PATCH 14/21] Add timing synchronization method to ci pipeline

---
 app/models/ci/pipeline.rb       |  6 ++++++
 spec/models/ci/pipeline_spec.rb | 24 ++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index e4de223e0f79..c9cab3d13f0c 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -331,6 +331,12 @@ def update_status
       end
     end
 
+    def update_timing
+      Gitlab::OptimisticLocking.retry_lock(self) do
+        touch if updated_at < 5.minutes.ago
+      end
+    end
+
     def update_duration
       return unless started_at
 
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index ea5602ca84f6..ba40e28a6cb1 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -982,6 +982,30 @@ def create_build(name, stage_idx)
     end
   end
 
+  describe '#update_timing' do
+    before do
+      pipeline.touch
+    end
+
+    context 'pipeline updated less than 5 minutes ago' do
+      it 'does not update pipeline timing' do
+        travel_to(2.minutes.from_now) do
+          expect { pipeline.update_timing }
+            .not_to change { pipeline.updated_at }
+        end
+      end
+    end
+
+    context 'pipeline not updated less than 5 minutes ago' do
+      it 'updates pipeline timing' do
+        travel_to(8.minutes.from_now) do
+          expect { pipeline.update_timing }
+            .to change { pipeline.updated_at }
+        end
+      end
+    end
+  end
+
   describe 'scopes' do
     before do
       create(:ci_empty_pipeline, status: :created)
-- 
GitLab


From 6ea718dc7081c295596b66f5611acd96c4b1fa96 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 13:27:44 +0100
Subject: [PATCH 15/21] Update pipeline updated_at time after processing it

---
 app/services/ci/process_pipeline_service.rb   |  3 +++
 .../ci/process_pipeline_service_spec.rb       | 27 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 6097ee4e110a..897199b990cb 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -8,7 +8,10 @@ def execute(pipeline)
       ensure_created_builds! # TODO, remove me in 9.0
 
       new_builds = enqueue_builds!
+
       pipeline.update_status
+      pipeline.update_timing
+
       new_builds.flatten.any?
     end
 
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index ebb11166964a..2692e3edce67 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -351,6 +351,33 @@
         expect(all_builds.count).to eq(4)
       end
     end
+
+    context 'when there are no changes in the pipeline' do
+      before do
+        # We process pipeline here so that subsequent call to
+        # pipeline processing facility does not have any work to do.
+        #
+        process_pipeline
+      end
+
+      context 'when updating pipeline after some time' do
+        it 'bumps updated_at as we processed this pipeline again' do
+          travel_to(10.minute.from_now) do
+            expect { process_pipeline }
+              .to change { pipeline.reload.updated_at }
+          end
+        end
+      end
+
+      context 'when processing pipeline again shortly after' do
+        it 'does not bump update_at' do
+          travel_to(2.minute.from_now) do
+            expect { process_pipeline }
+              .not_to change { pipeline.reload.updated_at }
+          end
+        end
+      end
+    end
   end
 
   def all_builds
-- 
GitLab


From 6b365104e406d4c4e69bed1b38824512676a703a Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 13:56:37 +0100
Subject: [PATCH 16/21] Blend timing changes into pipeline unlock worker

---
 app/workers/pipeline_unlock_worker.rb       | 12 +++++-------
 spec/workers/pipeline_unlock_worker_spec.rb | 12 ++++++++++++
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index f1a9e96e6ba3..111fb5cccef6 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -3,14 +3,12 @@ class PipelineUnlockWorker
   include CronjobQueue
 
   def perform
-    Ci::Pipeline.unfinished
-      .where('updated_at < ?', 6.hours.ago)
+    Ci::Pipeline.unfinished.with_builds
+      .where('ci_commits.updated_at < ?', 6.hours.ago)
+      .where('ci_commits.created_at > ?', 1.week.ago)
+      .select(:id)
       .find_each do |pipeline|
-        PipelineProcessWorker.new.perform(pipeline.id)
-
-        Gitlab::OptimisticLocking.retry_lock(pipeline) do |pipeline|
-          pipeline.touch
-        end
+        PipelineProcessWorker.perform_async(pipeline.id)
       end
   end
 end
diff --git a/spec/workers/pipeline_unlock_worker_spec.rb b/spec/workers/pipeline_unlock_worker_spec.rb
index 98fd2fa1e7e4..cfe5380fe51a 100644
--- a/spec/workers/pipeline_unlock_worker_spec.rb
+++ b/spec/workers/pipeline_unlock_worker_spec.rb
@@ -67,6 +67,18 @@
             .to change { pipeline.reload.updated_at }
         end
       end
+
+      context 'when locked pipeline is more than week old' do
+        before do
+          create(:ci_pipeline, status: :created,
+                               updated_at: 2.weeks.ago)
+        end
+
+        it 'does not trigger update' do
+          expect_any_instance_of(PipelineProcessWorker)
+            .not_to receive(:perform)
+        end
+      end
     end
 
     context 'when locked pipelines are not present' do
-- 
GitLab


From e0d6015c5cebcdf983c4bfad5310d15e286675b8 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Fri, 25 Nov 2016 15:20:07 +0100
Subject: [PATCH 17/21] Fix Rubocop offenses in process pipeline specs

---
 spec/services/ci/process_pipeline_service_spec.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index 2692e3edce67..363b93c88713 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -362,7 +362,7 @@
 
       context 'when updating pipeline after some time' do
         it 'bumps updated_at as we processed this pipeline again' do
-          travel_to(10.minute.from_now) do
+          travel_to(10.minutes.from_now) do
             expect { process_pipeline }
               .to change { pipeline.reload.updated_at }
           end
@@ -371,7 +371,7 @@
 
       context 'when processing pipeline again shortly after' do
         it 'does not bump update_at' do
-          travel_to(2.minute.from_now) do
+          travel_to(2.minutes.from_now) do
             expect { process_pipeline }
               .not_to change { pipeline.reload.updated_at }
           end
-- 
GitLab


From 57136446038bbf0a942369a0b06c76cd719eea34 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 28 Nov 2016 12:14:20 +0100
Subject: [PATCH 18/21] Use Sidekiq bulk push with pipeline unlock worker

---
 app/workers/pipeline_unlock_worker.rb       |  8 ++-
 spec/workers/pipeline_unlock_worker_spec.rb | 69 +++++++++++----------
 2 files changed, 41 insertions(+), 36 deletions(-)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index 111fb5cccef6..c2e142bfea58 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -6,9 +6,11 @@ def perform
     Ci::Pipeline.unfinished.with_builds
       .where('ci_commits.updated_at < ?', 6.hours.ago)
       .where('ci_commits.created_at > ?', 1.week.ago)
-      .select(:id)
-      .find_each do |pipeline|
-        PipelineProcessWorker.perform_async(pipeline.id)
+      .select(:id).group(:id).order(:id).pluck(:id).tap do |ids|
+        break if ids.empty?
+
+        Sidekiq::Client.push_bulk('class' => PipelineProcessWorker,
+                                  'args' => ids.in_groups_of(1))
       end
   end
 end
diff --git a/spec/workers/pipeline_unlock_worker_spec.rb b/spec/workers/pipeline_unlock_worker_spec.rb
index cfe5380fe51a..8c2f7a01729c 100644
--- a/spec/workers/pipeline_unlock_worker_spec.rb
+++ b/spec/workers/pipeline_unlock_worker_spec.rb
@@ -27,16 +27,10 @@
           create_build(running_pipeline, :failed, stage: 'test')
         end
 
-        it 'updates each pipeline status' do
-          expect(created_pipeline.reload).to be_created
-          expect(pending_pipeline.reload).to be_pending
-          expect(running_pipeline.reload).to be_running
+        it 'processes each pipeline that needs update' do
+          pipelines = [created_pipeline, pending_pipeline, running_pipeline]
 
-          worker.perform
-
-          expect(created_pipeline.reload).to be_pending
-          expect(pending_pipeline.reload).to be_success
-          expect(running_pipeline.reload).to be_failed
+          expect_pipeline_update(*pipelines) { worker.perform }
         end
       end
 
@@ -53,48 +47,45 @@
         end
 
         it 'retriggers pipeline processing' do
-          expect(pipeline.reload).to be_running
-
-          worker.perform
-
-          expect(pipeline.reload).to be_running
-          expect(find_build(pipeline, stage: 'test')).to be_pending
-          expect(find_build(pipeline, stage: 'deploy')).to be_created
-        end
-
-        it 'updates pipeline' do
-          expect { worker.perform }
-            .to change { pipeline.reload.updated_at }
+          expect_pipeline_update(pipeline) { worker.perform }
         end
       end
 
       context 'when locked pipeline is more than week old' do
-        before do
+        let(:pipeline) do
           create(:ci_pipeline, status: :created,
-                               updated_at: 2.weeks.ago)
+                               created_at: 2.weeks.ago)
+        end
+
+        before do
+          create_build(pipeline, :success)
         end
 
         it 'does not trigger update' do
-          expect_any_instance_of(PipelineProcessWorker)
-            .not_to receive(:perform)
+          expect_no_pipeline_update { worker.perform }
         end
       end
     end
 
     context 'when locked pipelines are not present' do
-      context 'when there are fresh running pipelines' do
-        before { create(:ci_pipeline, status: :running) }
+      context 'when there is a running pipeline' do
+        let(:pipeline) do
+          create(:ci_pipeline, status: :running,
+                               updated_at: 4.hours.ago)
+        end
+
+        before do
+          create_build(pipeline, :success)
+        end
 
         it 'does not trigger update' do
-          expect_any_instance_of(PipelineProcessWorker)
-            .not_to receive(:perform)
+          expect_no_pipeline_update { worker.perform }
         end
       end
 
       context 'when there are no pipelines at all' do
         it 'does nothing' do
-          expect_any_instance_of(PipelineProcessWorker)
-            .not_to receive(:perform)
+          expect_no_pipeline_update { worker.perform }
         end
       end
     end
@@ -104,7 +95,19 @@ def create_build(pipeline, status, opts = {})
     create(:ci_build, opts.merge(pipeline: pipeline, status: status))
   end
 
-  def find_build(pipeline, opts)
-    pipeline.builds.find_by(opts)
+  def expect_pipeline_update(*pipelines)
+    pipeline_ids = pipelines.map(&:id).in_groups_of(1)
+
+    expect(Sidekiq::Client).to receive(:push_bulk)
+      .with(hash_including('args' => pipeline_ids))
+      .once
+
+    yield
+  end
+
+  def expect_no_pipeline_update
+    expect(Sidekiq::Client).not_to receive(:push_bulk)
+
+    yield
   end
 end
-- 
GitLab


From 312ae1b5da00576df7e5945e8c931253eac8cc9c Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Mon, 28 Nov 2016 13:23:52 +0100
Subject: [PATCH 19/21] Find distinct records when using with_builds scope

---
 app/models/ci/pipeline.rb             | 2 +-
 app/workers/pipeline_unlock_worker.rb | 2 +-
 spec/models/ci/pipeline_spec.rb       | 5 +++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index c9cab3d13f0c..49eb1a635e28 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -22,7 +22,7 @@ class Pipeline < ActiveRecord::Base
     after_create :keep_around_commits, unless: :importing?
 
     scope :with_builds, -> do
-      joins(:builds).merge(Ci::Build.all)
+      joins(:builds).merge(Ci::Build.all).distinct
     end
 
     state_machine :status, initial: :created do
diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index c2e142bfea58..5b15db2df8ba 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -6,7 +6,7 @@ def perform
     Ci::Pipeline.unfinished.with_builds
       .where('ci_commits.updated_at < ?', 6.hours.ago)
       .where('ci_commits.created_at > ?', 1.week.ago)
-      .select(:id).group(:id).order(:id).pluck(:id).tap do |ids|
+      .select(:id).order(:id).pluck(:id).tap do |ids|
         break if ids.empty?
 
         Sidekiq::Client.push_bulk('class' => PipelineProcessWorker,
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index ba40e28a6cb1..35c11be05950 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -1034,13 +1034,14 @@ def create_build(name, stage_idx)
     end
 
     describe '.with_builds' do
-      context 'when pipeline has builds' do
+      context 'when pipeline has multiple builds' do
         before do
           create(:ci_build, pipeline: pipeline)
+          create(:ci_build, pipeline: pipeline)
         end
 
         it 'finds the pipeline with builds' do
-          expect(described_class.with_builds).to include pipeline
+          expect(described_class.with_builds).to contain_exactly(pipeline)
         end
       end
 
-- 
GitLab


From bef3cde04a6d7d2ac2f780cef37ddf98ddb27902 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 30 Nov 2016 13:44:28 +0100
Subject: [PATCH 20/21] Do not use redundant select in pipeline unlock worker

---
 app/workers/pipeline_unlock_worker.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/workers/pipeline_unlock_worker.rb b/app/workers/pipeline_unlock_worker.rb
index 5b15db2df8ba..60a10df5a9a9 100644
--- a/app/workers/pipeline_unlock_worker.rb
+++ b/app/workers/pipeline_unlock_worker.rb
@@ -6,7 +6,7 @@ def perform
     Ci::Pipeline.unfinished.with_builds
       .where('ci_commits.updated_at < ?', 6.hours.ago)
       .where('ci_commits.created_at > ?', 1.week.ago)
-      .select(:id).order(:id).pluck(:id).tap do |ids|
+      .order(:id).pluck(:id).tap do |ids|
         break if ids.empty?
 
         Sidekiq::Client.push_bulk('class' => PipelineProcessWorker,
-- 
GitLab


From f4c24e0d59aa2ed7fce587866f6a536f0cf2b05b Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Thu, 1 Dec 2016 11:01:39 +0100
Subject: [PATCH 21/21] Add processed_at datetime field to ci_commits table

---
 .../20161201095020_add_processed_at_to_ci_pipelines.rb   | 9 +++++++++
 db/schema.rb                                             | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20161201095020_add_processed_at_to_ci_pipelines.rb

diff --git a/db/migrate/20161201095020_add_processed_at_to_ci_pipelines.rb b/db/migrate/20161201095020_add_processed_at_to_ci_pipelines.rb
new file mode 100644
index 000000000000..ddd529745ca0
--- /dev/null
+++ b/db/migrate/20161201095020_add_processed_at_to_ci_pipelines.rb
@@ -0,0 +1,9 @@
+class AddProcessedAtToCiPipelines < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column(:ci_commits, :processed_at, :datetime)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 923ece86edb0..41ca42f9c1d6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -243,6 +243,7 @@
     t.integer "duration"
     t.integer "user_id"
     t.integer "lock_version"
+    t.datetime "processed_at"
   end
 
   add_index "ci_commits", ["gl_project_id", "sha"], name: "index_ci_commits_on_gl_project_id_and_sha", using: :btree
@@ -1306,4 +1307,4 @@
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
-end
\ No newline at end of file
+end
-- 
GitLab