diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index a4e6ef27687929fba2efeae3e604d5cd8725d9d6..a5d0eecdc7bf2400ecb356bd54727d580b3d2c1c 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -150,6 +150,24 @@ execute. More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/195). If you are implementing a worker that could benefit from a different strategy, please comment in the issue. +If the automatic deduplication were to cause issues in certain +queues. This can be temporarily disabled by enabling a feature flag +named `disable__deduplication`. For example to disable +deduplication for the `AuthorizedProjectsWorker`, we would enable the +feature flag `disable_authorized_projects_deduplication`. + +From chatops: + +```shell +/chatops run feature set disable_authorized_projects_deduplication true +``` + +From the rails console: + +```ruby +Feature.enable!(:disable_authorized_projects_deduplication) +``` + ## Job urgency Jobs can have an `urgency` attribute set, which can be `:high`, diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 79bbb99752ef33dedc086bb1f68ecdf0bc12913e..fa742d07af2a82e2ef380419f4794c36e76cbed4 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -67,7 +67,7 @@ def duplicate? end def droppable? - idempotent? && duplicate? + idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication") end private diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb index 6e8a8c03aad13ea6bc22475cfac1ef5a2aa0b2c5..929df0a7ffbdff958dd3f85198e27512866ba66a 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb @@ -113,22 +113,27 @@ end describe 'droppable?' do - where(:idempotent, :duplicate) do - # [true, false].repeated_permutation(2) - [[true, true], - [true, false], - [false, true], - [false, false]] + where(:idempotent, :duplicate, :prevent_deduplication) do + # [true, false].repeated_permutation(3) + [[true, true, true], + [true, true, false], + [true, false, true], + [true, false, false], + [false, true, true], + [false, true, false], + [false, false, true], + [false, false, false]] end with_them do before do allow(AuthorizedProjectsWorker).to receive(:idempotent?).and_return(idempotent) allow(duplicate_job).to receive(:duplicate?).and_return(duplicate) + stub_feature_flags("disable_#{queue}_deduplication" => prevent_deduplication) end it 'is droppable when all conditions are met' do - if idempotent && duplicate + if idempotent && duplicate && !prevent_deduplication expect(duplicate_job).to be_droppable else expect(duplicate_job).not_to be_droppable