diff --git a/app/controllers/concerns/integrations_actions.rb b/app/controllers/concerns/integrations_actions.rb index ff283f9bb62ea103445696bd120dc445430bda26..b3ad89f3227c568cf3b92dfaa94e6bd25d1e7c67 100644 --- a/app/controllers/concerns/integrations_actions.rb +++ b/app/controllers/concerns/integrations_actions.rb @@ -16,10 +16,12 @@ def edit def update saved = integration.update(service_params[:service]) + overwrite = ActiveRecord::Type::Boolean.new.cast(params[:overwrite]) respond_to do |format| format.html do if saved + PropagateIntegrationWorker.perform_async(integration.id, overwrite) redirect_to scoped_edit_integration_path(integration), notice: success_message else render 'shared/integrations/edit' diff --git a/app/models/data_list.rb b/app/models/data_list.rb new file mode 100644 index 0000000000000000000000000000000000000000..12011cb17f718be77216176ba0a572c0ab9242f0 --- /dev/null +++ b/app/models/data_list.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class DataList + def initialize(batch, data_fields_hash, klass) + @batch = batch + @data_fields_hash = data_fields_hash + @klass = klass + end + + def to_array + [klass, columns, values] + end + + private + + attr_reader :batch, :data_fields_hash, :klass + + def columns + data_fields_hash.keys << 'service_id' + end + + def values + batch.map { |row| data_fields_hash.values << row['id'] } + end +end diff --git a/app/models/service.rb b/app/models/service.rb index fb4d9a77077b76881681d160f896a9d30a67c45b..58bae93267a6aa77645217737451fe0c3e9c89fe 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -134,6 +134,14 @@ def json_fields %w(active) end + def to_service_hash + as_json(methods: :type, except: %w[id template instance project_id]) + end + + def to_data_fields_hash + data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id') + end + def test_data(project, user) Gitlab::DataBuilder::Push.build_sample(project, user) end diff --git a/app/models/service_list.rb b/app/models/service_list.rb new file mode 100644 index 0000000000000000000000000000000000000000..fa3760f0c561aab073a84d1536bfdbaaf6202711 --- /dev/null +++ b/app/models/service_list.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ServiceList + def initialize(batch, service_hash, extra_hash = {}) + @batch = batch + @service_hash = service_hash + @extra_hash = extra_hash + end + + def to_array + [Service, columns, values] + end + + private + + attr_reader :batch, :service_hash, :extra_hash + + def columns + (service_hash.keys << 'project_id') + extra_hash.keys + end + + def values + batch.map do |project_id| + (service_hash.values << project_id) + extra_hash.values + end + end +end diff --git a/app/services/admin/propagate_integration_service.rb b/app/services/admin/propagate_integration_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..0a3c61816f8534afe8028f1019a79671ea95f87b --- /dev/null +++ b/app/services/admin/propagate_integration_service.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +module Admin + class PropagateIntegrationService + BATCH_SIZE = 100 + + delegate :data_fields_present?, to: :integration + + def self.propagate(integration:, overwrite:) + new(integration, overwrite).propagate + end + + def initialize(integration, overwrite) + @integration = integration + @overwrite = overwrite + end + + def propagate + if overwrite + update_integration_for_all_projects + else + update_integration_for_inherited_projects + end + + create_integration_for_projects_without_integration + end + + private + + attr_reader :integration, :overwrite + + # rubocop: disable Cop/InBatches + # rubocop: disable CodeReuse/ActiveRecord + def update_integration_for_inherited_projects + Service.where(type: integration.type, inherit_from_id: integration.id).in_batches(of: BATCH_SIZE) do |batch| + bulk_update_from_integration(batch) + end + end + + def update_integration_for_all_projects + Service.where(type: integration.type).in_batches(of: BATCH_SIZE) do |batch| + bulk_update_from_integration(batch) + end + end + # rubocop: enable Cop/InBatches + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def bulk_update_from_integration(batch) + # Retrieving the IDs instantiates the ActiveRecord relation (batch) + # into concrete models, otherwise update_all will clear the relation. + # https://stackoverflow.com/q/34811646/462015 + batch_ids = batch.pluck(:id) + + Service.transaction do + batch.update_all(service_hash) + + if data_fields_present? + integration.data_fields.class.where(service_id: batch_ids).update_all(data_fields_hash) + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def create_integration_for_projects_without_integration + loop do + batch = Project.uncached { project_ids_without_integration } + + bulk_create_from_integration(batch) unless batch.empty? + + break if batch.size < BATCH_SIZE + end + end + + def bulk_create_from_integration(batch) + service_list = ServiceList.new(batch, service_hash, { 'inherit_from_id' => integration.id }).to_array + + Project.transaction do + results = bulk_insert(*service_list) + + if data_fields_present? + data_list = DataList.new(results, data_fields_hash, integration.data_fields.class).to_array + + bulk_insert(*data_list) + end + + run_callbacks(batch) + end + end + + def bulk_insert(klass, columns, values_array) + items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } + + klass.insert_all(items_to_insert, returning: [:id]) + end + + # rubocop: disable CodeReuse/ActiveRecord + def run_callbacks(batch) + if active_external_issue_tracker? + Project.where(id: batch).update_all(has_external_issue_tracker: true) + end + + if active_external_wiki? + Project.where(id: batch).update_all(has_external_wiki: true) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def active_external_issue_tracker? + integration.issue_tracker? && !integration.default + end + + def active_external_wiki? + integration.type == 'ExternalWikiService' + end + + def project_ids_without_integration + Project.connection.select_values( + <<-SQL + SELECT id + FROM projects + WHERE NOT EXISTS ( + SELECT true + FROM services + WHERE services.project_id = projects.id + AND services.type = #{ActiveRecord::Base.connection.quote(integration.type)} + ) + AND projects.pending_delete = false + AND projects.archived = false + LIMIT #{BATCH_SIZE} + SQL + ) + end + + def service_hash + @service_hash ||= integration.to_service_hash + .tap { |json| json['inherit_from_id'] = integration.id } + end + + def data_fields_hash + @data_fields_hash ||= integration.to_data_fields_hash + end + end +end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index 0483c951f1e07dd814895cddbdcb419397a2b6ed..ecca971594014f1ca6e260f02a41c4e05dfc10a6 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -35,17 +35,15 @@ def propagate_projects_with_template end def bulk_create_from_template(batch) - service_list = batch.map do |project_id| - service_hash.values << project_id - end + service_list = ServiceList.new(batch, service_hash).to_array Project.transaction do - results = bulk_insert(Service, service_hash.keys << 'project_id', service_list) + results = bulk_insert(*service_list) if data_fields_present? - data_list = results.map { |row| data_hash.values << row['id'] } + data_list = DataList.new(results, data_fields_hash, template.data_fields.class).to_array - bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list) + bulk_insert(*data_list) end run_callbacks(batch) @@ -77,11 +75,11 @@ def bulk_insert(klass, columns, values_array) end def service_hash - @service_hash ||= template.as_json(methods: :type, except: %w[id template project_id]) + @service_hash ||= template.to_service_hash end - def data_hash - @data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id') + def data_fields_hash + @data_fields_hash ||= template.to_data_fields_hash end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 1f9a53d64d9203bea422ad1f9a6e3be2985311e8..1454ededc047e1fbcf127947c1b4051152f2029b 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1291,6 +1291,13 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true +- :name: propagate_integration + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true - :name: propagate_service_template :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..cbab38465bc32bcb5928e73fc37f27767091575d --- /dev/null +++ b/app/workers/propagate_integration_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PropagateIntegrationWorker + include ApplicationWorker + + feature_category :integrations + + idempotent! + + def perform(integration_id, overwrite) + Admin::PropagateIntegrationService.propagate( + integration: Service.find(integration_id), + overwrite: overwrite + ) + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index e6e0b4b4409fedcb6e7a496f15f264e43db263d2..2079aad01700f9eaf3f1ff66bece46f3490dfc68 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -210,6 +210,8 @@ - 1 - - prometheus_create_default_alerts - 1 +- - propagate_integration + - 1 - - propagate_service_template - 1 - - reactive_caching diff --git a/spec/controllers/admin/integrations_controller_spec.rb b/spec/controllers/admin/integrations_controller_spec.rb index 817223bd91acfc812c4f69c01308c7e607cedc61..94d383531893085a02fe9699ba9f723ef71379ae 100644 --- a/spec/controllers/admin/integrations_controller_spec.rb +++ b/spec/controllers/admin/integrations_controller_spec.rb @@ -36,7 +36,9 @@ let(:integration) { create(:jira_service, :instance) } before do - put :update, params: { id: integration.class.to_param, service: { url: url } } + allow(PropagateIntegrationWorker).to receive(:perform_async) + + put :update, params: { id: integration.class.to_param, overwrite: true, service: { url: url } } end context 'valid params' do @@ -46,6 +48,10 @@ expect(response).to have_gitlab_http_status(:found) expect(integration.reload.url).to eq(url) end + + it 'calls to PropagateIntegrationWorker' do + expect(PropagateIntegrationWorker).to have_received(:perform_async).with(integration.id, true) + end end context 'invalid params' do @@ -56,6 +62,10 @@ expect(response).to render_template(:edit) expect(integration.reload.url).not_to eq(url) end + + it 'does not call to PropagateIntegrationWorker' do + expect(PropagateIntegrationWorker).not_to have_received(:perform_async) + end end end end diff --git a/spec/services/admin/propagate_integration_service_spec.rb b/spec/services/admin/propagate_integration_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..843b78a41e9d172aca6309364c27a7fe366bed5b --- /dev/null +++ b/spec/services/admin/propagate_integration_service_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Admin::PropagateIntegrationService do + describe '.propagate' do + let(:excluded_attributes) { %w[id project_id inherit_from_id instance created_at updated_at title description] } + let!(:project) { create(:project) } + let!(:instance_integration) do + JiraService.create!( + instance: true, + active: true, + push_events: true, + url: 'http://update-jira.instance.com', + username: 'user', + password: 'secret' + ) + end + + let!(:inherited_integration) do + JiraService.create!( + project: create(:project), + inherit_from_id: instance_integration.id, + instance: false, + active: true, + push_events: false, + url: 'http://jira.instance.com', + username: 'user', + password: 'secret' + ) + end + + let!(:not_inherited_integration) do + JiraService.create!( + project: create(:project), + inherit_from_id: nil, + instance: false, + active: true, + push_events: false, + url: 'http://jira.instance.com', + username: 'user', + password: 'secret' + ) + end + + let!(:another_inherited_integration) do + BambooService.create!( + project: create(:project), + inherit_from_id: instance_integration.id, + instance: false, + active: true, + push_events: false, + bamboo_url: 'http://gitlab.com', + username: 'mic', + password: 'password', + build_key: 'build' + ) + end + + shared_examples 'inherits settings from integration' do + it 'updates the inherited integrations' do + described_class.propagate(integration: instance_integration, overwrite: overwrite) + + expect(integration.reload.inherit_from_id).to eq(instance_integration.id) + expect(integration.attributes.except(*excluded_attributes)) + .to eq(instance_integration.attributes.except(*excluded_attributes)) + end + + context 'integration with data fields' do + let(:excluded_attributes) { %w[id service_id created_at updated_at] } + + it 'updates the data fields from inherited integrations' do + described_class.propagate(integration: instance_integration, overwrite: overwrite) + + expect(integration.reload.data_fields.attributes.except(*excluded_attributes)) + .to eq(instance_integration.data_fields.attributes.except(*excluded_attributes)) + end + end + end + + shared_examples 'does not inherit settings from integration' do + it 'does not update the not inherited integrations' do + described_class.propagate(integration: instance_integration, overwrite: overwrite) + + expect(integration.reload.attributes.except(*excluded_attributes)) + .not_to eq(instance_integration.attributes.except(*excluded_attributes)) + end + end + + context 'update only inherited integrations' do + let(:overwrite) { false } + + it_behaves_like 'inherits settings from integration' do + let(:integration) { inherited_integration } + end + + it_behaves_like 'does not inherit settings from integration' do + let(:integration) { not_inherited_integration } + end + + it_behaves_like 'does not inherit settings from integration' do + let(:integration) { another_inherited_integration } + end + + it_behaves_like 'inherits settings from integration' do + let(:integration) { project.jira_service } + end + end + + context 'update all integrations' do + let(:overwrite) { true } + + it_behaves_like 'inherits settings from integration' do + let(:integration) { inherited_integration } + end + + it_behaves_like 'inherits settings from integration' do + let(:integration) { not_inherited_integration } + end + + it_behaves_like 'does not inherit settings from integration' do + let(:integration) { another_inherited_integration } + end + + it_behaves_like 'inherits settings from integration' do + let(:integration) { project.jira_service } + end + end + + it 'updates project#has_external_issue_tracker for issue tracker services' do + described_class.propagate(integration: instance_integration, overwrite: true) + + expect(project.reload.has_external_issue_tracker).to eq(true) + end + + it 'updates project#has_external_wiki for external wiki services' do + instance_integration = ExternalWikiService.create!( + instance: true, + active: true, + push_events: false, + external_wiki_url: 'http://external-wiki-url.com' + ) + + described_class.propagate(integration: instance_integration, overwrite: true) + + expect(project.reload.has_external_wiki).to eq(true) + end + end +end diff --git a/spec/workers/propagate_integration_worker_spec.rb b/spec/workers/propagate_integration_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..e49869a38e90bb57fd24686a08e8291866cc038e --- /dev/null +++ b/spec/workers/propagate_integration_worker_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe PropagateIntegrationWorker do + describe '#perform' do + let(:integration) do + PushoverService.create( + template: true, + active: true, + device: 'MyDevice', + sound: 'mic', + priority: 4, + user_key: 'asdf', + api_key: '123456789' + ) + end + + it 'calls the propagate service with the integration' do + expect(Admin::PropagateIntegrationService).to receive(:propagate) + .with(integration: integration, overwrite: true) + + subject.perform(integration.id, true) + end + end +end