diff --git a/ee/lib/ee/bulk_imports/groups/graphql/get_epic_award_emoji_query.rb b/ee/lib/ee/bulk_imports/groups/graphql/get_epic_award_emoji_query.rb index 1a19f7e83a698eec842f199bff2e57ab222fc9b7..2d7086ac6d2a56de052ae5ffa309e9b340e95f73 100644 --- a/ee/lib/ee/bulk_imports/groups/graphql/get_epic_award_emoji_query.rb +++ b/ee/lib/ee/bulk_imports/groups/graphql/get_epic_award_emoji_query.rb @@ -7,52 +7,55 @@ module Graphql module GetEpicAwardEmojiQuery extend self - def to_s - <<-'GRAPHQL' - query($full_path: ID!, $epic_iid: ID!, $cursor: String, $per_page: Int) { + def to_s(emoji_pages) + <<-GRAPHQL + query($full_path: ID!) { group(fullPath: $full_path) { - epic(iid: $epic_iid) { - award_emoji: awardEmoji(first: $per_page, after: $cursor) { - page_info: pageInfo { - next_page: endCursor - has_next_page: hasNextPage - } - nodes { - name - user { - public_email: publicEmail - } - } - } - } + #{epics(emoji_pages)} } } GRAPHQL end def variables(context) - iid = context.extra[:epic_iid] - { full_path: context.entity.source_full_path, - cursor: context.tracker.next_page, - epic_iid: iid, - per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE } end def data_path - base_path << 'nodes' - end - - def page_info_path - base_path << 'page_info' + base_path << 'group' end private def base_path - %w[data group epic award_emoji] + %w[data] + end + + def epics(emoji_pages) + emoji_pages.map do |page| + epic_iid = page.first + next_page = page.last + + <<-GRAPHQL + epic_#{epic_iid}: epic(iid: #{epic_iid}) { + iid + award_emoji: awardEmoji(first: 50, after: "#{next_page}") { + page_info: pageInfo { + has_next_page: hasNextPage + next_page: endCursor + } + nodes { + name + user { + public_email: publicEmail + } + } + } + } + GRAPHQL + end.join("\n") end end end diff --git a/ee/lib/ee/bulk_imports/groups/graphql/get_epics_award_emoji_query.rb b/ee/lib/ee/bulk_imports/groups/graphql/get_epics_award_emoji_query.rb new file mode 100644 index 0000000000000000000000000000000000000000..c14d5595e8b5605fa25b69f6b81c437095a8d9ff --- /dev/null +++ b/ee/lib/ee/bulk_imports/groups/graphql/get_epics_award_emoji_query.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module EE + module BulkImports + module Groups + module Graphql + module GetEpicsAwardEmojiQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!, $cursor: String, $per_page: Int) { + group(fullPath: $full_path) { + epics( + includeDescendantGroups: false, + first: $per_page, + after: $cursor + ) { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + iid + award_emoji: awardEmoji(first: 20) { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + name + user { + public_email: publicEmail + } + } + } + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + + private + + def base_path + %w[data group epics] + end + end + end + end + end +end diff --git a/ee/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader.rb b/ee/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader.rb deleted file mode 100644 index bbfdaf878d991b568b57c4c81ea102ce2ab82aae..0000000000000000000000000000000000000000 --- a/ee/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module EE - module BulkImports - module Groups - module Loaders - class EpicAwardEmojiLoader - NotAllowedError = Class.new(StandardError) - - # rubocop: disable CodeReuse/ActiveRecord - def load(context, data) - return unless data - - epic = context.group.epics.find_by(iid: context.extra[:epic_iid]) - - return if award_emoji_exists?(epic, data) - - raise NotAllowedError unless Ability.allowed?(context.current_user, :award_emoji, epic) - - epic.award_emoji.create!(data) - end - - private - - def award_emoji_exists?(epic, data) - epic.award_emoji.exists?(user_id: data['user_id'], name: data['name']) - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/ee/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline.rb b/ee/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline.rb index 81f507a1a5282afb8f7aee6587aa1bc46454e7de..fe559895d3f81d3b9a85246de17fe81bfa13176e 100644 --- a/ee/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline.rb +++ b/ee/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline.rb @@ -7,37 +7,91 @@ module Pipelines class EpicAwardEmojiPipeline include ::BulkImports::Pipeline - extractor ::BulkImports::Common::Extractors::GraphqlExtractor, - query: EE::BulkImports::Groups::Graphql::GetEpicAwardEmojiQuery - transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer - transformer ::BulkImports::Common::Transformers::UserReferenceTransformer - - loader EE::BulkImports::Groups::Loaders::EpicAwardEmojiLoader - # rubocop: disable CodeReuse/ActiveRecord def initialize(context) @context = context - @group = context.group - @epic_iids = @group.epics.order(iid: :desc).pluck(:iid) + @next_pages = context.extra[:subrelation_next_pages] + @batch_size = 15 + end + + def extract(context) + return if @next_pages.blank? + + batch = @next_pages.pop(@batch_size) + + response = graphql_client.execute( + graphql_client.parse(query.to_s(batch)), + query.variables(context) + ).original_hash.deep_dup + + ::BulkImports::Pipeline::ExtractedData.new( + data: response.dig(*query.data_path).values, + page_info: nil + ) + end + + def transform(context, data) + nodes = data.dig('award_emoji', 'nodes') + + return if nodes.blank? + + track_next_page(data) + + data['award_emoji']['nodes'] = nodes.map do |node| + ::BulkImports::Common::Transformers::UserReferenceTransformer.new.transform(context, node) + end + + data + end - set_next_epic + def load(context, data) + return unless data + + epic = context.group.epics.find_by_iid(data['iid']) + + return unless epic + + raise NotAllowedError unless Ability.allowed?(context.current_user, :award_emoji, epic) + + data.dig('award_emoji', 'nodes').each do |award_emoji| + next if award_emoji_exists?(epic, award_emoji) + + epic.award_emoji.create!(award_emoji) + end end private def after_run(extracted_data) - set_next_epic unless extracted_data.has_next_page? + run unless @next_pages.blank? + end - if extracted_data.has_next_page? || context.extra[:epic_iid] - run - end + def graphql_client + @graphql_client ||= ::BulkImports::Clients::Graphql.new( + url: context.configuration.url, + token: context.configuration.access_token + ) end - def set_next_epic - context.extra[:epic_iid] = @epic_iids.pop + def query + ::EE::BulkImports::Groups::Graphql::GetEpicAwardEmojiQuery + end + + def award_emoji_exists?(epic, data) + epic.award_emoji.exists?(user_id: data['user_id'], name: data['name']) # rubocop: disable CodeReuse/ActiveRecord + end + + def track_next_page(data) + page_info = data.dig('award_emoji', 'page_info') + + if page_info['has_next_page'] + epic_iid = data['iid'] + award_emoji_next_page = page_info['next_page'] + + @next_pages << [epic_iid, award_emoji_next_page] + end end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/ee/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline.rb b/ee/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline.rb new file mode 100644 index 0000000000000000000000000000000000000000..5f8cf41a6f6d59bb826acde927f9568c95c729cd --- /dev/null +++ b/ee/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module EE + module BulkImports + module Groups + module Pipelines + class EpicsAwardEmojiPipeline + include ::BulkImports::Pipeline + + extractor ::BulkImports::Common::Extractors::GraphqlExtractor, + query: EE::BulkImports::Groups::Graphql::GetEpicsAwardEmojiQuery + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def transform(_, data) + nodes = data.dig('award_emoji', 'nodes') + + return if nodes.blank? + + track_next_page(data) + + data['award_emoji']['nodes'] = nodes.map do |node| + ::BulkImports::Common::Transformers::UserReferenceTransformer.new.transform(context, node) + end + + data + end + + def load(context, data) + return unless data + + epic = context.group.epics.find_by_iid(data['iid']) + + return unless epic + + raise NotAllowedError unless Ability.allowed?(context.current_user, :award_emoji, epic) + + data.dig('award_emoji', 'nodes').each do |award_emoji| + next if award_emoji_exists?(epic, award_emoji) + + epic.award_emoji.create!(award_emoji) + end + end + + def on_complete + pipeline_tracker = context.entity.trackers.create!( + pipeline_name: EpicAwardEmojiPipeline.name, + stage: context.tracker.stage + ) + + new_context = ::BulkImports::Pipeline::Context.new(pipeline_tracker) + new_context.extra = context.extra + + EpicAwardEmojiPipeline.new(new_context).run + end + + private + + def award_emoji_exists?(epic, data) + epic.award_emoji.exists?(user_id: data['user_id'], name: data['name']) # rubocop: disable CodeReuse/ActiveRecord + end + + def track_next_page(data) + page_info = data.dig('award_emoji', 'page_info') + + if page_info['has_next_page'] + epic_iid = data['iid'] + award_emoji_next_page = page_info['next_page'] + context.extra[:subrelation_next_pages] ||= [] + context.extra[:subrelation_next_pages] << [epic_iid, award_emoji_next_page] + end + end + end + end + end + end +end diff --git a/ee/lib/ee/bulk_imports/importers/group_importer.rb b/ee/lib/ee/bulk_imports/importers/group_importer.rb index ad05e52f9afa62d77e295047dfd0adf91f97d275..09ef3419c46af25ab8f1973493495fb2248cb5f5 100644 --- a/ee/lib/ee/bulk_imports/importers/group_importer.rb +++ b/ee/lib/ee/bulk_imports/importers/group_importer.rb @@ -12,9 +12,9 @@ module GroupImporter def pipelines super + [ EE::BulkImports::Groups::Pipelines::EpicsPipeline, - EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline, - EE::BulkImports::Groups::Pipelines::EpicEventsPipeline, - EE::BulkImports::Groups::Pipelines::IterationsPipeline + EE::BulkImports::Groups::Pipelines::EpicsAwardEmojiPipeline, + # EE::BulkImports::Groups::Pipelines::EpicEventsPipeline, + # EE::BulkImports::Groups::Pipelines::IterationsPipeline ] end end diff --git a/ee/spec/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader_spec.rb b/ee/spec/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader_spec.rb deleted file mode 100644 index 382a65a5f799eae48b51616ab262ac0ef4a0e9fc..0000000000000000000000000000000000000000 --- a/ee/spec/lib/ee/bulk_imports/groups/loaders/epic_award_emoji_loader_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe EE::BulkImports::Groups::Loaders::EpicAwardEmojiLoader do - describe '#load' do - let_it_be(:user) { create(:user) } - let_it_be(:group) { create(:group) } - let_it_be(:epic) { create(:epic, group: group, iid: 1) } - let_it_be(:bulk_import) { create(:bulk_import, user: user) } - let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) } - let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } - - let_it_be(:data) do - { - 'name' => 'banana', - 'user_id' => user.id - } - end - - before do - stub_licensed_features(epics: true) - context.extra[:epic_iid] = epic.iid - group.add_developer(user) - end - - context 'when emoji does not exist' do - it 'creates new emoji' do - expect { subject.load(context, data) }.to change(::AwardEmoji, :count).by(1) - - epic = group.epics.last - emoji = epic.award_emoji.first - - expect(emoji.name).to eq(data['name']) - expect(emoji.user).to eq(user) - end - end - - context 'when same emoji exists' do - it 'does not create a new emoji' do - epic.award_emoji.create!(data) - - expect { subject.load(context, data) }.not_to change(::AwardEmoji, :count) - end - end - - context 'when user is not allowed to award emoji' do - before do - allow(Ability).to receive(:allowed?).with(user, :award_emoji, epic).and_return(false) - end - - it 'raises NotAllowedError exception' do - expect { subject.load(context, data) }.to raise_error(described_class::NotAllowedError) - end - end - end -end diff --git a/ee/spec/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline_spec.rb b/ee/spec/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline_spec.rb similarity index 97% rename from ee/spec/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline_spec.rb rename to ee/spec/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline_spec.rb index b7327737ddb6272e1a7b407b38187323dc681786..b66adca4d8d4e43f898e9e83c0d2dd744f9081bb 100644 --- a/ee/spec/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline_spec.rb +++ b/ee/spec/lib/ee/bulk_imports/groups/pipelines/epics_award_emoji_pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline do +RSpec.describe EE::BulkImports::Groups::Pipelines::EpicsAwardEmojiPipeline do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } let_it_be(:epic) { create(:epic, group: group) } diff --git a/ee/spec/lib/ee/bulk_imports/importers/group_importer_spec.rb b/ee/spec/lib/ee/bulk_imports/importers/group_importer_spec.rb index 68aedb7830adce7fe3afb8a134ccb9238e53b3ce..7cf27d2bd00cc24f65298245ef49f1a9176c40ff 100644 --- a/ee/spec/lib/ee/bulk_imports/importers/group_importer_spec.rb +++ b/ee/spec/lib/ee/bulk_imports/importers/group_importer_spec.rb @@ -24,7 +24,7 @@ expect_to_run_pipeline BulkImports::Groups::Pipelines::LabelsPipeline, context: context expect_to_run_pipeline BulkImports::Groups::Pipelines::MilestonesPipeline, context: context expect_to_run_pipeline EE::BulkImports::Groups::Pipelines::EpicsPipeline, context: context - expect_to_run_pipeline EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline, context: context + expect_to_run_pipeline EE::BulkImports::Groups::Pipelines::EpicsAwardEmojiPipeline, context: context expect_to_run_pipeline EE::BulkImports::Groups::Pipelines::EpicEventsPipeline, context: context expect_to_run_pipeline EE::BulkImports::Groups::Pipelines::IterationsPipeline, context: context diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index ccc61ee787daf5685a649192106df934875510a3..434a816b827f9be32615fdd3a26b98bfc65d8537 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -31,10 +31,10 @@ def execute def pipelines [ BulkImports::Groups::Pipelines::GroupPipeline, - BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, - BulkImports::Groups::Pipelines::MembersPipeline, - BulkImports::Groups::Pipelines::LabelsPipeline, - BulkImports::Groups::Pipelines::MilestonesPipeline + # BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + # BulkImports::Groups::Pipelines::MembersPipeline, + # BulkImports::Groups::Pipelines::LabelsPipeline, + # BulkImports::Groups::Pipelines::MilestonesPipeline ] end end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index b756fba3bee5466684c0fb3d75d6f3d5661dbeee..4733c941420f2e9013ec0b1c97f14b50ad5de671 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -37,6 +37,10 @@ def run end end + run_pipeline_step(:on_complete) do + on_complete if respond_to?(:on_complete) + end + info(message: 'Pipeline finished') rescue MarkedAsFailedError skip!('Skipping pipeline due to failed entity') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 7bd55cce363f3dd5b580fac21079e6be1b07a68f..f2886b287fb976e58c1bccd2f79b5c5eabc52288 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -88,7 +88,7 @@ def self.measure(name) trans.observe("gitlab_#{name}_cpu_duration_seconds".to_sym, cpu_time) do docstring "Measure #{name}" buckets EXECUTION_MEASUREMENT_BUCKETS - with_feature "prometheus_metrics_measure_#{name}_cpu_duration" + # with_feature "prometheus_metrics_measure_#{name}_cpu_duration" end retval