From 7a19f4db6145750c11116249513b99e0b14e9475 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Tue, 12 Dec 2023 09:49:33 +0100 Subject: [PATCH 1/4] Send CI runner usage email EE: true --- ee/app/mailers/ee/notify.rb | 1 + .../emails/ci_runner_usage_by_project.rb | 19 ++++++ .../ci/runners/send_usage_csv_service.rb | 37 ++++++++++++ ...unner_usage_by_project_csv_email.html.haml | 5 ++ ...runner_usage_by_project_csv_email.text.erb | 5 ++ ee/spec/mailers/notify_spec.rb | 21 +++++++ .../ci/runners/send_usage_csv_service_spec.rb | 60 +++++++++++++++++++ locale/gitlab.pot | 12 ++++ 8 files changed, 160 insertions(+) create mode 100644 ee/app/mailers/emails/ci_runner_usage_by_project.rb create mode 100644 ee/app/services/ci/runners/send_usage_csv_service.rb create mode 100644 ee/app/views/notify/runner_usage_by_project_csv_email.html.haml create mode 100644 ee/app/views/notify/runner_usage_by_project_csv_email.text.erb create mode 100644 ee/spec/services/ci/runners/send_usage_csv_service_spec.rb diff --git a/ee/app/mailers/ee/notify.rb b/ee/app/mailers/ee/notify.rb index 6dec6659f733a9..edf98be4c7eb49 100644 --- a/ee/app/mailers/ee/notify.rb +++ b/ee/app/mailers/ee/notify.rb @@ -22,6 +22,7 @@ module Notify include ::Emails::ComplianceViolations include ::Emails::ComplianceStandards include ::Emails::Okr + include ::Emails::CiRunnerUsageByProject end attr_reader :group diff --git a/ee/app/mailers/emails/ci_runner_usage_by_project.rb b/ee/app/mailers/emails/ci_runner_usage_by_project.rb new file mode 100644 index 00000000000000..af9eee56972a27 --- /dev/null +++ b/ee/app/mailers/emails/ci_runner_usage_by_project.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Emails + module CiRunnerUsageByProject + def runner_usage_by_project_csv_email(user, csv_data, export_status) + @count = export_status.fetch(:rows_expected) + @written_count = export_status.fetch(:rows_written) + @truncated = export_status.fetch(:truncated) + @size_limit = ActiveSupport::NumberHelper + .number_to_human_size(ExportCsv::BaseService::TARGET_FILESIZE) + + filename = "ci_runner_usage_report_#{Date.current.iso8601}.csv" + attachments[filename] = { content: csv_data, mime_type: 'text/csv' } + email_with_layout( + to: user.notification_email_or_default, + subject: subject('Exported CI Runner usage')) + end + end +end diff --git a/ee/app/services/ci/runners/send_usage_csv_service.rb b/ee/app/services/ci/runners/send_usage_csv_service.rb new file mode 100644 index 00000000000000..8554f5f605dbd1 --- /dev/null +++ b/ee/app/services/ci/runners/send_usage_csv_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Ci + module Runners + # Sends a CSV report containing the runner usage for a given period + # (based on ClickHouse's ci_used_minutes_mv view) + # + class SendUsageCsvService + # @param [User] current_user The user performing the reporting + # @param [Symbol] runner_type The type of runners to report on. Defaults to nil, reporting on all runner types + # @param [DateTime] from_time The start date of the period to examine. Defaults to start of last full month + # @param [DateTime] to_time The end date of the period to examine. Defaults to end of month + def initialize(current_user:, runner_type: nil, from_time: nil, to_time: nil) + @current_user = current_user + @runner_type = runner_type + @from_time = from_time + @to_time = to_time + end + + def execute + result = GenerateUsageCsvService.new( + current_user: @current_user, + runner_type: @runner_type, + from_time: @from_time, + to_time: @to_time + ).execute + + return result if result.error? + + Notify.runner_usage_by_project_csv_email(@current_user, result.payload[:csv_data], result.payload[:status]) + .deliver_now + + ServiceResponse.success(payload: result.payload.slice(:status)) + end + end + end +end diff --git a/ee/app/views/notify/runner_usage_by_project_csv_email.html.haml b/ee/app/views/notify/runner_usage_by_project_csv_email.html.haml new file mode 100644 index 00000000000000..d616257ba6810d --- /dev/null +++ b/ee/app/views/notify/runner_usage_by_project_csv_email.html.haml @@ -0,0 +1,5 @@ +%p{ style: 'font-size:18px; text-align:center; line-height:30px;' } + = _('Your CI runner usage CSV export containing the top %{exported_objects} has been added to this email as an attachment.').html_safe % { exported_objects: pluralize(@written_count, 'project') } + - if @truncated + %p + = _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} projects have been included.') % { written_count: @written_count, count: @count, size_limit: @size_limit } diff --git a/ee/app/views/notify/runner_usage_by_project_csv_email.text.erb b/ee/app/views/notify/runner_usage_by_project_csv_email.text.erb new file mode 100644 index 00000000000000..179fba132942b0 --- /dev/null +++ b/ee/app/views/notify/runner_usage_by_project_csv_email.text.erb @@ -0,0 +1,5 @@ +<%= _('Your CSV export of the top %{exported_objects} has been added to this email as an attachment.') % { exported_objects: pluralize(@written_count, 'project') } %> + +<% if @truncated %> + <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{total_count} projects have been included.') % { written_count: @written_count, total_count: @count, size_limit: @size_limit } %> +<% end %> diff --git a/ee/spec/mailers/notify_spec.rb b/ee/spec/mailers/notify_spec.rb index 5b8256b25f2621..c2e169b9a4e0fa 100644 --- a/ee/spec/mailers/notify_spec.rb +++ b/ee/spec/mailers/notify_spec.rb @@ -459,6 +459,27 @@ end end + describe 'CI runner usage report is sent', feature_category: :fleet_visibility do + let(:export_status) do + { + rows_expected: 500, + rows_written: 480, + truncated: false + } + end + + subject { described_class.runner_usage_by_project_csv_email(user, 'CSV content', export_status) } + + it_behaves_like 'an email sent from GitLab' + it_behaves_like 'it should not have Gmail Actions links' + it_behaves_like "a user cannot unsubscribe through footer link" + + it 'has the correct subject and body', :aggregate_failures do + is_expected.to have_subject('Exported CI Runner usage') + is_expected.to have_body_text('480 projects') + end + end + def expect_sender(user) sender = subject.header[:from].addrs[0] expect(sender.display_name).to eq("#{user.name} (@#{user.username})") diff --git a/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb b/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb new file mode 100644 index 00000000000000..b4021ac7c7f6f1 --- /dev/null +++ b/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Runners::SendUsageCsvService, feature_category: :fleet_visibility do + let_it_be(:current_user) { create(:admin) } + + let(:service) do + described_class.new( + current_user: current_user, runner_type: :instance_type, from_time: 1.month.ago, to_time: DateTime.now) + end + + subject(:response) { service.execute } + + context 'when export is successful', :freeze_time do + include ClickHouseHelpers + + let_it_be(:instance_runner) { create(:ci_runner, :instance, :with_runner_manager) } + + before do + stub_licensed_features(runner_performance_insights: true) + + started_at = created_at = DateTime.current + project = build(:project) + build = build(:ci_build, :success, created_at: created_at, queued_at: created_at, started_at: started_at, + finished_at: started_at + 10.minutes, project: project, runner: instance_runner, + runner_manager: instance_runner.runner_managers.first) + insert_ci_builds_to_click_house([build]) + end + + it '', :enable_admin_mode, :click_house do + expect_next_instance_of(Ci::Runners::GenerateUsageCsvService) do |service| + expect(service).to receive(:execute).and_call_original + end + + expected_status = { rows_expected: 1, rows_written: 1, truncated: false } + expect(Notify).to receive(:runner_usage_by_project_csv_email) + .with(current_user, anything, expected_status) + .and_return(instance_double(ActionMailer::MessageDelivery, deliver_now: true)) + + expect(response).to be_success + expect(response.payload).to eq({ status: expected_status }) + end + end + + context 'when report fails to be generated' do + before do + allow_next_instance_of(Ci::Runners::GenerateUsageCsvService) do |service| + allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'Generation failed')) + end + end + + it 'returns error from GenerateUsageCsvService' do + expect(Notify).not_to receive(:runner_usage_by_project_csv_email) + + expect(response).to be_error + expect(response.message).to eq('Generation failed') + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 77f2418edab738..eb8aaacddec4de 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -49539,9 +49539,15 @@ msgstr "" msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}." msgstr "" +msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} projects have been included." +msgstr "" + msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{total_count} %{object_type} have been included. Consider re-exporting with a narrower selection of %{object_type}." msgstr "" +msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{total_count} projects have been included." +msgstr "" + msgid "This block is self-referential" msgstr "" @@ -56338,6 +56344,9 @@ msgstr "" msgid "Your Activity" msgstr "" +msgid "Your CI runner usage CSV export containing the top %{exported_objects} has been added to this email as an attachment." +msgstr "" + msgid "Your CI/CD configuration syntax is invalid. Select the Validate tab for more details." msgstr "" @@ -56350,6 +56359,9 @@ msgstr "" msgid "Your CSV export of %{exported_objects} from project %{project_name} (%{project_url}) has been added to this email as an attachment." msgstr "" +msgid "Your CSV export of the top %{exported_objects} has been added to this email as an attachment." +msgstr "" + msgid "Your CSV export request has succeeded. The result will be emailed to %{email}." msgstr "" -- GitLab From ffdaaa2477c69c05c0ab4934b32cd232dfd968d2 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Tue, 12 Dec 2023 11:27:44 +0100 Subject: [PATCH 2/4] Add CI runner usage export worker EE: true --- config/sidekiq_queues.yml | 2 + ee/app/workers/all_queues.yml | 9 ++++ .../ci/runners/export_usage_csv_worker.rb | 28 ++++++++++ .../runners/export_usage_csv_worker_spec.rb | 54 +++++++++++++++++++ spec/workers/every_sidekiq_worker_spec.rb | 3 +- 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 ee/app/workers/ci/runners/export_usage_csv_worker.rb create mode 100644 ee/spec/workers/ci/runners/export_usage_csv_worker_spec.rb diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index d10dd1528ea5a6..7af1f3422e8507 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -171,6 +171,8 @@ - 1 - - ci_parse_secure_file_metadata - 1 +- - ci_runners_export_usage_csv + - 1 - - ci_runners_process_runner_version_update - 1 - - ci_unlock_pipelines_in_queue diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 88723569313b2d..3ccd52194f5741 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1020,6 +1020,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: ci_runners_export_usage_csv + :worker_name: Ci::Runners::ExportUsageCsvWorker + :feature_category: :fleet_visibility + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: false + :tags: [] - :name: ci_upstream_projects_subscriptions_cleanup :worker_name: Ci::UpstreamProjectsSubscriptionsCleanupWorker :feature_category: :continuous_integration diff --git a/ee/app/workers/ci/runners/export_usage_csv_worker.rb b/ee/app/workers/ci/runners/export_usage_csv_worker.rb new file mode 100644 index 00000000000000..cfef67f2abd703 --- /dev/null +++ b/ee/app/workers/ci/runners/export_usage_csv_worker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Ci + module Runners + # rubocop: disable Scalability/IdempotentWorker -- this worker sends out emails + class ExportUsageCsvWorker + include ApplicationWorker + + data_consistency :delayed + + sidekiq_options retry: 3 + + feature_category :fleet_visibility + worker_resource_boundary :cpu + loggable_arguments 0, 1 + + def perform(current_user_id, runner_type) + user = User.find(current_user_id) + + result = Ci::Runners::SendUsageCsvService.new(current_user: user, runner_type: runner_type).execute + log_extra_metadata_on_done(:status, result.status) + log_extra_metadata_on_done(:message, result.message) if result.message + log_extra_metadata_on_done(:csv_status, result.payload[:status]) if result.payload[:status] + end + end + # rubocop: enable Scalability/IdempotentWorker + end +end diff --git a/ee/spec/workers/ci/runners/export_usage_csv_worker_spec.rb b/ee/spec/workers/ci/runners/export_usage_csv_worker_spec.rb new file mode 100644 index 00000000000000..224ca815cf53ab --- /dev/null +++ b/ee/spec/workers/ci/runners/export_usage_csv_worker_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Runners::ExportUsageCsvWorker, :click_house, :enable_admin_mode, feature_category: :fleet_visibility do + let_it_be(:admin) { create(:admin) } + let_it_be(:user) { create(:user) } + + let(:worker) { described_class.new } + + describe '#perform' do + subject(:perform) { worker.perform(current_user.id, runner_type) } + + let(:current_user) { admin } + let(:runner_type) { 1 } + + before do + stub_licensed_features(runner_performance_insights: true) + end + + it 'delegates to Ci::Runners::SendUsageCsvService' do + expect_next_instance_of(Ci::Runners::SendUsageCsvService, + { current_user: current_user, runner_type: runner_type }) do |service| + expect(service).to receive(:execute).and_call_original + end + + perform + + expect(worker.logging_extras).to eq({ + "extra.ci_runners_export_usage_csv_worker.status" => :success, + "extra.ci_runners_export_usage_csv_worker.csv_status" => { + rows_expected: 0, rows_written: 0, truncated: false + } + }) + end + + context 'when runner_performance_insights feature is not available' do + before do + stub_licensed_features(runner_performance_insights: false) + end + + let(:runner_type) { nil } + + it 'returns error' do + perform + + expect(worker.logging_extras).to eq({ + "extra.ci_runners_export_usage_csv_worker.status" => :error, + "extra.ci_runners_export_usage_csv_worker.message" => 'Insufficient permissions to generate export' + }) + end + end + end +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 136fa6b4aa0f6c..0b7c5b18f5a3b5 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -475,7 +475,8 @@ 'Zoekt::IndexerWorker' => 2, 'Issuable::RelatedLinksCreateWorker' => 3, 'BulkImports::RelationBatchExportWorker' => 6, - 'BulkImports::RelationExportWorker' => 6 + 'BulkImports::RelationExportWorker' => 6, + 'Ci::Runners::ExportUsageCsvWorker' => 3 }.merge(extra_retry_exceptions) end -- GitLab From 31a8a75e0980b89860ca81a73cb1c65fe3344182 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Mon, 11 Dec 2023 19:45:50 +0100 Subject: [PATCH 3/4] Add CI runner usage export mutation EE: true --- doc/api/graphql/reference/index.md | 22 ++++++ ee/app/graphql/ee/types/mutation_type.rb | 1 + .../mutations/ci/runners/export_usage.rb | 30 ++++++++ .../mutations/ci/runners/export_usage_spec.rb | 72 +++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 ee/app/graphql/mutations/ci/runners/export_usage.rb create mode 100644 ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index fbba2f2474b097..787a5a0f58388d 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -6731,6 +6731,28 @@ Input type: `RunnerUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `runner` | [`CiRunner`](#cirunner) | Runner after mutation. | +### `Mutation.runnersExportUsage` + +WARNING: +**Introduced** in 16.7. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `RunnersExportUsageInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `type` | [`CiRunnerType`](#cirunnertype) | Scope of the runners to include in the report. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.runnersRegistrationTokenReset` Input type: `RunnersRegistrationTokenResetInput` diff --git a/ee/app/graphql/ee/types/mutation_type.rb b/ee/app/graphql/ee/types/mutation_type.rb index 62cddaac3583c2..c686b7e0ea8b45 100644 --- a/ee/app/graphql/ee/types/mutation_type.rb +++ b/ee/app/graphql/ee/types/mutation_type.rb @@ -110,6 +110,7 @@ module MutationType mount_mutation ::Mutations::AuditEvents::ExternalAuditEventDestinations::Destroy mount_mutation ::Mutations::AuditEvents::ExternalAuditEventDestinations::Update mount_mutation ::Mutations::Ci::NamespaceCiCdSettingsUpdate + mount_mutation ::Mutations::Ci::Runners::ExportUsage, alpha: { milestone: '16.7' } mount_mutation ::Mutations::RemoteDevelopment::Workspaces::Create, alpha: { milestone: '16.0' } mount_mutation ::Mutations::RemoteDevelopment::Workspaces::Update, alpha: { milestone: '16.0' } mount_mutation ::Mutations::AuditEvents::Streaming::Headers::Destroy diff --git a/ee/app/graphql/mutations/ci/runners/export_usage.rb b/ee/app/graphql/mutations/ci/runners/export_usage.rb new file mode 100644 index 00000000000000..7ddd6be160b039 --- /dev/null +++ b/ee/app/graphql/mutations/ci/runners/export_usage.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Mutations + module Ci + module Runners + class ExportUsage < BaseMutation + graphql_name 'RunnersExportUsage' + + argument :type, ::Types::Ci::RunnerTypeEnum, + required: false, + description: 'Scope of the runners to include in the report.' + + def ready?(**args) + raise_resource_not_available_error! unless Ability.allowed?(current_user, :read_runner_usage) + + super + end + + def resolve(type: nil) + type = ::Ci::Runner.runner_types[type] + ::Ci::Runners::ExportUsageCsvWorker.perform_async(current_user.id, type) # rubocop: disable CodeReuse/Worker -- this worker sends out emails + + { + errors: [] + } + end + end + end + end +end diff --git a/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb b/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb new file mode 100644 index 00000000000000..da4d051b56d8d2 --- /dev/null +++ b/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'RunnersExportUsage', :click_house, :enable_admin_mode, :sidekiq_inline, :freeze_time, + feature_category: :fleet_visibility do + include GraphqlHelpers + include ClickHouseHelpers + + let_it_be(:current_user) { create(:admin) } + let_it_be(:instance_runner) { create(:ci_runner, :instance) } + let_it_be(:group_runner) { create(:ci_runner, :group) } + let_it_be(:start_time) { 1.month.ago } + let_it_be(:build1) do + build(:ci_build, :success, created_at: start_time, queued_at: start_time, started_at: start_time, + finished_at: start_time + 10.minutes, runner: instance_runner) + end + + let_it_be(:build2) do + build(:ci_build, :success, created_at: start_time, queued_at: start_time, started_at: start_time, + finished_at: start_time + 10.minutes, runner: group_runner) + end + + let(:runner_type) { 'GROUP_TYPE' } + let(:mutation) do + graphql_mutation(:runners_export_usage, type: runner_type) do + <<~QL + errors + QL + end + end + + let(:mutation_response) { graphql_mutation_response(:runners_export_usage) } + + subject(:post_response) { post_graphql_mutation(mutation, current_user: current_user) } + + before do + stub_licensed_features(runner_performance_insights: true) + travel_to DateTime.new(2023, 12, 14) + + insert_ci_builds_to_click_house([build1, build2]) + end + + it 'sends email with report' do + expect(Notify).to receive(:runner_usage_by_project_csv_email) + .with(current_user, anything, anything) do |_user, csv_data, status| + expect(status[:rows_written]).to eq 1 + + parsed_csv = CSV.parse(csv_data, headers: true) + expect(parsed_csv[0]['Project ID']).to eq build2.project.id.to_s + + instance_double(ActionMailer::MessageDelivery, deliver_now: true) + end + + post_response + expect_graphql_errors_to_be_empty + end + + context 'when feature is not available' do + before do + stub_licensed_features(runner_performance_insights: false) + end + + it 'returns an error' do + post_response + + expect(graphql_errors) + .to include(a_hash_including('message' => "The resource that you are attempting to access does " \ + "not exist or you don't have permission to perform this action")) + end + end +end -- GitLab From 8a623cbaf8e8156737c63ff8f14939875a83661d Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Fri, 15 Dec 2023 12:38:09 +0100 Subject: [PATCH 4/4] Address MR review comments --- .../emails/ci_runner_usage_by_project.rb | 6 +-- .../ci/runners/send_usage_csv_service.rb | 11 ++-- ee/spec/mailers/notify_spec.rb | 6 ++- .../mutations/ci/runners/export_usage_spec.rb | 4 +- .../ci/runners/send_usage_csv_service_spec.rb | 54 +++++++++---------- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/ee/app/mailers/emails/ci_runner_usage_by_project.rb b/ee/app/mailers/emails/ci_runner_usage_by_project.rb index af9eee56972a27..5576cbf1e08b76 100644 --- a/ee/app/mailers/emails/ci_runner_usage_by_project.rb +++ b/ee/app/mailers/emails/ci_runner_usage_by_project.rb @@ -2,18 +2,18 @@ module Emails module CiRunnerUsageByProject - def runner_usage_by_project_csv_email(user, csv_data, export_status) + def runner_usage_by_project_csv_email(user, from_date, to_date, csv_data, export_status) @count = export_status.fetch(:rows_expected) @written_count = export_status.fetch(:rows_written) @truncated = export_status.fetch(:truncated) @size_limit = ActiveSupport::NumberHelper .number_to_human_size(ExportCsv::BaseService::TARGET_FILESIZE) - filename = "ci_runner_usage_report_#{Date.current.iso8601}.csv" + filename = "ci_runner_usage_report_#{from_date.iso8601}_#{to_date.iso8601}.csv" attachments[filename] = { content: csv_data, mime_type: 'text/csv' } email_with_layout( to: user.notification_email_or_default, - subject: subject('Exported CI Runner usage')) + subject: subject("Exported CI Runner usage (#{from_date.strftime('%F')} - #{to_date.strftime('%F')})")) end end end diff --git a/ee/app/services/ci/runners/send_usage_csv_service.rb b/ee/app/services/ci/runners/send_usage_csv_service.rb index 8554f5f605dbd1..21697314e56229 100644 --- a/ee/app/services/ci/runners/send_usage_csv_service.rb +++ b/ee/app/services/ci/runners/send_usage_csv_service.rb @@ -18,17 +18,20 @@ def initialize(current_user:, runner_type: nil, from_time: nil, to_time: nil) end def execute - result = GenerateUsageCsvService.new( + generate_csv_service = GenerateUsageCsvService.new( current_user: @current_user, runner_type: @runner_type, from_time: @from_time, to_time: @to_time - ).execute + ) + result = generate_csv_service.execute return result if result.error? - Notify.runner_usage_by_project_csv_email(@current_user, result.payload[:csv_data], result.payload[:status]) - .deliver_now + Notify.runner_usage_by_project_csv_email( + @current_user, generate_csv_service.from_time, generate_csv_service.to_time, + result.payload[:csv_data], result.payload[:status] + ).deliver_now ServiceResponse.success(payload: result.payload.slice(:status)) end diff --git a/ee/spec/mailers/notify_spec.rb b/ee/spec/mailers/notify_spec.rb index c2e169b9a4e0fa..4ecf16fb2d596c 100644 --- a/ee/spec/mailers/notify_spec.rb +++ b/ee/spec/mailers/notify_spec.rb @@ -460,6 +460,8 @@ end describe 'CI runner usage report is sent', feature_category: :fleet_visibility do + let(:from_date) { DateTime.new(2023, 1, 1) } + let(:to_date) { DateTime.new(2023, 1, 31) } let(:export_status) do { rows_expected: 500, @@ -468,14 +470,14 @@ } end - subject { described_class.runner_usage_by_project_csv_email(user, 'CSV content', export_status) } + subject { described_class.runner_usage_by_project_csv_email(user, from_date, to_date, 'CSV content', export_status) } it_behaves_like 'an email sent from GitLab' it_behaves_like 'it should not have Gmail Actions links' it_behaves_like "a user cannot unsubscribe through footer link" it 'has the correct subject and body', :aggregate_failures do - is_expected.to have_subject('Exported CI Runner usage') + is_expected.to have_subject('Exported CI Runner usage (2023-01-01 - 2023-01-31)') is_expected.to have_body_text('480 projects') end end diff --git a/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb b/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb index da4d051b56d8d2..85b957e5205f63 100644 --- a/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb +++ b/ee/spec/requests/api/graphql/mutations/ci/runners/export_usage_spec.rb @@ -43,7 +43,9 @@ it 'sends email with report' do expect(Notify).to receive(:runner_usage_by_project_csv_email) - .with(current_user, anything, anything) do |_user, csv_data, status| + .with( + current_user, DateTime.new(2023, 11, 1), DateTime.new(2023, 11, 1).end_of_month, anything, anything + ) do |_user, csv_data, status| expect(status[:rows_written]).to eq 1 parsed_csv = CSV.parse(csv_data, headers: true) diff --git a/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb b/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb index b4021ac7c7f6f1..3b016e34c8b034 100644 --- a/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb +++ b/ee/spec/services/ci/runners/send_usage_csv_service_spec.rb @@ -2,45 +2,43 @@ require 'spec_helper' -RSpec.describe Ci::Runners::SendUsageCsvService, feature_category: :fleet_visibility do +RSpec.describe Ci::Runners::SendUsageCsvService, :enable_admin_mode, :click_house, :freeze_time, + feature_category: :fleet_visibility do + include ClickHouseHelpers + let_it_be(:current_user) { create(:admin) } + let_it_be(:instance_runner) { create(:ci_runner, :instance, :with_runner_manager) } + let(:from_time) { 1.month.ago } + let(:to_time) { DateTime.current } let(:service) do - described_class.new( - current_user: current_user, runner_type: :instance_type, from_time: 1.month.ago, to_time: DateTime.now) + described_class.new(current_user: current_user, runner_type: :instance_type, from_time: from_time, to_time: to_time) end subject(:response) { service.execute } - context 'when export is successful', :freeze_time do - include ClickHouseHelpers - - let_it_be(:instance_runner) { create(:ci_runner, :instance, :with_runner_manager) } + before do + stub_licensed_features(runner_performance_insights: true) + started_at = created_at = 1.hour.ago + project = build(:project) + build = build_stubbed(:ci_build, :success, created_at: created_at, queued_at: created_at, started_at: started_at, + finished_at: started_at + 10.minutes, project: project, runner: instance_runner, + runner_manager: instance_runner.runner_managers.first) + insert_ci_builds_to_click_house([build]) + end - before do - stub_licensed_features(runner_performance_insights: true) - - started_at = created_at = DateTime.current - project = build(:project) - build = build(:ci_build, :success, created_at: created_at, queued_at: created_at, started_at: started_at, - finished_at: started_at + 10.minutes, project: project, runner: instance_runner, - runner_manager: instance_runner.runner_managers.first) - insert_ci_builds_to_click_house([build]) + it 'sends the csv by email' do + expect_next_instance_of(Ci::Runners::GenerateUsageCsvService) do |service| + expect(service).to receive(:execute).and_call_original end - it '', :enable_admin_mode, :click_house do - expect_next_instance_of(Ci::Runners::GenerateUsageCsvService) do |service| - expect(service).to receive(:execute).and_call_original - end - - expected_status = { rows_expected: 1, rows_written: 1, truncated: false } - expect(Notify).to receive(:runner_usage_by_project_csv_email) - .with(current_user, anything, expected_status) - .and_return(instance_double(ActionMailer::MessageDelivery, deliver_now: true)) + expected_status = { rows_expected: 1, rows_written: 1, truncated: false } + expect(Notify).to receive(:runner_usage_by_project_csv_email) + .with(current_user, from_time, to_time, anything, expected_status) + .and_return(instance_double(ActionMailer::MessageDelivery, deliver_now: true)) - expect(response).to be_success - expect(response.payload).to eq({ status: expected_status }) - end + expect(response).to be_success + expect(response.payload).to eq({ status: expected_status }) end context 'when report fails to be generated' do -- GitLab