diff --git a/ee/app/services/security/secret_detection/update_token_status_service.rb b/ee/app/services/security/secret_detection/update_token_status_service.rb index dcf020cda50555b3fa2ed73366c255fcf1d05aa9..1a01dadf6da88412d54ac1e5924c39886aced6c0 100644 --- a/ee/app/services/security/secret_detection/update_token_status_service.rb +++ b/ee/app/services/security/secret_detection/update_token_status_service.rb @@ -3,6 +3,8 @@ module Security module SecretDetection class UpdateTokenStatusService + include Gitlab::InternalEventsTracking + DEFAULT_BATCH_SIZE = 100 attr_reader :project @@ -13,27 +15,67 @@ def initialize(token_lookup_service = TokenLookupService.new) # For Vulnerabilities::Finding (default branch pipelines) def execute_for_vulnerability_pipeline(pipeline_id) + start_time = Time.current return unless setup_and_validate_pipeline(pipeline_id) relation = ::Vulnerabilities::Finding .report_type('secret_detection') .by_latest_pipeline(pipeline_id) + track_internal_event( + 'update_token_status_service_invoked', + project: @project, + additional_properties: { + label: 'vulnerability', # Type of pipeline that triggered token status processing: security or vulnerability + value: relation.count # Number of secret detection findings sent to UpdateTokenStatusService for processing + } + ) + relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :vulnerability) end + + execution_time = Time.current - start_time + track_internal_event( + 'token_status_processing_metrics', + project: @project, + additional_properties: { + label: 'vulnerability', # Type of pipeline that triggered token status processing: security or vulnerability + value: execution_time.to_f * 1000 # Time in ms it took for the UpdateTokenStatusService to complete + } + ) end # For ::Security::Finding (MR pipelines) def execute_for_security_pipeline(pipeline_id) + start_time = Time.current return unless setup_and_validate_pipeline(pipeline_id) return unless Feature.enabled?(:validity_checks_security_finding_status, @project) relation = @pipeline.security_findings.by_report_types(['secret_detection']) + track_internal_event( + 'update_token_status_service_invoked', + project: @project, + additional_properties: { + label: 'security', # Type of pipeline that triggered token status processing: security or vulnerability + value: relation.count # Number of secret detection findings sent to UpdateTokenStatusService for processing + } + ) + relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :security) end + + execution_time = Time.current - start_time + track_internal_event( + 'token_status_processing_metrics', + project: @project, + additional_properties: { + label: 'security', # Type of pipeline that triggered token status processing: security or vulnerability + value: execution_time.to_f * 1000 # Time in ms it took for the UpdateTokenStatusService to complete + } + ) end # Single Vulnerabilities::Finding @@ -139,9 +181,27 @@ def sync_elasticsearch_for(findings, finding_type) def merge_token_status_into_attributes(tokens_by_raw, attrs_by_raw) tokens_by_raw.each do |raw_token, token| + token_type = if token.is_a?(Hash) && token[:type] + token[:type] + elsif token.respond_to?(:class) + token.class.name.demodulize + else + 'Unknown' + end + attrs_by_raw[raw_token]&.each do |finding_token_status_attr| - finding_token_status_attr[:status] = token_status(token) + cur_token_status = token_status(token) + finding_token_status_attr[:status] = cur_token_status finding_token_status_attr[:updated_at] = Time.current + + track_internal_event( + 'finding_token_status_detected', + project: @project, + additional_properties: { + label: token_type, + property: cur_token_status + } + ) end end end diff --git a/ee/config/events/finding_token_status_detected.yml b/ee/config/events/finding_token_status_detected.yml new file mode 100644 index 0000000000000000000000000000000000000000..6ca126ec54728bd8520bb742e968a7cc4739d223 --- /dev/null +++ b/ee/config/events/finding_token_status_detected.yml @@ -0,0 +1,20 @@ +--- +description: Tracks the types of tokens detected during UpdateTokenStatusService +internal_events: true +status: active +action: finding_token_status_detected +identifiers: + - project + - namespace +additional_properties: + label: + description: Status of finding token (one of active, inactive, unknown) + property: + description: Type of token detected +product_group: secret_detection +product_categories: + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 +tiers: + - ultimate diff --git a/ee/config/events/token_status_processing_metrics.yml b/ee/config/events/token_status_processing_metrics.yml new file mode 100644 index 0000000000000000000000000000000000000000..5b621cd8b8c86d1bf2635dc76fd3bd4eeb27b7cb --- /dev/null +++ b/ee/config/events/token_status_processing_metrics.yml @@ -0,0 +1,20 @@ +--- +description: The length of time that UpdateTokenStatusService takes to complete. +internal_events: true +status: active +action: token_status_processing_metrics +identifiers: + - project + - namespace +additional_properties: + label: + description: Type of pipeline that triggered token status processing (security or vulnerability) + value: + description: Time in milliseconds it took for the UpdateTokenStatusService to complete +product_group: secret_detection +product_categories: + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 +tiers: + - ultimate diff --git a/ee/config/events/update_token_status_service_invoked.yml b/ee/config/events/update_token_status_service_invoked.yml new file mode 100644 index 0000000000000000000000000000000000000000..305f8ad985d7730bdd9d64d2aa18ff1d4bd4307d --- /dev/null +++ b/ee/config/events/update_token_status_service_invoked.yml @@ -0,0 +1,20 @@ +--- +description: Secret detection findings processed by UpdateTokenStatusService to verify token validity status +internal_events: true +status: active +action: update_token_status_service_invoked +identifiers: + - project + - namespace +additional_properties: + label: + description: Type of pipeline that triggered token status processing (security or vulnerability) + value: + description: Number of secret detection findings sent to UpdateTokenStatusService for processing +product_group: secret_detection +product_categories: + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 +tiers: + - ultimate diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb index 4877ce870292758e3bff1567ddb559aa62dc0a62..6a4ba7343d0b6e118aa22ff9851c02867311a6cf 100644 --- a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb +++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb @@ -3,13 +3,17 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountProjectsWithValidityChecksEnabledMetric, feature_category: :service_ping do - let(:expected_value) { 3 } + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do + let(:expected_value) { 3 } + let(:expected_query) do + "SELECT COUNT(\"project_security_settings\".\"project_id\") FROM \"project_security_settings\" " \ + "WHERE \"project_security_settings\".\"validity_checks_enabled\" = TRUE" + end - before do - projects = create_list(:project, 3) - projects.each { |project| project.security_setting.update!(validity_checks_enabled: true) } - create(:project) + before do + projects = create_list(:project, 3) + projects.each { |project| project.security_setting.update!(validity_checks_enabled: true) } + create(:project) + end end - - it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } end