From 8c30d134d7b71e5eecac05f3bf3e56a62625432f Mon Sep 17 00:00:00 2001 From: Radu Birsan Date: Thu, 11 Sep 2025 02:20:38 -0400 Subject: [PATCH 1/5] Track data from UpdateTokenStatusService Tracks the number of tokens sent to UpdateTokenStatusService, the type of token and its corresponding status found by UpdateTokenStatusService, and how long it takes UpdateTokenStatusService to complete Changelog: added EE: true --- .../update_token_status_service.rb | 18 +++++++++++++++++ .../update_token_status_service_invoked.yml | 20 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 ee/config/events/update_token_status_service_invoked.yml diff --git a/ee/app/services/security/secret_detection/update_token_status_service.rb b/ee/app/services/security/secret_detection/update_token_status_service.rb index dcf020cda50555..16c6daab95ce30 100644 --- a/ee/app/services/security/secret_detection/update_token_status_service.rb +++ b/ee/app/services/security/secret_detection/update_token_status_service.rb @@ -19,6 +19,15 @@ def execute_for_vulnerability_pipeline(pipeline_id) .report_type('secret_detection') .by_latest_pipeline(pipeline_id) + track_internal_event( + 'update_token_status_service_invoked', + project: @project, + additional_properties: { + label: 'vulnerability', # Type of pipeline that triggered token status processing: security or vulnerability + value: relation.count # Number of secret detection findings sent to UpdateTokenStatusService for processing + } + ) + relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :vulnerability) end @@ -31,6 +40,15 @@ def execute_for_security_pipeline(pipeline_id) relation = @pipeline.security_findings.by_report_types(['secret_detection']) + track_internal_event( + 'update_token_status_service_invoked', + project: @project, + additional_properties: { + label: 'security', # Type of pipeline that triggered token status processing: security or vulnerability + value: relation.count # Number of secret detection findings sent to UpdateTokenStatusService for processing + } + ) + relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :security) end diff --git a/ee/config/events/update_token_status_service_invoked.yml b/ee/config/events/update_token_status_service_invoked.yml new file mode 100644 index 00000000000000..39aa5d0bb7cd6b --- /dev/null +++ b/ee/config/events/update_token_status_service_invoked.yml @@ -0,0 +1,20 @@ +--- +description: Secret detection findings processed by UpdateTokenStatusService to verify token validity status +internal_events: true +status: active +action: update_token_status_service_invoked +identifiers: +- project +- namespace +additional_properties: + label: + description: Type of pipeline that triggered token status processing (security or vulnerability) + value: + description: Number of secret detection findings sent to UpdateTokenStatusService for processing +product_group: secret_detection +product_categories: +- secret_detection +milestone: '18.4' +introduced_by_url: TODO +tiers: +- ultimate -- GitLab From 19bc4d9bf2a5753242809e8e430d200d74bbe823 Mon Sep 17 00:00:00 2001 From: Radu Birsan Date: Thu, 11 Sep 2025 02:38:46 -0400 Subject: [PATCH 2/5] Track time it takes to run UpdateTokenStatusService --- .../update_token_status_service.rb | 22 +++++++++++++++++++ .../token_status_processing_metrics.yml | 20 +++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 ee/config/events/token_status_processing_metrics.yml diff --git a/ee/app/services/security/secret_detection/update_token_status_service.rb b/ee/app/services/security/secret_detection/update_token_status_service.rb index 16c6daab95ce30..28bdf3b6248510 100644 --- a/ee/app/services/security/secret_detection/update_token_status_service.rb +++ b/ee/app/services/security/secret_detection/update_token_status_service.rb @@ -13,6 +13,7 @@ def initialize(token_lookup_service = TokenLookupService.new) # For Vulnerabilities::Finding (default branch pipelines) def execute_for_vulnerability_pipeline(pipeline_id) + start_time = Time.current return unless setup_and_validate_pipeline(pipeline_id) relation = ::Vulnerabilities::Finding @@ -31,10 +32,21 @@ def execute_for_vulnerability_pipeline(pipeline_id) relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :vulnerability) end + + execution_time = Time.current - start_time + track_internal_event( + 'token_status_processing_metrics', + project: @project, + additional_properties: { + label: 'vulnerability', # Type of pipeline that triggered token status processing: security or vulnerability + value: execution_time.to_f * 1000 # Time in ms it took for the UpdateTokenStatusService to complete + } + ) end # For ::Security::Finding (MR pipelines) def execute_for_security_pipeline(pipeline_id) + start_time = Time.current return unless setup_and_validate_pipeline(pipeline_id) return unless Feature.enabled?(:validity_checks_security_finding_status, @project) @@ -52,6 +64,16 @@ def execute_for_security_pipeline(pipeline_id) relation.each_batch(of: DEFAULT_BATCH_SIZE) do |batch| process_findings_batch(batch, :security) end + + execution_time = Time.current - start_time + track_internal_event( + 'token_status_processing_metrics', + project: @project, + additional_properties: { + label: 'security', # Type of pipeline that triggered token status processing: security or vulnerability + value: execution_time.to_f * 1000 # Time in ms it took for the UpdateTokenStatusService to complete + } + ) end # Single Vulnerabilities::Finding diff --git a/ee/config/events/token_status_processing_metrics.yml b/ee/config/events/token_status_processing_metrics.yml new file mode 100644 index 00000000000000..19d3faa68a2adb --- /dev/null +++ b/ee/config/events/token_status_processing_metrics.yml @@ -0,0 +1,20 @@ +--- +description: The length of time that UpdateTokenStatusService takes to complete. +internal_events: true +status: active +action: token_status_processing_metrics +identifiers: +- project +- namespace +additional_properties: + label: + description: Type of pipeline that triggered token status processing (security or vulnerability) + value: + description: Time in milliseconds it took for the UpdateTokenStatusService to complete +product_group: secret_detection +product_categories: +- secret_detection +milestone: '18.4' +introduced_by_url: TODO +tiers: +- ultimate -- GitLab From 035302de602753fcf9da655395c1c62bfe17ced6 Mon Sep 17 00:00:00 2001 From: Radu Birsan Date: Fri, 12 Sep 2025 01:16:34 -0400 Subject: [PATCH 3/5] Updated spec for count vd project metric --- ...with_validity_checks_enabled_metric_spec.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb index 4877ce87029275..6a4ba7343d0b6e 100644 --- a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb +++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_projects_with_validity_checks_enabled_metric_spec.rb @@ -3,13 +3,17 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountProjectsWithValidityChecksEnabledMetric, feature_category: :service_ping do - let(:expected_value) { 3 } + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do + let(:expected_value) { 3 } + let(:expected_query) do + "SELECT COUNT(\"project_security_settings\".\"project_id\") FROM \"project_security_settings\" " \ + "WHERE \"project_security_settings\".\"validity_checks_enabled\" = TRUE" + end - before do - projects = create_list(:project, 3) - projects.each { |project| project.security_setting.update!(validity_checks_enabled: true) } - create(:project) + before do + projects = create_list(:project, 3) + projects.each { |project| project.security_setting.update!(validity_checks_enabled: true) } + create(:project) + end end - - it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } end -- GitLab From 60905beb2bf70695488800bb68bbcfd1b4c9c417 Mon Sep 17 00:00:00 2001 From: Radu Birsan Date: Wed, 24 Sep 2025 11:05:39 -0400 Subject: [PATCH 4/5] Fix the include InternalEventsTracking --- .../security/secret_detection/update_token_status_service.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ee/app/services/security/secret_detection/update_token_status_service.rb b/ee/app/services/security/secret_detection/update_token_status_service.rb index 28bdf3b6248510..04e7059de8dfeb 100644 --- a/ee/app/services/security/secret_detection/update_token_status_service.rb +++ b/ee/app/services/security/secret_detection/update_token_status_service.rb @@ -3,6 +3,8 @@ module Security module SecretDetection class UpdateTokenStatusService + include Gitlab::InternalEventsTracking + DEFAULT_BATCH_SIZE = 100 attr_reader :project -- GitLab From 25846d99fe8a78f810f96dcb9cff8b760a924142 Mon Sep 17 00:00:00 2001 From: Radu Birsan Date: Thu, 2 Oct 2025 06:04:21 -0400 Subject: [PATCH 5/5] Added metric for detected token type and status --- .../update_token_status_service.rb | 20 ++++++++++++++++++- .../events/finding_token_status_detected.yml | 20 +++++++++++++++++++ .../token_status_processing_metrics.yml | 12 +++++------ .../update_token_status_service_invoked.yml | 12 +++++------ 4 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 ee/config/events/finding_token_status_detected.yml diff --git a/ee/app/services/security/secret_detection/update_token_status_service.rb b/ee/app/services/security/secret_detection/update_token_status_service.rb index 04e7059de8dfeb..1a01dadf6da884 100644 --- a/ee/app/services/security/secret_detection/update_token_status_service.rb +++ b/ee/app/services/security/secret_detection/update_token_status_service.rb @@ -181,9 +181,27 @@ def sync_elasticsearch_for(findings, finding_type) def merge_token_status_into_attributes(tokens_by_raw, attrs_by_raw) tokens_by_raw.each do |raw_token, token| + token_type = if token.is_a?(Hash) && token[:type] + token[:type] + elsif token.respond_to?(:class) + token.class.name.demodulize + else + 'Unknown' + end + attrs_by_raw[raw_token]&.each do |finding_token_status_attr| - finding_token_status_attr[:status] = token_status(token) + cur_token_status = token_status(token) + finding_token_status_attr[:status] = cur_token_status finding_token_status_attr[:updated_at] = Time.current + + track_internal_event( + 'finding_token_status_detected', + project: @project, + additional_properties: { + label: token_type, + property: cur_token_status + } + ) end end end diff --git a/ee/config/events/finding_token_status_detected.yml b/ee/config/events/finding_token_status_detected.yml new file mode 100644 index 00000000000000..6ca126ec54728b --- /dev/null +++ b/ee/config/events/finding_token_status_detected.yml @@ -0,0 +1,20 @@ +--- +description: Tracks the types of tokens detected during UpdateTokenStatusService +internal_events: true +status: active +action: finding_token_status_detected +identifiers: + - project + - namespace +additional_properties: + label: + description: Status of finding token (one of active, inactive, unknown) + property: + description: Type of token detected +product_group: secret_detection +product_categories: + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 +tiers: + - ultimate diff --git a/ee/config/events/token_status_processing_metrics.yml b/ee/config/events/token_status_processing_metrics.yml index 19d3faa68a2adb..5b621cd8b8c86d 100644 --- a/ee/config/events/token_status_processing_metrics.yml +++ b/ee/config/events/token_status_processing_metrics.yml @@ -4,8 +4,8 @@ internal_events: true status: active action: token_status_processing_metrics identifiers: -- project -- namespace + - project + - namespace additional_properties: label: description: Type of pipeline that triggered token status processing (security or vulnerability) @@ -13,8 +13,8 @@ additional_properties: description: Time in milliseconds it took for the UpdateTokenStatusService to complete product_group: secret_detection product_categories: -- secret_detection -milestone: '18.4' -introduced_by_url: TODO + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 tiers: -- ultimate + - ultimate diff --git a/ee/config/events/update_token_status_service_invoked.yml b/ee/config/events/update_token_status_service_invoked.yml index 39aa5d0bb7cd6b..305f8ad985d773 100644 --- a/ee/config/events/update_token_status_service_invoked.yml +++ b/ee/config/events/update_token_status_service_invoked.yml @@ -4,8 +4,8 @@ internal_events: true status: active action: update_token_status_service_invoked identifiers: -- project -- namespace + - project + - namespace additional_properties: label: description: Type of pipeline that triggered token status processing (security or vulnerability) @@ -13,8 +13,8 @@ additional_properties: description: Number of secret detection findings sent to UpdateTokenStatusService for processing product_group: secret_detection product_categories: -- secret_detection -milestone: '18.4' -introduced_by_url: TODO + - secret_detection +milestone: '18.5' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204639 tiers: -- ultimate + - ultimate -- GitLab