From 892c19264f643e23ee33aaa7a625ca96826ffec5 Mon Sep 17 00:00:00 2001 From: Subashis Date: Thu, 4 Dec 2025 21:06:46 -0700 Subject: [PATCH 01/16] Add undetected since --- ...y_dashboard_exclude_no_longer_detected.yml | 10 + ee/app/models/vulnerabilities/finding.rb | 9 + .../ingestion/mark_as_resolved_service.rb | 7 + .../mark_resolved_as_detected.rb | 6 + ...starboard_vulnerability_resolve_service.rb | 7 + ...ndetected_since_field_to_vulnerability.yml | 10 + ...undetected_since_field_to_vulnerability.rb | 17 ++ .../vulnerability/enhanced_proxy.rb | 10 +- .../vulnerability/undetected_since.rb | 36 +++ .../elastic/record_proxy/vulnerability.rb | 5 +- .../elastic/references/vulnerability.rb | 2 + ee/lib/search/elastic/types/vulnerability.rb | 3 +- ...ected_since_field_to_vulnerability_spec.rb | 10 + .../vulnerability/undetected_since_spec.rb | 212 ++++++++++++++++++ .../models/vulnerabilities/finding_spec.rb | 63 ++++++ 15 files changed, 402 insertions(+), 5 deletions(-) create mode 100644 config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml create mode 100644 ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml create mode 100644 ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb create mode 100644 ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb create mode 100644 ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb create mode 100644 ee/spec/lib/search/elastic/preloaders/vulnerability/undetected_since_spec.rb diff --git a/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml b/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml new file mode 100644 index 00000000000000..7ef7df5f862954 --- /dev/null +++ b/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml @@ -0,0 +1,10 @@ +--- +name: new_security_dashboard_exclude_no_longer_detected +description: +feature_issue_url: https://gitlab.com/groups/gitlab-org/-/work_items/19780 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215334 +rollout_issue_url: +milestone: '18.7' +group: group::security insights +type: wip +default_enabled: false diff --git a/ee/app/models/vulnerabilities/finding.rb b/ee/app/models/vulnerabilities/finding.rb index dc3728087db84e..8af09cdd24ed3c 100644 --- a/ee/app/models/vulnerabilities/finding.rb +++ b/ee/app/models/vulnerabilities/finding.rb @@ -199,6 +199,11 @@ class Finding < ::SecApplicationRecord ) end + scope :with_latest_detection_transition, -> do + last_detection_transition = Vulnerabilities::DetectionTransition.where(arel_table[:id].eq(Vulnerabilities::DetectionTransition.arel_table[:vulnerability_occurrence_id])).order(id: :desc).limit(1) + includes(:detection_transitions).where(detection_transitions: { id: last_detection_transition }) + end + scope :with_fix_available, ->(fix_available) do remediation = ::Vulnerabilities::FindingRemediation.arel_table solution_query = where(fix_available ? 'vulnerability_occurrences.solution IS NOT NULL' : 'vulnerability_occurrences.solution IS NULL') @@ -612,6 +617,10 @@ def ai_resolution_supported_cwe? end end + def latest_detection_transition + @latest_detection_transition ||= detection_transitions.last + end + protected def primary_identifier_fingerprint diff --git a/ee/app/services/security/ingestion/mark_as_resolved_service.rb b/ee/app/services/security/ingestion/mark_as_resolved_service.rb index eaad8a141e8941..26204d4ed1c553 100644 --- a/ee/app/services/security/ingestion/mark_as_resolved_service.rb +++ b/ee/app/services/security/ingestion/mark_as_resolved_service.rb @@ -98,9 +98,16 @@ def mark_as_no_longer_detected(vulnerabilities) CreateVulnerabilityRepresentationInformation.execute(pipeline, vulnerabilities_relation) + create_detection_transitions(no_longer_detected_vulnerability_ids) + track_no_longer_detected_vulnerabilities(no_longer_detected_vulnerability_ids.count) end + def create_detection_transitions(vulnerability_ids) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) + ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: false).execute + end + def auto_resolve(no_longer_detected_vulnerability_ids) budget = AUTO_RESOLVE_LIMIT - auto_resolved_count return unless budget > 0 diff --git a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb index 2cffab13b55434..e4a6fd59cec2a0 100644 --- a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb +++ b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb @@ -60,6 +60,7 @@ def update_vulnerability_records SecApplicationRecord.current_transaction.after_commit do ::Vulnerabilities::BulkEsOperationService.new(vulnerabilities_relation).execute(&:itself) + create_detection_transitions(redetected_vulnerability_ids) end end @@ -67,6 +68,11 @@ def create_state_transitions ::Vulnerabilities::StateTransition.bulk_insert!(state_transitions) end + def create_detection_transitions(vulnerability_ids) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) + ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: true).execute + end + def state_transitions redetected_vulnerability_ids.map do |vulnerability_id| ::Vulnerabilities::StateTransition.new( diff --git a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb index 391ccc1d4d8a9b..c3cfdd4b31252a 100644 --- a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb +++ b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb @@ -34,6 +34,8 @@ def execute end end + create_detection_transitions(undetected.map(&:id)) + ServiceResponse.success end @@ -47,6 +49,11 @@ def undetected .with_cluster_agent_ids([agent.id.to_s]) end + def create_detection_transitions(vulnerability_ids) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) + ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: false).execute + end + def authorized? can?(author, :admin_vulnerability, project) end diff --git a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml new file mode 100644 index 00000000000000..99d5989478dffc --- /dev/null +++ b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml @@ -0,0 +1,10 @@ +--- +name: AddUndetectedSinceFieldToVulnerability +version: '20251204112110' +description: Add undetected_since field to Vulnerabilities ES index +group: group::group::security insights +milestone: '18.7' +introduced_by_url: +obsolete: false +marked_obsolete_by_url: +marked_obsolete_in_milestone: diff --git a/ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb b/ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb new file mode 100644 index 00000000000000..5e42104e1a7aa2 --- /dev/null +++ b/ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddUndetectedSinceFieldToVulnerability < Elastic::Migration + include ::Search::Elastic::MigrationUpdateMappingsHelper + + DOCUMENT_TYPE = Vulnerability + + private + + def new_mappings + { + undetected_since: { + type: 'date' + } + } + end +end diff --git a/ee/lib/search/elastic/preloaders/vulnerability/enhanced_proxy.rb b/ee/lib/search/elastic/preloaders/vulnerability/enhanced_proxy.rb index d6138fcbeffdba..e7963305fa51c3 100644 --- a/ee/lib/search/elastic/preloaders/vulnerability/enhanced_proxy.rb +++ b/ee/lib/search/elastic/preloaders/vulnerability/enhanced_proxy.rb @@ -9,7 +9,7 @@ module Vulnerability # # This class: # 1. Coordinates multiple individual preloaders (EPSS, CVE, Reachability, TokenStatus, - # PolicyViolations, RiskScore, FalsePositive) + # PolicyViolations, RiskScore, FalsePositive, UndetectedSince) # 2. Creates enhanced proxies with all preloaded data # 3. Assigns proxies back to reference objects # 4. Provides a single entry point for all vulnerability data preloading @@ -37,6 +37,7 @@ def preload_all_data @policy_violations_data = PolicyViolations.new(records).preload @risk_score_data = RiskScore.new(records).preload @false_positive_data = FalsePositive.new(records).preload + @undetected_since_data = UndetectedSince.new(records).preload end # Create enhanced proxies and assign them to references @@ -59,7 +60,8 @@ def create_enhanced_proxy(record) 'token_status' => fetch_token_status_value(vulnerability_id), 'policy_violations' => fetch_policy_violation_value(vulnerability_id), 'risk_score' => fetch_risk_score_value(vulnerability_id), - 'false_positive' => fetch_false_positive_value(vulnerability_id) + 'false_positive' => fetch_false_positive_value(vulnerability_id), + 'undetected_since' => fetch_undetected_since_value(vulnerability_id) }.with_indifferent_access ::Search::Elastic::RecordProxy::Vulnerability.create_with_enhancements(record, enhancements) @@ -84,6 +86,10 @@ def fetch_risk_score_value(vulnerability_id) def fetch_false_positive_value(vulnerability_id) @false_positive_data[vulnerability_id] || false end + + def fetch_undetected_since_value(vulnerability_id) + @undetected_since_data[vulnerability_id] + end end end end diff --git a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb new file mode 100644 index 00000000000000..ebaf24accebeca --- /dev/null +++ b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Search + module Elastic + module Preloaders + module Vulnerability + # Preloads whether vulnerabilities have a FalsePositive flag. + # + # Returns a Hash like: + # { 101 => date, 102 => date } + class UndetectedSince < Base + # @return [Hash{Integer => date}] vulnerability_id => undetected_since_timestamp + def perform_preload + vulnerability_ids = record_identifiers + + return {} if vulnerability_ids.empty? + + fetch_undetected_since_data(vulnerability_ids) + end + + private + + def fetch_undetected_since_data(vulnerability_ids) + ::Vulnerabilities::Finding + .by_vulnerability(vulnerability_ids) + .with_latest_detection_transition + .each_with_object({}) do |finding, result| + transition = finding.latest_detection_transition + result[finding.vulnerability_id] = transition&.created_at unless transition.detected? + end + end + end + end + end + end +end diff --git a/ee/lib/search/elastic/record_proxy/vulnerability.rb b/ee/lib/search/elastic/record_proxy/vulnerability.rb index 5abb0be0d2c093..c43990017a3f0d 100644 --- a/ee/lib/search/elastic/record_proxy/vulnerability.rb +++ b/ee/lib/search/elastic/record_proxy/vulnerability.rb @@ -5,7 +5,7 @@ module Elastic module RecordProxy # Vulnerability-specific record proxy that provides optimized access # to elasticsearch indexing data including EPSS scores, CVE values, - # reachability, token status, policy_violations, risk_score, and false_positive information. + # reachability, token status, policy_violations, risk_score, false_positive and undetected_since information. class Vulnerability < Base # Creates a vulnerability proxy with all optimizations applied def self.create_with_enhancements(record, enhancements) @@ -16,7 +16,8 @@ def self.create_with_enhancements(record, enhancements) token_status: enhancements[:token_status], policy_violations: enhancements[:policy_violations], risk_score: enhancements[:risk_score], - false_positive: enhancements[:false_positive] + false_positive: enhancements[:false_positive], + undetected_since: enhancements[:undetected_since] }) proxy diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index cec15c198a4cd9..2bcd700d76d770 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -110,6 +110,8 @@ def as_indexed_json fields["false_positive"] = fetch_record_attribute(database_record, :false_positive) end + fields["undetected_since"] = fetch_record_attribute(database_record, :undetected_since) + internal_es_fields.merge(fields) end diff --git a/ee/lib/search/elastic/types/vulnerability.rb b/ee/lib/search/elastic/types/vulnerability.rb index 1301d88b56fc5c..6db841d2686078 100644 --- a/ee/lib/search/elastic/types/vulnerability.rb +++ b/ee/lib/search/elastic/types/vulnerability.rb @@ -77,7 +77,8 @@ def base_mappings policy_violations: { type: 'short' }, # enum false_positive: { type: 'boolean' }, schema_version: { type: 'short' }, - security_project_tracked_context_id: { type: 'long' } + security_project_tracked_context_id: { type: 'long' }, + undetected_since: { type: 'date' } } end diff --git a/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb b/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb new file mode 100644 index 00000000000000..395c4a058b4e65 --- /dev/null +++ b/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' +require File.expand_path('ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb') + +RSpec.describe AddUndetectedSinceFieldToVulnerability, feature_category: :vulnerability_management do + let(:version) { 20251204112110 } + + include_examples 'migration adds mapping' +end diff --git a/ee/spec/lib/search/elastic/preloaders/vulnerability/undetected_since_spec.rb b/ee/spec/lib/search/elastic/preloaders/vulnerability/undetected_since_spec.rb new file mode 100644 index 00000000000000..72f95329643dc1 --- /dev/null +++ b/ee/spec/lib/search/elastic/preloaders/vulnerability/undetected_since_spec.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Search::Elastic::Preloaders::Vulnerability::UndetectedSince, + feature_category: :vulnerability_management do + let_it_be(:project) { create(:project) } + + def create_vulnerability_with_detection_transition(detected:, created_at: Time.current) + vulnerability = create(:vulnerability, :with_read, project: project) + finding = create(:vulnerabilities_finding, vulnerability: vulnerability) + + create(:vulnerability_detection_transition, + finding: finding, + detected: detected, + created_at: created_at + ) + + vulnerability.vulnerability_read + end + + describe '#preload' do + subject(:preloader) { described_class.new(records) } + + context 'with vulnerabilities having different detection states' do + let_it_be(:vuln_undetected) do + create_vulnerability_with_detection_transition(detected: false, created_at: 2.days.ago) + end + + let_it_be(:vuln_detected) do + create_vulnerability_with_detection_transition(detected: true, created_at: 1.day.ago) + end + + let(:records) { [vuln_undetected, vuln_detected] } + + it 'returns undetected_since timestamp for undetected vulnerabilities' do + result = preloader.preload + + expect(result[vuln_undetected.vulnerability_id]).to be_present + expect(result[vuln_undetected.vulnerability_id]).to be_within(5.seconds).of(2.days.ago) + end + + it 'does not include detected vulnerabilities in the result' do + result = preloader.preload + + expect(result).not_to have_key(vuln_detected.vulnerability_id) + end + end + + context 'with vulnerabilities having multiple detection transitions' do + let_it_be(:vuln_with_multiple_transitions) do + vulnerability = create(:vulnerability, :with_read, project: project) + finding = create(:vulnerabilities_finding, vulnerability: vulnerability) + + create(:vulnerability_detection_transition, + finding: finding, + detected: true, + created_at: 3.days.ago + ) + + create(:vulnerability_detection_transition, + finding: finding, + detected: false, + created_at: 1.day.ago + ) + + vulnerability.vulnerability_read + end + + let(:records) { [vuln_with_multiple_transitions] } + + it 'returns the timestamp from the latest detection transition' do + result = preloader.preload + + expect(result[vuln_with_multiple_transitions.vulnerability_id]).to be_within(1.second).of(1.day.ago) + end + end + + context 'with edge cases' do + context 'when records are empty' do + let(:records) { [] } + + it 'returns an empty hash' do + expect(preloader.preload).to eq({}) + end + end + + context 'when vulnerability has no detection transitions' do + let_it_be(:vuln_without_transition) do + vulnerability = create(:vulnerability, :with_read, project: project) + create(:vulnerabilities_finding, vulnerability: vulnerability) + vulnerability.vulnerability_read + end + + let(:records) { [vuln_without_transition] } + + it 'does not include the vulnerability in the result' do + result = preloader.preload + + expect(result).not_to have_key(vuln_without_transition.vulnerability_id) + end + end + + context 'when database query raises an error' do + let(:records) { [create_vulnerability_with_detection_transition(detected: false)] } + + before do + allow(::Vulnerabilities::Finding) + .to receive(:by_vulnerability) + .and_raise(StandardError, 'DB failure') + + allow(::Gitlab::ErrorTracking).to receive(:track_exception) + end + + it 'handles gracefully and tracks the error' do + result = begin + preloader.preload + rescue StandardError + {} + end + + expect(result).to eq({}) + expect(::Gitlab::ErrorTracking).to have_received(:track_exception) + .with(instance_of(StandardError), class: described_class.name) + end + end + end + end + + describe '#data_for' do + let_it_be(:vuln_undetected) do + create_vulnerability_with_detection_transition(detected: false, created_at: 2.days.ago) + end + + let_it_be(:vuln_detected) do + create_vulnerability_with_detection_transition(detected: true) + end + + subject(:preloader) { described_class.new([vuln_undetected, vuln_detected]) } + + before do + preloader.preload + end + + it 'returns undetected_since timestamp for undetected vulnerabilities' do + expect(preloader.data_for(vuln_undetected)).to be_within(5.seconds).of(2.days.ago) + end + + it 'returns nil for detected vulnerabilities' do + expect(preloader.data_for(vuln_detected)).to be_nil + end + + it 'returns nil for unknown vulnerability' do + unknown_read = create(:vulnerability, :with_read, project: project).vulnerability_read + expect(preloader.data_for(unknown_read)).to be_nil + end + end + + describe 'integration with PreloaderBase' do + let_it_be(:vuln_undetected) do + create_vulnerability_with_detection_transition(detected: false) + end + + subject(:preloader) { described_class.new([vuln_undetected]) } + + it 'tracks preload state correctly' do + expect(preloader.preloaded?).to be false + preloader.preload + expect(preloader.preloaded?).to be true + end + + it 'deduplicates vulnerabilities before querying' do + duplicates = [vuln_undetected, vuln_undetected] + preloader = described_class.new(duplicates) + + expect(::Vulnerabilities::Finding).to receive(:by_vulnerability).once.and_call_original + preloader.preload + end + + it 'caches preloaded data to avoid duplicate queries' do + expect(::Vulnerabilities::Finding).to receive(:by_vulnerability).once.and_call_original + + result1 = preloader.preload + result2 = preloader.preload + + expect(result1).to eq(result2) + end + end + + describe 'performance characteristics' do + let_it_be(:large_vulnerability_set) do + Array.new(5) do + create_vulnerability_with_detection_transition(detected: false, created_at: rand(1..10).days.ago) + end + end + + it 'performs batch queries efficiently' do + preloader = described_class.new(large_vulnerability_set) + expect { preloader.preload }.not_to exceed_query_limit(3) + end + + it 'avoids N+1 queries with larger datasets' do + initial_set = large_vulnerability_set.take(2) + preloader = described_class.new(initial_set) + control = ActiveRecord::QueryRecorder.new { preloader.preload } + + larger_preloader = described_class.new(large_vulnerability_set) + + expect { larger_preloader.preload }.not_to exceed_query_limit(control) + end + end +end diff --git a/ee/spec/models/vulnerabilities/finding_spec.rb b/ee/spec/models/vulnerabilities/finding_spec.rb index c9413c1f11406f..de03ecc866baab 100644 --- a/ee/spec/models/vulnerabilities/finding_spec.rb +++ b/ee/spec/models/vulnerabilities/finding_spec.rb @@ -1991,4 +1991,67 @@ def create_finding(state) end end end + + describe '.with_latest_detection_transition' do + let_it_be(:finding1) { create(:vulnerabilities_finding) } + let_it_be(:finding2) { create(:vulnerabilities_finding) } + let_it_be(:finding3) { create(:vulnerabilities_finding) } + + let_it_be(:transition1_old) do + create(:vulnerability_detection_transition, finding: finding1, detected: true, created_at: 2.days.ago) + end + + let_it_be(:transition1_latest) do + create(:vulnerability_detection_transition, finding: finding1, detected: false, created_at: 1.day.ago) + end + + let_it_be(:transition2_latest) do + create(:vulnerability_detection_transition, finding: finding2, detected: true, created_at: 1.day.ago) + end + + subject { described_class.with_latest_detection_transition } + + it 'includes only the latest detection transition for each finding' do + findings = subject.to_a + + expect(findings).to contain_exactly(finding1, finding2) + expect(findings.find { |f| f.id == finding1.id }.detection_transitions).to contain_exactly(transition1_latest) + expect(findings.find { |f| f.id == finding2.id }.detection_transitions).to contain_exactly(transition2_latest) + end + + it 'does not include findings without detection transitions' do + expect(subject).not_to include(finding3) + end + + it 'orders by detection transition id descending' do + finding_with_transitions = subject.find_by(id: finding1.id) + + expect(finding_with_transitions.detection_transitions.first).to eq(transition1_latest) + end + end + + describe '#latest_detection_transition' do + let_it_be(:finding) { create(:vulnerabilities_finding) } + + context 'when finding has multiple detection transitions' do + let_it_be(:old_transition) do + create(:vulnerability_detection_transition, finding: finding, detected: true, created_at: 2.days.ago) + end + + let_it_be(:latest_transition) do + create(:vulnerability_detection_transition, finding: finding, detected: false, created_at: 1.day.ago) + end + + it 'returns the last detection transition' do + expect(finding.latest_detection_transition).to eq(latest_transition) + end + + it 'memoizes the result' do + expect(finding.latest_detection_transition).to eq(latest_transition) + + # Call again to verify memoization + expect(finding.instance_variable_get(:@latest_detection_transition)).to eq(latest_transition) + end + end + end end -- GitLab From a9aba0099254f57d0f68fabd9c4adb4ca28c555d Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 11:51:46 +0100 Subject: [PATCH 02/16] Fix typo, empty description + comment preloader --- .../wip/new_security_dashboard_exclude_no_longer_detected.yml | 2 +- ...1204112110_add_undetected_since_field_to_vulnerability.yml | 2 +- .../elastic/preloaders/vulnerability/undetected_since.rb | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml b/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml index 7ef7df5f862954..7c17f241313e93 100644 --- a/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml +++ b/config/feature_flags/wip/new_security_dashboard_exclude_no_longer_detected.yml @@ -1,6 +1,6 @@ --- name: new_security_dashboard_exclude_no_longer_detected -description: +description: Excludes no longer detected vulnerabilities from security dashboard using undetected_since timestamp feature_issue_url: https://gitlab.com/groups/gitlab-org/-/work_items/19780 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215334 rollout_issue_url: diff --git a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml index 99d5989478dffc..654b134672cc6b 100644 --- a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml +++ b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml @@ -2,7 +2,7 @@ name: AddUndetectedSinceFieldToVulnerability version: '20251204112110' description: Add undetected_since field to Vulnerabilities ES index -group: group::group::security insights +group: group::security insights milestone: '18.7' introduced_by_url: obsolete: false diff --git a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb index ebaf24accebeca..6cb8bd7a241565 100644 --- a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb +++ b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb @@ -4,10 +4,10 @@ module Search module Elastic module Preloaders module Vulnerability - # Preloads whether vulnerabilities have a FalsePositive flag. + # Preloads the undetected_since timestamp for vulnerabilities. # # Returns a Hash like: - # { 101 => date, 102 => date } + # { vulnerability_id => undetected_since_timestamp } class UndetectedSince < Base # @return [Hash{Integer => date}] vulnerability_id => undetected_since_timestamp def perform_preload -- GitLab From 88868b09213dc28e1a5d981452c54f63449cc3cd Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 12:37:36 +0100 Subject: [PATCH 03/16] Fix fialing specs --- .../starboard_vulnerability_resolve_service.rb | 4 +++- .../security/ingestion/mark_as_resolved_service_spec.rb | 8 ++++++++ .../mark_resolved_as_detected_spec.rb | 8 ++++++++ .../starboard_vulnerability_resolve_service_spec.rb | 9 +++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb index c3cfdd4b31252a..12995352eb90c5 100644 --- a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb +++ b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb @@ -24,6 +24,8 @@ def initialize(agent, uuids) def execute raise Gitlab::Access::AccessDeniedError unless authorized? + undetected_vulnerability_ids = undetected.pluck(:id) + undetected.each_batch(of: BATCH_SIZE) do |batch| Vulnerabilities::BulkEsOperationService.new(batch).execute do |vulnerabilities| Vulnerability.transaction do @@ -34,7 +36,7 @@ def execute end end - create_detection_transitions(undetected.map(&:id)) + create_detection_transitions(undetected_vulnerability_ids) ServiceResponse.success end diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index d3e6dc725eb1f7..19751af3ee6de1 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -53,6 +53,14 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) expect(vulnerability.vulnerability_read.resolved_on_default_branch).to be_truthy end + it 'creates a detection transition with detected: false' do + expect { command.execute }.to change { Vulnerabilities::DetectionTransition.count }.by(1) + + transition = Vulnerabilities::DetectionTransition.last + expect(transition.detected).to be(false) + expect(vulnerability.findings.map(&:id)).to include(transition.vulnerability_occurrence_id) + end + context 'when there is a no longer detected vulnerability' do let_it_be_with_reload(:no_longer_detected) do create(:vulnerability, :sast, diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 3ade67ee6faf39..115bd8114aaac5 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -73,6 +73,14 @@ expect(state_transition.vulnerability_id).to eq(resolved_vulnerability.id) end + it 'creates a detection transition with detected: true' do + expect { mark_resolved_as_detected }.to change { Vulnerabilities::DetectionTransition.count }.by(1) + + transition = Vulnerabilities::DetectionTransition.last + expect(transition.detected).to be(true) + expect(resolved_vulnerability.findings.map(&:id)).to include(transition.vulnerability_occurrence_id) + end + it 'marks the findings as transitioned_to_detected' do expect { mark_resolved_as_detected }.to change { existing_resolved_finding_map.transitioned_to_detected }.to(true) .and not_change { existing_detected_finding_map.transitioned_to_detected } diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index 0d6dfc92a9a322..e182f1b3e67032 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -55,6 +55,15 @@ )) end + it 'creates detection transitions with detected: false' do + expect { service.execute }.to change { + Vulnerabilities::DetectionTransition.count + }.by(undetected_vulnerabilities.count) + + transitions = Vulnerabilities::DetectionTransition.last(undetected_vulnerabilities.count) + expect(transitions).to all(have_attributes(detected: false)) + end + it 'touches the updated_at timestamp', :freeze_time do service.execute -- GitLab From 2085a8bd7082107ddc1994836be2722d2ae5e58c Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 14:50:05 +0100 Subject: [PATCH 04/16] Add elastic tag --- ...04112110_add_undetected_since_field_to_vulnerability_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb b/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb index 395c4a058b4e65..261bd5d0004733 100644 --- a/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb +++ b/ee/spec/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require File.expand_path('ee/elastic/migrate/20251204112110_add_undetected_since_field_to_vulnerability.rb') -RSpec.describe AddUndetectedSinceFieldToVulnerability, feature_category: :vulnerability_management do +RSpec.describe AddUndetectedSinceFieldToVulnerability, :elastic, feature_category: :vulnerability_management do let(:version) { 20251204112110 } include_examples 'migration adds mapping' -- GitLab From 3895a54e9d884253273ce4a704f5064f7904521f Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 14:57:31 +0100 Subject: [PATCH 05/16] Fix enhanced proxy spec --- .../preloaders/vulnerability/enhanced_proxy_spec.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb b/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb index dd122a773edefe..8a48cfb8eab504 100644 --- a/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb +++ b/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb @@ -206,7 +206,8 @@ 'token_status' => ::Security::TokenStatus::ACTIVE, 'policy_violations' => ::Security::PolicyViolations::DISMISSED_IN_MR, 'risk_score' => 0.4, - 'false_positive' => true + 'false_positive' => true, + 'undetected_since' => nil }.with_indifferent_access expected_enhancements_not_found = { @@ -214,7 +215,8 @@ 'token_status' => ::Security::TokenStatus::INACTIVE, 'policy_violations' => nil, 'risk_score' => 0.6, - 'false_positive' => false + 'false_positive' => false, + 'undetected_since' => nil }.with_indifferent_access expected_enhancements_unknown = { @@ -222,7 +224,8 @@ 'token_status' => ::Security::TokenStatus::UNKNOWN, 'policy_violations' => nil, 'risk_score' => 0.0, - 'false_positive' => false + 'false_positive' => false, + 'undetected_since' => nil }.with_indifferent_access expect(::Search::Elastic::RecordProxy::Vulnerability) -- GitLab From 9e6a2486795f83a149980cb1e8c018c1345caab4 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 15:14:27 +0100 Subject: [PATCH 06/16] Fix rubo --- ee/lib/search/elastic/references/vulnerability.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index 2bcd700d76d770..1745bea0b9f2c0 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -71,6 +71,7 @@ def serialize end # Generate the JSON representation for elasticsearch indexing + # rubocop:disable Metrics/AbcSize -- We want this to stay a single method override :as_indexed_json def as_indexed_json fields = {} @@ -114,6 +115,7 @@ def as_indexed_json internal_es_fields.merge(fields) end + # rubocop:enable Metrics/AbcSize override :index_name def index_name -- GitLab From f967b449671855fae4be08198625f1d5deebe47c Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 15:27:27 +0100 Subject: [PATCH 07/16] Fix rubocop issues + specs --- .../starboard_vulnerability_resolve_service.rb | 7 ++++++- .../security/ingestion/mark_as_resolved_service_spec.rb | 9 ++++++--- .../mark_resolved_as_detected_spec.rb | 9 ++++++--- .../starboard_vulnerability_resolve_service_spec.rb | 6 ++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb index 12995352eb90c5..7b5c0b1f8d891f 100644 --- a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb +++ b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb @@ -24,9 +24,14 @@ def initialize(agent, uuids) def execute raise Gitlab::Access::AccessDeniedError unless authorized? - undetected_vulnerability_ids = undetected.pluck(:id) + undetected_vulnerability_ids = [] undetected.each_batch(of: BATCH_SIZE) do |batch| + # rubocop:disable CodeReuse/ActiveRecord, Database/AvoidUsingPluckWithoutLimit -- batch limited to 250 + batch_ids = batch.pluck(:id) + # rubocop:enable CodeReuse/ActiveRecord, Database/AvoidUsingPluckWithoutLimit + undetected_vulnerability_ids.concat(batch_ids) + Vulnerabilities::BulkEsOperationService.new(batch).execute do |vulnerabilities| Vulnerability.transaction do vulnerabilities.update_all(resolved_on_default_branch: true, state: :resolved, updated_at: now) diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index 19751af3ee6de1..602c50178eb1c2 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -54,11 +54,14 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) end it 'creates a detection transition with detected: false' do - expect { command.execute }.to change { Vulnerabilities::DetectionTransition.count }.by(1) + finding_ids = vulnerability.findings.pluck(:id) - transition = Vulnerabilities::DetectionTransition.last + expect { command.execute }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) + + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) expect(transition.detected).to be(false) - expect(vulnerability.findings.map(&:id)).to include(transition.vulnerability_occurrence_id) end context 'when there is a no longer detected vulnerability' do diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 115bd8114aaac5..30dc9669b8775d 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -74,11 +74,14 @@ end it 'creates a detection transition with detected: true' do - expect { mark_resolved_as_detected }.to change { Vulnerabilities::DetectionTransition.count }.by(1) + finding_ids = resolved_vulnerability.findings.pluck(:id) - transition = Vulnerabilities::DetectionTransition.last + expect { mark_resolved_as_detected }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) + + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) expect(transition.detected).to be(true) - expect(resolved_vulnerability.findings.map(&:id)).to include(transition.vulnerability_occurrence_id) end it 'marks the findings as transitioned_to_detected' do diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index e182f1b3e67032..7b59bda969c3b5 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -56,11 +56,13 @@ end it 'creates detection transitions with detected: false' do + finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } + expect { service.execute }.to change { - Vulnerabilities::DetectionTransition.count + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count }.by(undetected_vulnerabilities.count) - transitions = Vulnerabilities::DetectionTransition.last(undetected_vulnerabilities.count) + transitions = Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids) expect(transitions).to all(have_attributes(detected: false)) end -- GitLab From 49a7ac9e10cbe4289f964b9811ada5ffba716909 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 15:52:33 +0100 Subject: [PATCH 08/16] Add feature flag check --- .../ingestion/mark_as_resolved_service.rb | 2 ++ .../mark_resolved_as_detected.rb | 2 ++ ...starboard_vulnerability_resolve_service.rb | 2 ++ .../elastic/references/vulnerability.rb | 8 +++++++- .../mark_as_resolved_service_spec.rb | 20 ++++++++++++------- .../mark_resolved_as_detected_spec.rb | 20 ++++++++++++------- ...oard_vulnerability_resolve_service_spec.rb | 20 ++++++++++++------- 7 files changed, 52 insertions(+), 22 deletions(-) diff --git a/ee/app/services/security/ingestion/mark_as_resolved_service.rb b/ee/app/services/security/ingestion/mark_as_resolved_service.rb index 26204d4ed1c553..1bda2c1785f36c 100644 --- a/ee/app/services/security/ingestion/mark_as_resolved_service.rb +++ b/ee/app/services/security/ingestion/mark_as_resolved_service.rb @@ -104,6 +104,8 @@ def mark_as_no_longer_detected(vulnerabilities) end def create_detection_transitions(vulnerability_ids) + return unless Feature.enabled?(:new_security_dashboard_exclude_no_longer_detected, project) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: false).execute end diff --git a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb index e4a6fd59cec2a0..3d20b0b2bf5275 100644 --- a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb +++ b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb @@ -69,6 +69,8 @@ def create_state_transitions end def create_detection_transitions(vulnerability_ids) + return unless Feature.enabled?(:new_security_dashboard_exclude_no_longer_detected, pipeline.project) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: true).execute end diff --git a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb index 7b5c0b1f8d891f..a32d96b6157c5c 100644 --- a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb +++ b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb @@ -57,6 +57,8 @@ def undetected end def create_detection_transitions(vulnerability_ids) + return unless Feature.enabled?(:new_security_dashboard_exclude_no_longer_detected, project) + findings = Vulnerabilities::Finding.by_vulnerability(vulnerability_ids) ::Vulnerabilities::DetectionTransitions::InsertService.new(findings, detected: false).execute end diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index 1745bea0b9f2c0..d3120edb259e4c 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -111,7 +111,9 @@ def as_indexed_json fields["false_positive"] = fetch_record_attribute(database_record, :false_positive) end - fields["undetected_since"] = fetch_record_attribute(database_record, :undetected_since) + if undetected_since_migration_completed? + fields["undetected_since"] = fetch_record_attribute(database_record, :undetected_since) + end internal_es_fields.merge(fields) end @@ -209,6 +211,10 @@ def false_positive_migration_completed? ::Elastic::DataMigrationService.migration_has_finished?(:add_false_positive_field_to_vulnerability) end + def undetected_since_migration_completed? + ::Elastic::DataMigrationService.migration_has_finished?(:add_undetected_since_field_to_vulnerability) + end + # # Private class methods for implementation details # diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index 602c50178eb1c2..71483ed428b023 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -53,15 +53,21 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) expect(vulnerability.vulnerability_read.resolved_on_default_branch).to be_truthy end - it 'creates a detection transition with detected: false' do - finding_ids = vulnerability.findings.pluck(:id) + context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) + end - expect { command.execute }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(1) + it 'creates a detection transition with detected: false' do + finding_ids = vulnerability.findings.pluck(:id) - transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) - expect(transition.detected).to be(false) + expect { command.execute }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) + + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) + expect(transition.detected).to be(false) + end end context 'when there is a no longer detected vulnerability' do diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 30dc9669b8775d..89fa6538f14e6b 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -73,15 +73,21 @@ expect(state_transition.vulnerability_id).to eq(resolved_vulnerability.id) end - it 'creates a detection transition with detected: true' do - finding_ids = resolved_vulnerability.findings.pluck(:id) + context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) + end - expect { mark_resolved_as_detected }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(1) + it 'creates a detection transition with detected: true' do + finding_ids = resolved_vulnerability.findings.pluck(:id) - transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) - expect(transition.detected).to be(true) + expect { mark_resolved_as_detected }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) + + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) + expect(transition.detected).to be(true) + end end it 'marks the findings as transitioned_to_detected' do diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index 7b59bda969c3b5..7614f12ef5ffe1 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -55,15 +55,21 @@ )) end - it 'creates detection transitions with detected: false' do - finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } + context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) + end - expect { service.execute }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(undetected_vulnerabilities.count) + it 'creates detection transitions with detected: false' do + finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } - transitions = Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids) - expect(transitions).to all(have_attributes(detected: false)) + expect { service.execute }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(undetected_vulnerabilities.count) + + transitions = Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids) + expect(transitions).to all(have_attributes(detected: false)) + end end it 'touches the updated_at timestamp', :freeze_time do -- GitLab From 3a40571c48ca2e7d10e3a36f186d81d200e07395 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 10 Dec 2025 17:27:50 +0100 Subject: [PATCH 09/16] Add missing undetected_since + add real spec cases to enhanced proxy spec --- .../vulnerability/enhanced_proxy_spec.rb | 28 +++++++++++++++++-- .../elastic/references/vulnerability_spec.rb | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb b/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb index 8a48cfb8eab504..fcd54b01b6608e 100644 --- a/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb +++ b/ee/spec/lib/search/elastic/preloaders/vulnerability/enhanced_proxy_spec.rb @@ -25,6 +25,7 @@ let(:mock_risk_score_preloader) { instance_double(Search::Elastic::Preloaders::Vulnerability::RiskScore) } let(:mock_false_positive_preloader) { instance_double(Search::Elastic::Preloaders::Vulnerability::FalsePositive) } + let(:mock_undetected_since_preloader) { instance_double(Search::Elastic::Preloaders::Vulnerability::UndetectedSince) } let(:reachability_data) do { @@ -64,6 +65,13 @@ } end + let(:undetected_since_data) do + { + vulnerability_reads[1].vulnerability_id => 2.days.ago + # first and third intentionally missing (still detected) + } + end + let(:enhanced_proxy) { described_class.new(refs, vulnerability_reads) } before do @@ -112,6 +120,15 @@ allow(mock_false_positive_preloader) .to receive(:preload) .and_return(false_positive_data) + + allow(Search::Elastic::Preloaders::Vulnerability::UndetectedSince) + .to receive(:new) + .with(vulnerability_reads) + .and_return(mock_undetected_since_preloader) + + allow(mock_undetected_since_preloader) + .to receive(:preload) + .and_return(undetected_since_data) end describe '#initialize' do @@ -133,7 +150,7 @@ end describe '#preload_and_enhance!' do - it 'preloads reachability, token status, policy_violations, risk score and false_positive data' do + it 'preloads reachability, token status, policy_violations, risk score, false_positive and undetected_since data' do enhanced_proxy.preload_and_enhance! expect(Search::Elastic::Preloaders::Vulnerability::Reachability) @@ -155,6 +172,10 @@ expect(Search::Elastic::Preloaders::Vulnerability::FalsePositive) .to have_received(:new).with(vulnerability_reads) expect(mock_false_positive_preloader).to have_received(:preload) + + expect(Search::Elastic::Preloaders::Vulnerability::UndetectedSince) + .to have_received(:new).with(vulnerability_reads) + expect(mock_undetected_since_preloader).to have_received(:preload) end it 'creates enhanced proxies for matching refs' do @@ -200,7 +221,8 @@ 'token_status, ' \ 'policy_violations, ' \ 'risk_score, ' \ - 'and false_positive' do + 'false_positive, ' \ + 'and undetected_since' do expected_enhancements_in_use = { 'reachability' => ::Enums::Sbom::REACHABILITY_TYPES[::Enums::Sbom::IN_USE], 'token_status' => ::Security::TokenStatus::ACTIVE, @@ -216,7 +238,7 @@ 'policy_violations' => nil, 'risk_score' => 0.6, 'false_positive' => false, - 'undetected_since' => nil + 'undetected_since' => undetected_since_data[vulnerability_reads[1].vulnerability_id] }.with_indifferent_access expected_enhancements_unknown = { diff --git a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb index ad24b418ab14be..1e9efd855206c7 100644 --- a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb +++ b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb @@ -81,6 +81,7 @@ policy_violations: [], risk_score: [], false_positive: [], + undetected_since: [], type: described_class::DOC_TYPE, schema_version: described_class::SCHEMA_VERSION, security_project_tracked_context_id: object.security_project_tracked_context_id -- GitLab From 7085ed2dd25a660936c454a6ea7c7cf06e1fd43d Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 12 Dec 2025 15:31:47 +0100 Subject: [PATCH 10/16] Remove memoization --- ee/app/models/vulnerabilities/finding.rb | 2 +- ee/spec/models/vulnerabilities/finding_spec.rb | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/ee/app/models/vulnerabilities/finding.rb b/ee/app/models/vulnerabilities/finding.rb index 8af09cdd24ed3c..3245a91c030d34 100644 --- a/ee/app/models/vulnerabilities/finding.rb +++ b/ee/app/models/vulnerabilities/finding.rb @@ -618,7 +618,7 @@ def ai_resolution_supported_cwe? end def latest_detection_transition - @latest_detection_transition ||= detection_transitions.last + detection_transitions.last end protected diff --git a/ee/spec/models/vulnerabilities/finding_spec.rb b/ee/spec/models/vulnerabilities/finding_spec.rb index de03ecc866baab..85cab7149b1608 100644 --- a/ee/spec/models/vulnerabilities/finding_spec.rb +++ b/ee/spec/models/vulnerabilities/finding_spec.rb @@ -2045,13 +2045,6 @@ def create_finding(state) it 'returns the last detection transition' do expect(finding.latest_detection_transition).to eq(latest_transition) end - - it 'memoizes the result' do - expect(finding.latest_detection_transition).to eq(latest_transition) - - # Call again to verify memoization - expect(finding.instance_variable_get(:@latest_detection_transition)).to eq(latest_transition) - end end end end -- GitLab From 9a10378110129b95f15207e64a91c2e09976a8ca Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 15 Dec 2025 14:37:54 +0100 Subject: [PATCH 11/16] Go over Bala's review comments --- .../ingestion/mark_as_resolved_service.rb | 3 +- .../mark_resolved_as_detected.rb | 5 +- ...starboard_vulnerability_resolve_service.rb | 6 +- ...ndetected_since_field_to_vulnerability.yml | 2 +- .../vulnerability/undetected_since.rb | 4 +- .../elastic/references/vulnerability.rb | 6 +- .../elastic/references/vulnerability_spec.rb | 55 +++++++++++++++++++ 7 files changed, 68 insertions(+), 13 deletions(-) diff --git a/ee/app/services/security/ingestion/mark_as_resolved_service.rb b/ee/app/services/security/ingestion/mark_as_resolved_service.rb index 1bda2c1785f36c..3867f7c04d7040 100644 --- a/ee/app/services/security/ingestion/mark_as_resolved_service.rb +++ b/ee/app/services/security/ingestion/mark_as_resolved_service.rb @@ -89,6 +89,7 @@ def mark_as_no_longer_detected(vulnerabilities) ::Vulnerabilities::BulkEsOperationService.new(vulnerabilities_relation).execute do |relation| relation.update_all(resolved_on_default_branch: true) + create_detection_transitions(no_longer_detected_vulnerability_ids) end Vulnerabilities::Reads::UpsertService.new(vulnerabilities_relation, @@ -98,8 +99,6 @@ def mark_as_no_longer_detected(vulnerabilities) CreateVulnerabilityRepresentationInformation.execute(pipeline, vulnerabilities_relation) - create_detection_transitions(no_longer_detected_vulnerability_ids) - track_no_longer_detected_vulnerabilities(no_longer_detected_vulnerability_ids.count) end diff --git a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb index 3d20b0b2bf5275..aaa2d11ac5377b 100644 --- a/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb +++ b/ee/app/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected.rb @@ -59,8 +59,9 @@ def update_vulnerability_records ) SecApplicationRecord.current_transaction.after_commit do - ::Vulnerabilities::BulkEsOperationService.new(vulnerabilities_relation).execute(&:itself) - create_detection_transitions(redetected_vulnerability_ids) + ::Vulnerabilities::BulkEsOperationService.new(vulnerabilities_relation).execute do + create_detection_transitions(redetected_vulnerability_ids) + end end end diff --git a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb index a32d96b6157c5c..f803410a14b91b 100644 --- a/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb +++ b/ee/app/services/vulnerabilities/starboard_vulnerability_resolve_service.rb @@ -24,13 +24,10 @@ def initialize(agent, uuids) def execute raise Gitlab::Access::AccessDeniedError unless authorized? - undetected_vulnerability_ids = [] - undetected.each_batch(of: BATCH_SIZE) do |batch| # rubocop:disable CodeReuse/ActiveRecord, Database/AvoidUsingPluckWithoutLimit -- batch limited to 250 batch_ids = batch.pluck(:id) # rubocop:enable CodeReuse/ActiveRecord, Database/AvoidUsingPluckWithoutLimit - undetected_vulnerability_ids.concat(batch_ids) Vulnerabilities::BulkEsOperationService.new(batch).execute do |vulnerabilities| Vulnerability.transaction do @@ -38,11 +35,10 @@ def execute Vulnerabilities::Reads::UpsertService.new(vulnerabilities, { resolved_on_default_branch: true, state: :resolved }, projects: @project).execute end + create_detection_transitions(batch_ids) end end - create_detection_transitions(undetected_vulnerability_ids) - ServiceResponse.success end diff --git a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml index 654b134672cc6b..67c3a936e3c7ed 100644 --- a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml +++ b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml @@ -4,7 +4,7 @@ version: '20251204112110' description: Add undetected_since field to Vulnerabilities ES index group: group::security insights milestone: '18.7' -introduced_by_url: +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215334 obsolete: false marked_obsolete_by_url: marked_obsolete_in_milestone: diff --git a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb index 6cb8bd7a241565..dcd5ee1199ef77 100644 --- a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb +++ b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb @@ -26,7 +26,9 @@ def fetch_undetected_since_data(vulnerability_ids) .with_latest_detection_transition .each_with_object({}) do |finding, result| transition = finding.latest_detection_transition - result[finding.vulnerability_id] = transition&.created_at unless transition.detected? + next if transition.nil? || transition.detected? + + result[finding.vulnerability_id] = transition.created_at end end end diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index d3120edb259e4c..0dbeb92943d270 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -7,7 +7,7 @@ class Vulnerability < Reference include Search::Elastic::Concerns::DatabaseReference include ::Gitlab::Utils::StrongMemoize - SCHEMA_VERSION = 25_49 + SCHEMA_VERSION = 25_50 DOC_TYPE = 'vulnerability' INDEX_NAME = 'vulnerabilities' @@ -159,8 +159,10 @@ def internal_es_fields end def fetch_schema_version - if security_project_tracked_context_id_migration_completed? + if undetected_since_migration_completed? SCHEMA_VERSION + elsif security_project_tracked_context_id_migration_completed? + 25_49 elsif false_positive_migration_completed? 25_48 elsif risk_score_migration_completed? diff --git a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb index 1e9efd855206c7..e152f4f0db6e20 100644 --- a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb +++ b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb @@ -410,6 +410,40 @@ end end + context 'with undetected_since migration mappings' do + context 'when undetected_since migration has finished' do + let(:undetected_since_data) { 2.days.ago } + + before do + set_elasticsearch_migration_to(:add_undetected_since_field_to_vulnerability) + allow(object).to receive(:undetected_since).and_return(undetected_since_data) + allow(vulnerability_reference_object).to receive(:database_record).and_return(object) + end + + it 'returns schema version' do + expect(indexed_json[:schema_version]).to eq(25_50) + end + + it 'includes undetected_since in the indexed json' do + expect(indexed_json[:undetected_since]).to be_within(0.1.seconds).of(undetected_since_data) + end + end + + context 'when migration has not completed' do + before do + set_elasticsearch_migration_to(:add_undetected_since_field_to_vulnerability, including: false) + end + + it 'returns schema version with security_project_tracked_context_id' do + expect(indexed_json[:schema_version]).to eq(25_49) + end + + it 'does not assign undetected_since on the indexed json' do + expect(indexed_json[:undetected_since]).to be_nil + end + end + end + context 'when all migrations have completed' do it 'returns the current schema version' do expect(indexed_json[:schema_version]).to eq(described_class::SCHEMA_VERSION) @@ -531,6 +565,20 @@ end end + context 'with undetected_since data' do + let(:transition_time) { 2.days.ago } + let!(:transition) do + create(:vulnerability_detection_transition, finding: finding, detected: false, created_at: transition_time) + end + + it 'preloads undetected_since' do + described_class.preload_indexing_data(refs) + + expect(vulnerability_ref.database_record.undetected_since).to be_within(0.1.seconds).of(transition_time) + expect(user_vulnerability_ref.database_record.undetected_since).to be_nil + end + end + context 'with token_status data' do let!(:secret_finding) do create( @@ -646,6 +694,10 @@ create(:finding_token_status, finding: finding2, project: project2, status: ::Security::TokenStatus::INACTIVE) + # Create detection transitions + create(:vulnerability_detection_transition, finding: finding1, detected: false, created_at: 2.days.ago) + create(:vulnerability_detection_transition, finding: finding2, detected: false, created_at: 1.day.ago) + # Create refs array for initial batch refs = [ described_class.new(vulnerability_read1.vulnerability_id, vulnerability_read1.es_parent) @@ -692,6 +744,9 @@ # Create finding with token_status create(:finding_token_status, finding: finding3, project: project2, status: ::Security::TokenStatus::UNKNOWN) + # Create detection transition for third finding + create(:vulnerability_detection_transition, finding: finding3, detected: false, created_at: 3.days.ago) + # Add new refs to the array refs += [ described_class.new(vulnerability_read2.vulnerability_id, vulnerability_read2.es_parent), -- GitLab From 58e6df5499a01ecd776b9bc2bc936ddeff86fb67 Mon Sep 17 00:00:00 2001 From: Charlie Kroon Date: Mon, 15 Dec 2025 14:50:39 +0100 Subject: [PATCH 12/16] Apply 6 suggestion(s) to 4 file(s) Co-authored-by: Ravi Kumar --- .../models/vulnerabilities/finding_spec.rb | 6 +++--- .../mark_as_resolved_service_spec.rb | 20 +++++++------------ .../mark_resolved_as_detected_spec.rb | 20 +++++++------------ ...oard_vulnerability_resolve_service_spec.rb | 20 +++++++------------ 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/ee/spec/models/vulnerabilities/finding_spec.rb b/ee/spec/models/vulnerabilities/finding_spec.rb index 85cab7149b1608..9fbe0103e21303 100644 --- a/ee/spec/models/vulnerabilities/finding_spec.rb +++ b/ee/spec/models/vulnerabilities/finding_spec.rb @@ -1997,7 +1997,7 @@ def create_finding(state) let_it_be(:finding2) { create(:vulnerabilities_finding) } let_it_be(:finding3) { create(:vulnerabilities_finding) } - let_it_be(:transition1_old) do + let_it_be(:_transition1_old) do create(:vulnerability_detection_transition, finding: finding1, detected: true, created_at: 2.days.ago) end @@ -2034,12 +2034,12 @@ def create_finding(state) let_it_be(:finding) { create(:vulnerabilities_finding) } context 'when finding has multiple detection transitions' do - let_it_be(:old_transition) do + let_it_be(:_old_transition) do create(:vulnerability_detection_transition, finding: finding, detected: true, created_at: 2.days.ago) end let_it_be(:latest_transition) do - create(:vulnerability_detection_transition, finding: finding, detected: false, created_at: 1.day.ago) + create(:vulnerability_detection_transition, :not_detected, finding: finding, created_at: 1.day.ago) end it 'returns the last detection transition' do diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index 71483ed428b023..602c50178eb1c2 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -53,21 +53,15 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) expect(vulnerability.vulnerability_read.resolved_on_default_branch).to be_truthy end - context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do - before do - stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) - end + it 'creates a detection transition with detected: false' do + finding_ids = vulnerability.findings.pluck(:id) - it 'creates a detection transition with detected: false' do - finding_ids = vulnerability.findings.pluck(:id) + expect { command.execute }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) - expect { command.execute }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(1) - - transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) - expect(transition.detected).to be(false) - end + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) + expect(transition.detected).to be(false) end context 'when there is a no longer detected vulnerability' do diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 89fa6538f14e6b..30dc9669b8775d 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -73,21 +73,15 @@ expect(state_transition.vulnerability_id).to eq(resolved_vulnerability.id) end - context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do - before do - stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) - end + it 'creates a detection transition with detected: true' do + finding_ids = resolved_vulnerability.findings.pluck(:id) - it 'creates a detection transition with detected: true' do - finding_ids = resolved_vulnerability.findings.pluck(:id) + expect { mark_resolved_as_detected }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(1) - expect { mark_resolved_as_detected }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(1) - - transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) - expect(transition.detected).to be(true) - end + transition = Vulnerabilities::DetectionTransition.find_by(vulnerability_occurrence_id: finding_ids) + expect(transition.detected).to be(true) end it 'marks the findings as transitioned_to_detected' do diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index 7614f12ef5ffe1..7b59bda969c3b5 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -55,21 +55,15 @@ )) end - context 'when new_security_dashboard_exclude_no_longer_detected is enabled' do - before do - stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: true) - end + it 'creates detection transitions with detected: false' do + finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } - it 'creates detection transitions with detected: false' do - finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } + expect { service.execute }.to change { + Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count + }.by(undetected_vulnerabilities.count) - expect { service.execute }.to change { - Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids).count - }.by(undetected_vulnerabilities.count) - - transitions = Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids) - expect(transitions).to all(have_attributes(detected: false)) - end + transitions = Vulnerabilities::DetectionTransition.where(vulnerability_occurrence_id: finding_ids) + expect(transitions).to all(have_attributes(detected: false)) end it 'touches the updated_at timestamp', :freeze_time do -- GitLab From 517c646707f0ec6dae07072aee1ec8f740e4c066 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 15 Dec 2025 15:17:32 +0100 Subject: [PATCH 13/16] go over ravi's comments --- debug_json.py | 1 + .../preloaders/vulnerability/undetected_since.rb | 2 ++ .../ingestion/mark_as_resolved_service_spec.rb | 10 ++++++++++ .../mark_resolved_as_detected_spec.rb | 10 ++++++++++ .../starboard_vulnerability_resolve_service_spec.rb | 10 ++++++++++ 5 files changed, 33 insertions(+) create mode 100644 debug_json.py diff --git a/debug_json.py b/debug_json.py new file mode 100644 index 00000000000000..8b137891791fe9 --- /dev/null +++ b/debug_json.py @@ -0,0 +1 @@ + diff --git a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb index dcd5ee1199ef77..b672c845b57fad 100644 --- a/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb +++ b/ee/lib/search/elastic/preloaders/vulnerability/undetected_since.rb @@ -25,6 +25,8 @@ def fetch_undetected_since_data(vulnerability_ids) .by_vulnerability(vulnerability_ids) .with_latest_detection_transition .each_with_object({}) do |finding, result| + next unless finding.vulnerability_id + transition = finding.latest_detection_transition next if transition.nil? || transition.detected? diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index 602c50178eb1c2..5aab14f4b79efc 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -64,6 +64,16 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) expect(transition.detected).to be(false) end + context 'when new_security_dashboard_exclude_no_longer_detected is disabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: false) + end + + it 'does not create detection transitions' do + expect { command.execute }.not_to change { Vulnerabilities::DetectionTransition.count } + end + end + context 'when there is a no longer detected vulnerability' do let_it_be_with_reload(:no_longer_detected) do create(:vulnerability, :sast, diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 30dc9669b8775d..9badefb3cb98cd 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -84,6 +84,16 @@ expect(transition.detected).to be(true) end + context 'when new_security_dashboard_exclude_no_longer_detected is disabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: false) + end + + it 'does not create detection transitions' do + expect { mark_resolved_as_detected }.not_to change { Vulnerabilities::DetectionTransition.count } + end + end + it 'marks the findings as transitioned_to_detected' do expect { mark_resolved_as_detected }.to change { existing_resolved_finding_map.transitioned_to_detected }.to(true) .and not_change { existing_detected_finding_map.transitioned_to_detected } diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index 7b59bda969c3b5..d8ef4481969d6d 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -66,6 +66,16 @@ expect(transitions).to all(have_attributes(detected: false)) end + context 'when new_security_dashboard_exclude_no_longer_detected is disabled' do + before do + stub_feature_flags(new_security_dashboard_exclude_no_longer_detected: false) + end + + it 'does not create detection transitions' do + expect { service.execute }.not_to change { Vulnerabilities::DetectionTransition.count } + end + end + it 'touches the updated_at timestamp', :freeze_time do service.execute -- GitLab From 6950c8c3bdd504820b3f2a0c9f5af2d5cfc0acc3 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 15 Dec 2025 15:30:56 +0100 Subject: [PATCH 14/16] Remove unneccessary file --- debug_json.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 debug_json.py diff --git a/debug_json.py b/debug_json.py deleted file mode 100644 index 8b137891791fe9..00000000000000 --- a/debug_json.py +++ /dev/null @@ -1 +0,0 @@ - -- GitLab From 00b9f2a6e830b09e053e03bcedeb3ddffce25f4a Mon Sep 17 00:00:00 2001 From: Charlie Kroon Date: Tue, 16 Dec 2025 11:20:21 +0100 Subject: [PATCH 15/16] Apply 4 suggestion(s) to 4 file(s) Co-authored-by: Bala Kumar --- ee/spec/models/vulnerabilities/finding_spec.rb | 2 +- .../security/ingestion/mark_as_resolved_service_spec.rb | 2 +- .../ingest_vulnerabilities/mark_resolved_as_detected_spec.rb | 2 +- .../starboard_vulnerability_resolve_service_spec.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/spec/models/vulnerabilities/finding_spec.rb b/ee/spec/models/vulnerabilities/finding_spec.rb index 9fbe0103e21303..0e91610af528e5 100644 --- a/ee/spec/models/vulnerabilities/finding_spec.rb +++ b/ee/spec/models/vulnerabilities/finding_spec.rb @@ -2011,7 +2011,7 @@ def create_finding(state) subject { described_class.with_latest_detection_transition } - it 'includes only the latest detection transition for each finding' do + it 'includes only the latest detection transition for each finding', :aggregate_failures do findings = subject.to_a expect(findings).to contain_exactly(finding1, finding2) diff --git a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb index 5aab14f4b79efc..28237e521b905c 100644 --- a/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb +++ b/ee/spec/services/security/ingestion/mark_as_resolved_service_spec.rb @@ -53,7 +53,7 @@ def expect_vulnerability_not_to_be_resolved(vulnerability) expect(vulnerability.vulnerability_read.resolved_on_default_branch).to be_truthy end - it 'creates a detection transition with detected: false' do + it 'creates a detection transition with detected: false', :aggregate_failures do finding_ids = vulnerability.findings.pluck(:id) expect { command.execute }.to change { diff --git a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb index 9badefb3cb98cd..78c53c5be01139 100644 --- a/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb +++ b/ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/mark_resolved_as_detected_spec.rb @@ -73,7 +73,7 @@ expect(state_transition.vulnerability_id).to eq(resolved_vulnerability.id) end - it 'creates a detection transition with detected: true' do + it 'creates a detection transition with detected: true', :aggregate_failures do finding_ids = resolved_vulnerability.findings.pluck(:id) expect { mark_resolved_as_detected }.to change { diff --git a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb index d8ef4481969d6d..eef9a85238d411 100644 --- a/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb +++ b/ee/spec/services/vulnerabilities/starboard_vulnerability_resolve_service_spec.rb @@ -55,7 +55,7 @@ )) end - it 'creates detection transitions with detected: false' do + it 'creates detection transitions with detected: false', :aggregate_failures do finding_ids = undetected_vulnerabilities.flat_map { |v| v.findings.pluck(:id) } expect { service.execute }.to change { -- GitLab From fc24fb9e951334f3545d7e42f99a48c383a206a8 Mon Sep 17 00:00:00 2001 From: Charlie Kroon Date: Wed, 17 Dec 2025 08:22:51 +0100 Subject: [PATCH 16/16] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Bala Kumar --- ...251204112110_add_undetected_since_field_to_vulnerability.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml index 67c3a936e3c7ed..c66230d186d500 100644 --- a/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml +++ b/ee/elastic/docs/20251204112110_add_undetected_since_field_to_vulnerability.yml @@ -3,7 +3,7 @@ name: AddUndetectedSinceFieldToVulnerability version: '20251204112110' description: Add undetected_since field to Vulnerabilities ES index group: group::security insights -milestone: '18.7' +milestone: '18.8' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215334 obsolete: false marked_obsolete_by_url: -- GitLab