From f2af7ed1c68fc551bb488d22581384ace73bed8a Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Thu, 23 Oct 2025 15:22:47 +0200 Subject: [PATCH 1/7] Add detected_at field to ES vulnerability index --- ...152050_add_detected_at_field_to_vulnerability.yml | 10 ++++++++++ ...3152050_add_detected_at_field_to_vulnerability.rb | 4 ++++ ee/lib/search/elastic/references/vulnerability.rb | 12 ++++++++++-- ee/lib/search/elastic/types/vulnerability.rb | 1 + ...50_add_detected_at_field_to_vulnerability_spec.rb | 10 ++++++++++ 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml create mode 100644 ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb create mode 100644 ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb diff --git a/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml b/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml new file mode 100644 index 00000000000000..550e062c4f4e20 --- /dev/null +++ b/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml @@ -0,0 +1,10 @@ +--- +name: AddDetectedAtFieldToVulnerability +version: '20251023152050' +description: Adding the detected_at field to Vulnerability +group: group::security insights +milestone: '18.6' +introduced_by_url: +obsolete: false +marked_obsolete_by_url: +marked_obsolete_in_milestone: diff --git a/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb new file mode 100644 index 00000000000000..ce629e2bc3cdc5 --- /dev/null +++ b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class AddDetectedAtFieldToVulnerability < Elastic::Migration +end diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index 3d426a210f0135..a7c22a4e82a87e 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -7,7 +7,7 @@ class Vulnerability < Reference include Search::Elastic::Concerns::DatabaseReference include ::Gitlab::Utils::StrongMemoize - SCHEMA_VERSION = 25_43 + SCHEMA_VERSION = 25_44 DOC_TYPE = 'vulnerability' INDEX_NAME = 'vulnerabilities' @@ -83,6 +83,8 @@ def as_indexed_json fields.merge!(build_other_fields) + fields["detected_at"] = database_record.vulnerability.detected_at if detected_at_migration_completed? + fields["risk_score"] = fetch_record_attribute(database_record, :risk_score) if risk_score_migration_completed? fields["reachability"] = fetch_record_attribute(database_record, :reachability) @@ -143,8 +145,10 @@ def internal_es_fields end def fetch_schema_version - if risk_score_migration_completed? + if detected_at_migration_completed? SCHEMA_VERSION + elsif risk_score_migration_completed? + 25_43 elsif policy_violations_migration_finished? 25_42 elsif token_status_migration_finished? @@ -175,6 +179,10 @@ def risk_score_migration_completed? ::Elastic::DataMigrationService.migration_has_finished?(:add_risk_score_field_to_vulnerability) end + def detected_at_migration_completed? + ::Elastic::DataMigrationService.migration_has_finished?(:add_detected_at_field_to_vulnerability) + end + # # Private class methods for implementation details # diff --git a/ee/lib/search/elastic/types/vulnerability.rb b/ee/lib/search/elastic/types/vulnerability.rb index deb2cd3a23fa48..c833ddd8c93900 100644 --- a/ee/lib/search/elastic/types/vulnerability.rb +++ b/ee/lib/search/elastic/types/vulnerability.rb @@ -30,6 +30,7 @@ def base_mappings vulnerability_id: { type: 'long' }, created_at: { type: 'date' }, updated_at: { type: 'date' }, + detected_at: { type: 'date' }, project_id: { type: 'long' }, scanner_id: { type: 'long' }, scanner_external_id: { type: 'keyword' }, diff --git a/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb b/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb new file mode 100644 index 00000000000000..d8d82604d3763f --- /dev/null +++ b/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' +require File.expand_path('ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb') + +# See https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#elasticsearch-specs +# for more information on how to write search migration specs for GitLab. +RSpec.describe AddDetectedAtFieldToVulnerability, feature_category: :security_insights do + let(:version) { 20251023152050 } +end -- GitLab From 3fb3c82a30467d74f59a17a22713104b4e25a9f4 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Thu, 23 Oct 2025 15:30:09 +0200 Subject: [PATCH 2/7] modify migration spec --- ...2050_add_detected_at_field_to_vulnerability.yml | 2 +- ...52050_add_detected_at_field_to_vulnerability.rb | 14 ++++++++++++++ ..._add_detected_at_field_to_vulnerability_spec.rb | 6 +++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml b/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml index 550e062c4f4e20..03515d536200f1 100644 --- a/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml +++ b/ee/elastic/docs/20251023152050_add_detected_at_field_to_vulnerability.yml @@ -4,7 +4,7 @@ version: '20251023152050' description: Adding the detected_at field to Vulnerability group: group::security insights milestone: '18.6' -introduced_by_url: +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/209927 obsolete: false marked_obsolete_by_url: marked_obsolete_in_milestone: diff --git a/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb index ce629e2bc3cdc5..2565c5296ec981 100644 --- a/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb +++ b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb @@ -1,4 +1,18 @@ # frozen_string_literal: true class AddDetectedAtFieldToVulnerability < Elastic::Migration + include ::Search::Elastic::MigrationUpdateMappingsHelper + + DOCUMENT_TYPE = Vulnerability + + private + + def new_mappings + { + detected_at: { + type: 'date' + } + } + end +end end diff --git a/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb b/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb index d8d82604d3763f..82836080e7b33f 100644 --- a/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb +++ b/ee/spec/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' require File.expand_path('ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb') -# See https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#elasticsearch-specs -# for more information on how to write search migration specs for GitLab. -RSpec.describe AddDetectedAtFieldToVulnerability, feature_category: :security_insights do +RSpec.describe AddDetectedAtFieldToVulnerability, :elastic, feature_category: :vulnerability_management do let(:version) { 20251023152050 } + + include_examples 'migration adds mapping' end -- GitLab From c5e0b96c5574046c58475826d28a81dfdb0a1abe Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 24 Oct 2025 12:37:07 +0200 Subject: [PATCH 3/7] Rubocop --- .../20251023152050_add_detected_at_field_to_vulnerability.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb index 2565c5296ec981..86b1d2e1df1936 100644 --- a/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb +++ b/ee/elastic/migrate/20251023152050_add_detected_at_field_to_vulnerability.rb @@ -15,4 +15,3 @@ def new_mappings } end end -end -- GitLab From adb381503ee9fdea65f330162a2c19b5fd9ba476 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 24 Oct 2025 12:42:39 +0200 Subject: [PATCH 4/7] Add detected_at to references/vulnerability spec --- ee/spec/lib/search/elastic/references/vulnerability_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb index a3102a04d1ba5f..e28900b96fbf36 100644 --- a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb +++ b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb @@ -52,6 +52,7 @@ vulnerability_id: object.vulnerability_id, created_at: be_within(0.1.seconds).of(object.vulnerability.created_at), updated_at: be_within(0.1.seconds).of(object.vulnerability.updated_at), + detected_at: be_within(0.1.seconds).of(object.vulnerability.detected_at), resolved_at: nil, dismissed_at: nil, project_id: object.project_id, -- GitLab From 6275dfbcb5fede145596bd1e4031a70f5b2be901 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 24 Oct 2025 12:50:18 +0200 Subject: [PATCH 5/7] Small modifuication --- ee/lib/search/elastic/references/vulnerability.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index a7c22a4e82a87e..7f0b902b44e9af 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -83,7 +83,10 @@ def as_indexed_json fields.merge!(build_other_fields) - fields["detected_at"] = database_record.vulnerability.detected_at if detected_at_migration_completed? + if detected_at_migration_completed? + fields["detected_at"] = + fetch_record_attribute(database_record, :detected_at) + end fields["risk_score"] = fetch_record_attribute(database_record, :risk_score) if risk_score_migration_completed? -- GitLab From c3b38670d7b5e2e6e195d2fce39ca95479ae3609 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 24 Oct 2025 12:55:27 +0200 Subject: [PATCH 6/7] Undo last change and use database_record --- ee/lib/search/elastic/references/vulnerability.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ee/lib/search/elastic/references/vulnerability.rb b/ee/lib/search/elastic/references/vulnerability.rb index 7f0b902b44e9af..a7c22a4e82a87e 100644 --- a/ee/lib/search/elastic/references/vulnerability.rb +++ b/ee/lib/search/elastic/references/vulnerability.rb @@ -83,10 +83,7 @@ def as_indexed_json fields.merge!(build_other_fields) - if detected_at_migration_completed? - fields["detected_at"] = - fetch_record_attribute(database_record, :detected_at) - end + fields["detected_at"] = database_record.vulnerability.detected_at if detected_at_migration_completed? fields["risk_score"] = fetch_record_attribute(database_record, :risk_score) if risk_score_migration_completed? -- GitLab From 5953d77e82d06f400b253157dd17fd81c9cf7d2b Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 27 Oct 2025 15:22:39 +0100 Subject: [PATCH 7/7] Add extra spec --- .../elastic/references/vulnerability_spec.rb | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb index e28900b96fbf36..9aa1108f1f54f7 100644 --- a/ee/spec/lib/search/elastic/references/vulnerability_spec.rb +++ b/ee/spec/lib/search/elastic/references/vulnerability_spec.rb @@ -303,6 +303,46 @@ end end + context 'with detected_at mappings' do + before do + set_elasticsearch_migration_to(:add_detected_at_field_to_vulnerability) + end + + context 'when migrations are completed' do + context 'with detected_at' do + let(:detected_at) { 2.days.ago } + let(:detected_vulnerability) do + create(:vulnerability, :with_read, project: project, detected_at: detected_at) + end + + let(:detected_vulnerability_read) { detected_vulnerability.vulnerability_read } + let(:object) { detected_vulnerability_read } + + it 'sets detected_at field on the indexed json' do + expect(indexed_json[:detected_at]).to be_within(0.1.seconds).of(detected_at) + end + + it 'returns schema version' do + expect(indexed_json[:schema_version]).to eq(25_44) + end + end + + context 'when the detected_at migration has not completed' do + before do + set_elasticsearch_migration_to(:add_detected_at_field_to_vulnerability, including: false) + end + + it 'returns schema version with risk_score only' do + expect(indexed_json[:schema_version]).to eq(25_43) + end + + it 'does not assign detected_at on the indexed json', :aggregate_failures do + expect(indexed_json[:detected_at]).to be_nil + end + end + end + end + context 'when all migrations have completed' do it 'returns the current schema version' do expect(indexed_json[:schema_version]).to eq(described_class::SCHEMA_VERSION) -- GitLab