From 3c92f9a72602370259705d0c19cfc78273c67c68 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 8 Aug 2025 15:53:16 +0200 Subject: [PATCH 01/13] WIP - Create feature flag --- .../resolve_vulnerability_claude_4_0_rollout.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml diff --git a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml new file mode 100644 index 00000000000000..e2793a2bb57c61 --- /dev/null +++ b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml @@ -0,0 +1,9 @@ +--- +name: resolve_vulnerability_claude_4_0_rollout +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/545698 +introduced_by_url: xxx +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/560585 +milestone: '18.4' +group: group::security insights +type: gitlab_com_derisk +default_enabled: false -- GitLab From 0b72cdfc5074f9fc93e3c9f1dc62a3f9dfadbb37 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Wed, 13 Aug 2025 16:37:22 +0200 Subject: [PATCH 02/13] Modify model version --- .../completions/resolve_vulnerability.rb | 4 +-- .../namespace_feature_settings.rb | 4 +-- .../completions/resolve_vulnerability_spec.rb | 6 ++-- .../resolve_vulnerability/shared_examples.rb | 34 +++++++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index d5f0a0021439b4..3fcaa96216e77b 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -14,7 +14,7 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) - DEV_PROMPT = "^1.0.1" + DEV_PROMPT = "1.0.2" override :execute def execute @@ -61,7 +61,7 @@ def root_namespace override :prompt_version def prompt_version - '1.0.1' + DEV_PROMPT if Feature.enabled?(:resolve_vulnerability_claude_4_0_rollout, user) end def request! diff --git a/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb b/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb index cf5ea238f919c7..47c72465db3239 100644 --- a/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb +++ b/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb @@ -72,8 +72,8 @@ }, { "feature_setting" => "resolve_vulnerability", - "default_model" => "claude_sonnet_3_7", - "selectable_models" => %w[claude_sonnet_3_7 claude-3-5-sonnet-20240620], + "default_model" => "claude_sonnet_4_0", + "selectable_models" => %w[claude_sonnet_3_7 claude-3-5-sonnet-20240620 claude_sonnet_4_0], "beta_models" => [], "unit_primitives" => ["resolve_vulnerability"] }, diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index 7432e84a50a638..aa9a08ba2629d9 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -13,7 +13,7 @@ let(:finding_location_file) { 'main.c' } let(:changed_code) { "somecode\nexecute" } let(:content) { "```\n#{changed_code}\n``` and a ```\nsecond code block\n```" } - let(:prompt_version) { "1.0.1" } + let(:prompt_version) { "1.0.2" } let_it_be(:source_code) { "#include \n\nint main() { printf(\"hello, world!\"); }" } let_it_be(:project) do @@ -108,6 +108,7 @@ def allow_llm_client_to_return_message(response) describe '#execute' do before do stub_feature_flags(ai_model_switching: false) + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) end it_behaves_like 'a resolve vulnerability completion' do @@ -131,6 +132,7 @@ def allow_llm_client_to_return_message(response) before do stub_feature_flags(ai_model_switching: true) + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) end context 'when the model is pinned to a specific model' do @@ -146,7 +148,7 @@ def allow_llm_client_to_return_message(response) let(:model_metadata) do { feature_setting: feature_setting_name, - identifier: 'claude_sonnet_3_7', + identifier: 'claude_sonnet_4_0', provider: 'gitlab' } end diff --git a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb index 1d915b6cb77fd8..4c1afcca451e72 100644 --- a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb +++ b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb @@ -81,6 +81,8 @@ def expect_tracked_internal_event(event_name, status) before do stub_licensed_features(security_dashboard: true) + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + [:admin_all_resources, :resolve_vulnerability_with_ai].each do |permission| allow(user).to receive(:can?).with(permission).and_return(true) allow(user2).to receive(:can?).with(permission).and_return(true) @@ -315,6 +317,38 @@ def expect_tracked_internal_event(event_name, status) end end + context "when passing in prompt version for Claude 4.0" do + let(:prompt_version) { "1.0.2" } + + before do + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + end + + it 'requests that a MR be created with the extracted patch' do + resolve.execute + + expect(merge_request_service).to have_received(:new).with( + project, + vulnerability, + user, + llm_patch: code_patch, + description_options: description_options + ) + end + + it 'publishes the created merge request for the fix' do + resolve.execute + + expect_published_graphql_content(mr_url) + end + + it 'tracks internal event with success' do + expect_tracked_internal_event("track_mr_creation_from_vr", "success") + + resolve.execute + end + end + context 'when the AIGW responds with a typed code block' do let(:content) { "```java\n#{code_patch}\n```" } -- GitLab From bd9617ff9d3028cfa53051befe1c3be49d4ca469 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 15 Aug 2025 14:56:11 +0200 Subject: [PATCH 03/13] Modify dev prompt --- .../gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index 3fcaa96216e77b..3ee3f95bb6ef46 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -14,7 +14,7 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) - DEV_PROMPT = "1.0.2" + DEV_PROMPT = "^1.0.0" override :execute def execute -- GitLab From 91af49e5f286bd49157e4130e08af63894bd683e Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 15 Aug 2025 15:21:15 +0200 Subject: [PATCH 04/13] Modify versioning in spec + shared exampels --- .../llm/ai_gateway/completions/resolve_vulnerability_spec.rb | 2 +- .../llm/completions/resolve_vulnerability/shared_examples.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index aa9a08ba2629d9..6114bf4cfc6ab1 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -13,7 +13,7 @@ let(:finding_location_file) { 'main.c' } let(:changed_code) { "somecode\nexecute" } let(:content) { "```\n#{changed_code}\n``` and a ```\nsecond code block\n```" } - let(:prompt_version) { "1.0.2" } + let(:prompt_version) { "^1.0.0" } let_it_be(:source_code) { "#include \n\nint main() { printf(\"hello, world!\"); }" } let_it_be(:project) do diff --git a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb index 4c1afcca451e72..dd5e9c9bf824a6 100644 --- a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb +++ b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb @@ -318,7 +318,7 @@ def expect_tracked_internal_event(event_name, status) end context "when passing in prompt version for Claude 4.0" do - let(:prompt_version) { "1.0.2" } + let(:prompt_version) { "^1.0.0" } before do stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) -- GitLab From 1bbace8b4537162361e527800f196aa21c84dd2b Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 18 Aug 2025 10:21:08 +0200 Subject: [PATCH 05/13] Make spec work --- .../ai/model_selection/namespace_feature_settings.rb | 4 ++-- .../llm/ai_gateway/completions/resolve_vulnerability_spec.rb | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb b/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb index 47c72465db3239..5c7fcdd350ba34 100644 --- a/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb +++ b/ee/spec/factories/ai/model_selection/namespace_feature_settings.rb @@ -72,8 +72,8 @@ }, { "feature_setting" => "resolve_vulnerability", - "default_model" => "claude_sonnet_4_0", - "selectable_models" => %w[claude_sonnet_3_7 claude-3-5-sonnet-20240620 claude_sonnet_4_0], + "default_model" => "claude_sonnet_4_20250514", + "selectable_models" => %w[claude_sonnet_3_7 claude-3-5-sonnet-20240620 claude_sonnet_4_20250514], "beta_models" => [], "unit_primitives" => ["resolve_vulnerability"] }, diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index 6114bf4cfc6ab1..5df1368b501daf 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -139,7 +139,8 @@ def allow_llm_client_to_return_message(response) before do create(:ai_namespace_feature_setting, namespace: group, - feature: feature_setting_name + feature: 'resolve_vulnerability', + offered_model_ref: 'claude_sonnet_4_20250514' ) end @@ -148,7 +149,7 @@ def allow_llm_client_to_return_message(response) let(:model_metadata) do { feature_setting: feature_setting_name, - identifier: 'claude_sonnet_4_0', + identifier: 'claude_sonnet_4_20250514', provider: 'gitlab' } end -- GitLab From b1e799cf70bc1dcb40ab4d30201ea9e4e076de76 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 18 Aug 2025 14:49:02 +0200 Subject: [PATCH 06/13] Modify logic --- .../completions/resolve_vulnerability.rb | 7 ++++++- .../completions/resolve_vulnerability_spec.rb | 15 ++++++++++++++- .../resolve_vulnerability/shared_examples.rb | 4 +--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index 3ee3f95bb6ef46..d9e53cc7d093f7 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -15,6 +15,7 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) DEV_PROMPT = "^1.0.0" + DEV_PROMPT_2 = "1.0.2" override :execute def execute @@ -61,7 +62,11 @@ def root_namespace override :prompt_version def prompt_version - DEV_PROMPT if Feature.enabled?(:resolve_vulnerability_claude_4_0_rollout, user) + if Feature.enabled?(:resolve_vulnerability_claude_4_0_rollout, user) + DEV_PROMPT_2 + else + DEV_PROMPT + end end def request! diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index 5df1368b501daf..48e2afc578cf22 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -108,13 +108,25 @@ def allow_llm_client_to_return_message(response) describe '#execute' do before do stub_feature_flags(ai_model_switching: false) - stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: false) end it_behaves_like 'a resolve vulnerability completion' do let(:expected_description) { vulnerability.description } end + context 'when resolve_vulnerability_claude_4_0_rollout is enabled' do + before do + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + end + + let(:prompt_version) { "1.0.2" } + + it_behaves_like 'a resolve vulnerability completion' do + let(:expected_description) { vulnerability.description } + end + end + context 'when the vulnerability description is nil' do it_behaves_like 'a resolve vulnerability completion' do let(:vulnerability_description) { nil } @@ -145,6 +157,7 @@ def allow_llm_client_to_return_message(response) end it_behaves_like 'a resolve vulnerability completion' do + let(:prompt_version) { "1.0.2" } let(:expected_description) { vulnerability.description } let(:model_metadata) do { diff --git a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb index dd5e9c9bf824a6..8ebb6d3a5c764f 100644 --- a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb +++ b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb @@ -81,8 +81,6 @@ def expect_tracked_internal_event(event_name, status) before do stub_licensed_features(security_dashboard: true) - stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) - [:admin_all_resources, :resolve_vulnerability_with_ai].each do |permission| allow(user).to receive(:can?).with(permission).and_return(true) allow(user2).to receive(:can?).with(permission).and_return(true) @@ -318,7 +316,7 @@ def expect_tracked_internal_event(event_name, status) end context "when passing in prompt version for Claude 4.0" do - let(:prompt_version) { "^1.0.0" } + let(:prompt_version) { "1.0.2" } before do stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) -- GitLab From f116f16584d98d805b9108c698a0879054edd473 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 18 Aug 2025 16:04:18 +0200 Subject: [PATCH 07/13] Clean up spec --- .../completions/resolve_vulnerability_spec.rb | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index 48e2afc578cf22..c49e0f16b7a03d 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -115,18 +115,6 @@ def allow_llm_client_to_return_message(response) let(:expected_description) { vulnerability.description } end - context 'when resolve_vulnerability_claude_4_0_rollout is enabled' do - before do - stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) - end - - let(:prompt_version) { "1.0.2" } - - it_behaves_like 'a resolve vulnerability completion' do - let(:expected_description) { vulnerability.description } - end - end - context 'when the vulnerability description is nil' do it_behaves_like 'a resolve vulnerability completion' do let(:vulnerability_description) { nil } @@ -144,30 +132,56 @@ def allow_llm_client_to_return_message(response) before do stub_feature_flags(ai_model_switching: true) - stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: false) end context 'when the model is pinned to a specific model' do before do create(:ai_namespace_feature_setting, namespace: group, - feature: 'resolve_vulnerability', - offered_model_ref: 'claude_sonnet_4_20250514' + feature: feature_setting_name ) end it_behaves_like 'a resolve vulnerability completion' do - let(:prompt_version) { "1.0.2" } let(:expected_description) { vulnerability.description } let(:model_metadata) do { feature_setting: feature_setting_name, - identifier: 'claude_sonnet_4_20250514', + identifier: 'claude_sonnet_3_7', provider: 'gitlab' } end end end + + context 'when resolve_vulnerability_claude_4_0_rollout is enabled' do + before do + stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) + end + + context 'when the model is pinned to a specific model' do + before do + create(:ai_namespace_feature_setting, + namespace: group, + feature: 'resolve_vulnerability', + offered_model_ref: 'claude_sonnet_4_20250514' + ) + end + + it_behaves_like 'a resolve vulnerability completion' do + let(:prompt_version) { "1.0.2" } + let(:expected_description) { vulnerability.description } + let(:model_metadata) do + { + feature_setting: feature_setting_name, + identifier: 'claude_sonnet_4_20250514', + provider: 'gitlab' + } + end + end + end + end end end end -- GitLab From eaf237b6bde7caeae37858d53876b6017ec84af7 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 18 Aug 2025 16:07:17 +0200 Subject: [PATCH 08/13] Remove redundant shared example --- .../resolve_vulnerability/shared_examples.rb | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb index 8ebb6d3a5c764f..1d915b6cb77fd8 100644 --- a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb +++ b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb @@ -315,38 +315,6 @@ def expect_tracked_internal_event(event_name, status) end end - context "when passing in prompt version for Claude 4.0" do - let(:prompt_version) { "1.0.2" } - - before do - stub_feature_flags(resolve_vulnerability_claude_4_0_rollout: true) - end - - it 'requests that a MR be created with the extracted patch' do - resolve.execute - - expect(merge_request_service).to have_received(:new).with( - project, - vulnerability, - user, - llm_patch: code_patch, - description_options: description_options - ) - end - - it 'publishes the created merge request for the fix' do - resolve.execute - - expect_published_graphql_content(mr_url) - end - - it 'tracks internal event with success' do - expect_tracked_internal_event("track_mr_creation_from_vr", "success") - - resolve.execute - end - end - context 'when the AIGW responds with a typed code block' do let(:content) { "```java\n#{code_patch}\n```" } -- GitLab From dc89902a030195c3803736764793ceb7b184afeb Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Mon, 18 Aug 2025 16:16:58 +0200 Subject: [PATCH 09/13] better naming --- .../llm/ai_gateway/completions/resolve_vulnerability.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index d9e53cc7d093f7..2c549d2eb0c89f 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -14,8 +14,8 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) - DEV_PROMPT = "^1.0.0" - DEV_PROMPT_2 = "1.0.2" + CLAUDE_3_5_PROMPT_VERSION = "^1.0.0" + CLAUDE_4_0_PROMPT_VERSION = "1.0.2" override :execute def execute @@ -63,9 +63,9 @@ def root_namespace override :prompt_version def prompt_version if Feature.enabled?(:resolve_vulnerability_claude_4_0_rollout, user) - DEV_PROMPT_2 + CLAUDE_4_0_PROMPT_VERSION else - DEV_PROMPT + CLAUDE_3_5_PROMPT_VERSION end end -- GitLab From 02be8c6f9147d7829650d706642cc048d86dac74 Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Thu, 21 Aug 2025 14:57:23 +0200 Subject: [PATCH 10/13] Add introduction url --- .../resolve_vulnerability_claude_4_0_rollout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml index e2793a2bb57c61..c869ac57c64016 100644 --- a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml +++ b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_4_0_rollout.yml @@ -1,7 +1,7 @@ --- name: resolve_vulnerability_claude_4_0_rollout feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/545698 -introduced_by_url: xxx +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/200813 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/560585 milestone: '18.4' group: group::security insights -- GitLab From 57ce9e17c69f4baadb42886bd9b1d26811295def Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Thu, 21 Aug 2025 15:28:37 +0200 Subject: [PATCH 11/13] Remove ^ --- .../gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index 2c549d2eb0c89f..0d9dc3aaebb5dc 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -14,7 +14,7 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) - CLAUDE_3_5_PROMPT_VERSION = "^1.0.0" + CLAUDE_3_5_PROMPT_VERSION = "1.0.1" CLAUDE_4_0_PROMPT_VERSION = "1.0.2" override :execute -- GitLab From 19eda38bcf634b1c06674078332f70c4dc55615a Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Fri, 22 Aug 2025 16:23:45 +0200 Subject: [PATCH 12/13] Add spec case for not a pinned model --- .../completions/resolve_vulnerability_spec.rb | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index c49e0f16b7a03d..56ec9a677c02f2 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -13,7 +13,7 @@ let(:finding_location_file) { 'main.c' } let(:changed_code) { "somecode\nexecute" } let(:content) { "```\n#{changed_code}\n``` and a ```\nsecond code block\n```" } - let(:prompt_version) { "^1.0.0" } + let(:prompt_version) { "1.0.1" } let_it_be(:source_code) { "#include \n\nint main() { printf(\"hello, world!\"); }" } let_it_be(:project) do @@ -181,6 +181,28 @@ def allow_llm_client_to_return_message(response) end end end + + context 'when the model is not pinned to a specific model' do + before do + create(:ai_namespace_feature_setting, + namespace: group, + feature: 'resolve_vulnerability', + offered_model_ref: nil + ) + end + + it_behaves_like 'a resolve vulnerability completion' do + let(:prompt_version) { "1.0.2" } + let(:expected_description) { vulnerability.description } + let(:model_metadata) do + { + feature_setting: feature_setting_name, + identifier: 'claude_sonnet_4_20250514', + provider: 'gitlab' + } + end + end + end end end end -- GitLab From 938b7f0705afe1d5a69426aae5d18627aa7e05bf Mon Sep 17 00:00:00 2001 From: charlieeekroon Date: Tue, 9 Sep 2025 13:41:56 +0200 Subject: [PATCH 13/13] Fix for spec --- .../concerns/ai/model_selection/features_configurable.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ee/app/models/concerns/ai/model_selection/features_configurable.rb b/ee/app/models/concerns/ai/model_selection/features_configurable.rb index 72fab0caaf9e3f..cc64241fe579e1 100644 --- a/ee/app/models/concerns/ai/model_selection/features_configurable.rb +++ b/ee/app/models/concerns/ai/model_selection/features_configurable.rb @@ -132,7 +132,7 @@ def model_metadata_params { provider: provider, feature_setting: feature, - identifier: offered_model_ref + identifier: offered_model_ref || claude_4_0_rollout_default_model } end @@ -142,6 +142,13 @@ def model_request_params private + def claude_4_0_rollout_default_model + return unless feature.to_s == 'resolve_vulnerability' + return unless Feature.enabled?(:resolve_vulnerability_claude_4_0_rollout, model_selection_scope) + + 'claude_sonnet_4_20250514' + end + def validate_model_selection_enabled return if model_selection_enabled? -- GitLab