From d175391b7782aa9dacc986f852f8db8291b844d8 Mon Sep 17 00:00:00 2001 From: Nathan Weinshenker Date: Wed, 5 Mar 2025 16:41:29 -0700 Subject: [PATCH 1/2] Resolve vulnerability using Claude 3.7 Sonnet prompt version moves the VR tool to use the pinned dev prompt version in the prompt registry Changelog: added EE: true --- ...solve_vulnerability_claude_3_7_rollout.yml | 9 +++++ .../completions/resolve_vulnerability.rb | 7 ++++ .../completions/resolve_vulnerability_spec.rb | 25 +++++++++----- .../resolve_vulnerability/shared_examples.rb | 33 +++++++++++++++++++ 4 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml diff --git a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml new file mode 100644 index 00000000000000..45255ecc506766 --- /dev/null +++ b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml @@ -0,0 +1,9 @@ +--- +name: resolve_vulnerability_claude_3_7_rollout +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/523496 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/183607 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/523503 +milestone: '17.10' +group: group::security insights +type: gitlab_com_derisk +default_enabled: false diff --git a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb index 41800a443e5b17..a0f0d8f1f1a903 100644 --- a/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb +++ b/ee/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability.rb @@ -14,6 +14,8 @@ class ResolveVulnerability < Base EmptyResponseError = Class.new(StandardError) + DEV_PROMPT = "0.0.1-dev" + override :execute def execute return unless valid? @@ -52,6 +54,11 @@ def inputs private + override :prompt_version + def prompt_version + DEV_PROMPT if Feature.enabled?(:resolve_vulnerability_claude_3_7_rollout, user) + end + def request! response = perform_ai_gateway_request! diff --git a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb index 89f8c81600ac0e..60122b7e7784fd 100644 --- a/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb +++ b/ee/spec/lib/gitlab/llm/ai_gateway/completions/resolve_vulnerability_spec.rb @@ -25,6 +25,8 @@ build(:ai_message, :resolve_vulnerability, user: user, resource: vulnerability, request_id: 'uuid') end + let(:prompt_version) { nil } + let(:options) { {} } let(:logger) { instance_double(Gitlab::Llm::Logger) } @@ -61,21 +63,26 @@ def allow_llm_client_to_raise_error(error) def allow_llm_client_to_return_message(response) identifier_names = vulnerability.finding.identifier_names.join("\n* ") + request_body = { + 'prompt_version' => prompt_version, + 'inputs' => { + filename: finding_location_file, + identifiers: "\n * #{identifier_names}\n", + name: vulnerability.title, + source_code: source_code, + vulnerability_description: vulnerability.description, + vulnerable_code: source_code + } + }.compact + expect_next_instance_of(Gitlab::Llm::AiGateway::Client) do |client| expect(client).to receive(:complete_prompt) .with( base_url: Gitlab::AiGateway.url, prompt_name: :resolve_vulnerability, - inputs: { - filename: finding_location_file, - identifiers: "\n * #{identifier_names}\n", - name: vulnerability.title, - source_code: source_code, - vulnerability_description: vulnerability.description, - vulnerable_code: source_code - }, + inputs: request_body['inputs'], model_metadata: nil, - prompt_version: nil + prompt_version: request_body['prompt_version'] ) .and_return(response) end diff --git a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb index 9704e33a4d2c71..cb747bbfd2e947 100644 --- a/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb +++ b/ee/spec/lib/gitlab/llm/completions/resolve_vulnerability/shared_examples.rb @@ -65,6 +65,7 @@ def expect_tracked_internal_event(event_name, status) RSpec.shared_examples "a resolve vulnerability completion" do before do stub_licensed_features(security_dashboard: true) + stub_feature_flags(resolve_vulnerability_claude_3_7_rollout: false) [:admin_all_resources, :resolve_vulnerability_with_ai].each do |permission| allow(user).to receive(:can?).with(permission).and_return(true) @@ -300,6 +301,38 @@ def expect_tracked_internal_event(event_name, status) end end + context "when passing in prompt version for Claude 3.7" do + let(:prompt_version) { "0.0.1-dev" } + + before do + stub_feature_flags(resolve_vulnerability_claude_3_7_rollout: true) + end + + it 'requests that a MR be created with the extracted patch' do + resolve.execute + + expect(merge_request_service).to have_received(:new).with( + project, + vulnerability, + user, + llm_patch: code_patch, + description_options: description_options + ) + end + + it 'publishes the created merge request for the fix' do + resolve.execute + + expect_published_graphql_content(mr_url) + end + + it 'tracks internal event with success' do + expect_tracked_internal_event("track_mr_creation_from_vr", "success") + + resolve.execute + end + end + context 'when the AIGW responds with a typed code block' do let(:content) { "```java\n#{code_patch}\n```" } -- GitLab From e20b1d9c703e129da4ea686168efccc99f28ddb3 Mon Sep 17 00:00:00 2001 From: Nathan Weinshenker Date: Mon, 17 Mar 2025 09:41:11 -0700 Subject: [PATCH 2/2] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Michael Becker --- .../resolve_vulnerability_claude_3_7_rollout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml index 45255ecc506766..50a1877a6c284b 100644 --- a/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml +++ b/ee/config/feature_flags/gitlab_com_derisk/resolve_vulnerability_claude_3_7_rollout.yml @@ -3,7 +3,7 @@ name: resolve_vulnerability_claude_3_7_rollout feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/523496 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/183607 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/523503 -milestone: '17.10' +milestone: '17.11' group: group::security insights type: gitlab_com_derisk default_enabled: false -- GitLab