From 36943003429e1a2467fbeef8a376f033d65d6709 Mon Sep 17 00:00:00 2001 From: lesley Date: Sun, 26 May 2024 01:44:03 -0500 Subject: [PATCH 1/6] Add ability to add experimental feature flag prompt changes This MR makes it easier for anyone contributing to Duo Chat to make changes under FF. --- .../beta/prevent_issue_epic_search.yml | 9 +++ ee/app/models/ai/ai_resource/epic.rb | 2 +- ee/app/models/ai/ai_resource/issue.rb | 2 +- .../llm/chain/agents/zero_shot/executor.rb | 64 ++++++++++++++- .../llm/chain/tools/epic_reader/executor.rb | 17 ++++ .../llm/chain/tools/issue_reader/executor.rb | 17 ++++ ee/lib/gitlab/llm/chain/tools/tool.rb | 10 ++- .../chain/agents/zero_shot/executor_spec.rb | 79 ++++++++++++++++++- .../lib/gitlab/llm/chain/tools/tool_spec.rb | 52 +++++++++++- 9 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 config/feature_flags/beta/prevent_issue_epic_search.yml diff --git a/config/feature_flags/beta/prevent_issue_epic_search.yml b/config/feature_flags/beta/prevent_issue_epic_search.yml new file mode 100644 index 00000000000000..d6bdc28ac663cf --- /dev/null +++ b/config/feature_flags/beta/prevent_issue_epic_search.yml @@ -0,0 +1,9 @@ +--- +name: prevent_issue_epic_search +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457756 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/153668 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/463698 +milestone: '17.1' +group: group::duo chat +type: beta +default_enabled: false diff --git a/ee/app/models/ai/ai_resource/epic.rb b/ee/app/models/ai/ai_resource/epic.rb index f176749e9f5b46..dd5ab3b7b08e0e 100644 --- a/ee/app/models/ai/ai_resource/epic.rb +++ b/ee/app/models/ai/ai_resource/epic.rb @@ -23,7 +23,7 @@ def current_page_sentence def current_page_short_description <<~SENTENCE - The user is currently on a page that displays an epic with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the epic is '#{resource.title}'. Remember to use the 'EpicReader' tool if they ask a question about the epic. + The user is currently on a page that displays an epic with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the epic is '#{resource.title}'. SENTENCE end end diff --git a/ee/app/models/ai/ai_resource/issue.rb b/ee/app/models/ai/ai_resource/issue.rb index 77d209be3b70c6..f91380d5635ee2 100644 --- a/ee/app/models/ai/ai_resource/issue.rb +++ b/ee/app/models/ai/ai_resource/issue.rb @@ -23,7 +23,7 @@ def current_page_sentence def current_page_short_description <<~SENTENCE - The user is currently on a page that displays an issue with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the issue is '#{resource.title}'. Remember to use the 'IssueReader' tool if they ask a question about the issue. + The user is currently on a page that displays an issue with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the issue is '#{resource.title}'. SENTENCE end end diff --git a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb index e4eff63854528a..e3d7575e340514 100644 --- a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb +++ b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb @@ -85,6 +85,13 @@ def execute end traceable :execute, name: 'Run ReAct' + # Returns the current feature flag that is being tested as a prompt experiment + # Set this back to nil after rolling out a feature flag experiment fully + # @return [Symbol, nil] The current state or nil + def experimental_feature_flag + :prevent_issue_epic_search + end + private def execute_streamed_request @@ -102,7 +109,7 @@ def options @options ||= { tool_names: tools.map { |tool_class| tool_class::Executor::NAME }.join(', '), tools_definitions: tools.map do |tool_class| - tool_class::Executor.full_definition + tool_class::Executor.full_definition(use_experimental_prompt: use_experimental_prompt?) end.join("\n"), user_input: user_input, agent_scratchpad: +"", @@ -154,7 +161,14 @@ def prompt_version end def zero_shot_prompt - ZERO_SHOT_PROMPT + use_experimental_prompt? ? ZERO_SHOT_EXPERIMENTAL_PROMPT : ZERO_SHOT_PROMPT + end + + def use_experimental_prompt? + return false unless experimental_feature_flag.present? + + ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(experimental_feature_flag, + context.current_user) end def last_conversation @@ -255,6 +269,52 @@ def source_template Begin! PROMPT + ZERO_SHOT_EXPERIMENTAL_PROMPT = <<~PROMPT.freeze + Answer the question as accurate as you can. + + You have access only to the following tools: + + %s + + Consider every tool before making a decision. + Ensure that your answer is accurate and contain only information directly supported by the information retrieved using provided tools. + + When you can answer the question directly you must use this response format: + Thought: you should always think about how to answer the question + Action: DirectAnswer + Final Answer: the final answer to the original input question if you have a direct answer to the user's question. + + You must always use the following format when using a tool: + Question: the input question you must answer + Thought: you should always think about what to do + Action: the action to take, should be one tool from this list: [%s] + Action Input: the input to the action needs to be provided for every action that uses a tool. + Observation: the result of the tool actions. But remember that you're still #{AGENT_NAME}. + + + ... (this Thought/Action/Action Input/Observation sequence can repeat N times) + + Thought: I know the final answer. + Final Answer: the final answer to the original input question. + + When concluding your response, provide the final answer as "Final Answer:". It should contain everything that user needs to see, including answer from "Observation" section. + %s + + You have access to the following GitLab resources: %s. + You also have access to all information that can be helpful to someone working in software development of any kind. + At the moment, you do not have access to the following GitLab resources: Merge Requests, Pipelines, Vulnerabilities. + At the moment, you do not have the ability to search Issues or Epics based on a description or keywords. You can only read information about a specific issue/epic IF the user is on the specific issue/epic's page, or provides a URL or ID. + Do not use the IssueReader or EpicReader tool if you do not have these specified identifiers. + + %s + + Ask user to leave feedback. + + %s + + Begin! + PROMPT + PROMPT_TEMPLATE = [ Utils::Prompt.as_system(ZERO_SHOT_PROMPT), Utils::Prompt.as_user("Question: %s"), diff --git a/ee/lib/gitlab/llm/chain/tools/epic_reader/executor.rb b/ee/lib/gitlab/llm/chain/tools/epic_reader/executor.rb index 535d7ee945f0cd..5e6696561f1400 100644 --- a/ee/lib/gitlab/llm/chain/tools/epic_reader/executor.rb +++ b/ee/lib/gitlab/llm/chain/tools/epic_reader/executor.rb @@ -16,6 +16,23 @@ class Executor < Identifier 'high-level plans and discussions. Epic can contain multiple issues. ' \ 'Action Input for this tool should be the original question or epic identifier.' + EXPERIMENTAL_TOOL_DESCRIPTION = <<~PROMPT + This tool retrieves the content of a specific epic + ONLY if the user question fulfills the strict usage conditions below. + + **Strict Usage Conditions:** + * **Condition 1: epic ID Provided:** This tool MUST be used ONLY when the user provides a valid epic ID. + * **Condition 2: epic URL Context:** This tool MUST be used ONLY when the user is actively viewing a specific epic URL or a specific URL is provided by the user. + + **Do NOT** attempt to search for or identify epics based on descriptions, keywords, or user questions. + + **Action Input:** + * The original question asked by the user. + + **Important:** Reject any input that does not strictly adhere to the usage conditions above. + Return a message stating you are unable to search for epics without a valid identifier. + PROMPT + EXAMPLE = <<~PROMPT Question: Please identify the author of &123 epic. diff --git a/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb b/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb index c3a706526b56bf..5b63630dcfec0a 100644 --- a/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb +++ b/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb @@ -17,6 +17,23 @@ class Executor < Identifier 'collaboration, discussions, planning and tracking of work.' \ 'Action Input for this tool should be the original question or issue identifier.' + EXPERIMENTAL_TOOL_DESCRIPTION = <<~PROMPT + This tool retrieves the content of a specific issue + ONLY if the user question fulfills the strict usage conditions below. + + **Strict Usage Conditions:** + * **Condition 1: Issue ID Provided:** This tool MUST be used ONLY when the user provides a valid issue ID. + * **Condition 2: Issue URL Context:** This tool MUST be used ONLY when the user is actively viewing a specific issue URL or a specific URL is provided by the user. + + **Do NOT** attempt to search for or identify issues based on descriptions, keywords, or user questions. + + **Action Input:** + * The original question asked by the user. + + **Important:** Reject any input that does not strictly adhere to the usage conditions above. + Return a message stating you are unable to search for issues without a valid identifier. + PROMPT + EXAMPLE = <<~PROMPT Question: Please identify the author of #123 issue diff --git a/ee/lib/gitlab/llm/chain/tools/tool.rb b/ee/lib/gitlab/llm/chain/tools/tool.rb index 4e23d40ddd4f4c..3d331a660b69c7 100644 --- a/ee/lib/gitlab/llm/chain/tools/tool.rb +++ b/ee/lib/gitlab/llm/chain/tools/tool.rb @@ -16,12 +16,12 @@ class Tool delegate :resource, :resource=, to: :context - def self.full_definition + def self.full_definition(use_experimental_prompt: false) [ "", "#{self::NAME}", "", - description, + description(use_experimental_prompt), "", "", self::EXAMPLE, @@ -85,8 +85,10 @@ def group_from_context attr_reader :logger, :stream_response_handler - def self.description - self::DESCRIPTION + def self.description(use_experimental_prompt) + experiment = use_experimental_prompt && defined?(self::EXPERIMENTAL_TOOL_DESCRIPTION) + + experiment ? self::EXPERIMENTAL_TOOL_DESCRIPTION : self::DESCRIPTION end def not_found diff --git a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb index 54b5cae44164f0..3cfaea17bb82b0 100644 --- a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb @@ -233,11 +233,82 @@ agent.prompt end - it 'includes prompt in the options' do - expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) - .to receive(:prompt).once.with(a_hash_including(prompt_options)) + context 'when experimental prompt is not provided' do + let(:prompt_options) { { zero_shot_prompt: described_class::ZERO_SHOT_PROMPT } } - agent.prompt + before do + allow(agent).to receive(:experimental_feature_flag).and_return(:prevent_issue_epic_search) + stub_const("#{described_class.name}::ZERO_SHOT_PROMPT", 'I am not an experimental prompt.') + stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", "") + end + + context 'when experimental feature flag is enabled' do + it 'includes the default prompt options' do + expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) + .to receive(:prompt).once.with(a_hash_including(prompt_options)) + + agent.prompt + end + end + + context 'when experimental feature flag is not enabled' do + before do + stub_feature_flags(prevent_issue_epic_search: false) + end + + it 'includes the default prompt options' do + expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) + .to receive(:prompt).once.with(a_hash_including(prompt_options)) + + agent.prompt + end + end + end + + context 'when experimental prompt is provided' do + before do + allow(agent).to receive(:experimental_feature_flag).and_return(:prevent_issue_epic_search) + stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", 'I am an experimental prompt.') + end + + context 'when experimental feature flag is enabled' do + let(:prompt_options) { { zero_shot_prompt: described_class::ZERO_SHOT_EXPERIMENTAL_PROMPT } } + + it 'includes the experimental prompt in the prompt options' do + expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) + .to receive(:prompt).once.with(a_hash_including(prompt_options)) + + agent.prompt + end + end + + context 'when experimental feature flag is not enabled' do + let(:prompt_options) { { zero_shot_prompt: described_class::ZERO_SHOT_PROMPT } } + + before do + stub_feature_flags(prevent_issue_epic_search: false) + end + + it 'includes the default prompt options' do + expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) + .to receive(:prompt).once.with(a_hash_including(prompt_options)) + + agent.prompt + end + end + + context 'when experimental feature flag is nil' do + before do + allow(agent).to receive(:experimental_feature_flag).and_return(nil) + end + + it 'includes the default prompt options' do + expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) + .to receive(:prompt).once.with(a_hash_including(prompt_options)) + + agent.prompt + end + end end context 'when duo_chat_display_source feature flag is enabled' do diff --git a/ee/spec/lib/gitlab/llm/chain/tools/tool_spec.rb b/ee/spec/lib/gitlab/llm/chain/tools/tool_spec.rb index e9fbaf3bce2860..18c30150959be1 100644 --- a/ee/spec/lib/gitlab/llm/chain/tools/tool_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/tools/tool_spec.rb @@ -136,7 +136,21 @@ XML end - let(:expected_description) { 'TEST' } + let(:experimental_definition) do + <<~XML.chomp + + TEST_TOOL + + #{experimental_description} + + + EXAMPLE + + + XML + end + + let(:expected_description) { 'No feature flag description' } before do stub_const("#{described_class.name}::NAME", 'TEST_TOOL') @@ -144,9 +158,39 @@ stub_const("#{described_class.name}::EXAMPLE", 'EXAMPLE') end - context 'when description is defined' do - it 'returns detailed description of the tool' do - expect(described_class.full_definition).to eq(definition) + context 'when experimental description constant is not defined' do + context 'when experimental prompt is enabled' do + it 'returns default description of the tool' do + expect(described_class.full_definition(use_experimental_prompt: true)).to eq(definition) + end + end + + context 'when experimental prompt is not enabled' do + stub_feature_flags(prevent_issue_epic_search: false) + + it 'returns default description of the tool' do + expect(described_class.full_definition).to eq(definition) + end + end + end + + context 'when experimental description constant is defined' do + let(:experimental_description) { 'Experimental description' } + + before do + stub_const("#{described_class.name}::EXPERIMENTAL_TOOL_DESCRIPTION", experimental_description) + end + + context 'when experimental prompt is enabled' do + it 'returns experimental description of the tool' do + expect(described_class.full_definition(use_experimental_prompt: true)).to eq(experimental_definition) + end + end + + context 'when experimental prompt is not enabled' do + it 'returns default description of the tool' do + expect(described_class.full_definition).to eq(definition) + end end end end -- GitLab From e3a78cd9f7a7b5038972b40c432145872fb8d76e Mon Sep 17 00:00:00 2001 From: lesley Date: Sun, 26 May 2024 21:26:12 -0500 Subject: [PATCH 2/6] Add documentation --- doc/development/ai_features/duo_chat.md | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index a74f8f4b316974..bb47cd7ba4f835 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -423,6 +423,33 @@ the single source of truth and should be the most up-to-date. Please, see the video ([internal link](https://drive.google.com/file/d/1X6CARf0gebFYX4Rc9ULhcfq9LLLnJ_O-)) that covers the full setup. +## How to test prompt changes on production using a feature flag + +Only one experiment may be conducted on production at a time. If you go to merge your change and there is a conflict, you must wait for the current experiment to finish. +Because of this, it is important that follow the cleanup instructions below as promptly as possible after rolling out your feature flag. + +### 1. Add feature flag to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#experimental_feature_flag` method + +Return a symbol with the name of the feature flag you want to use. + +### 2A. For Zero Shot Executor + +Add the constant `ZERO_SHOT_EXPERIMENTAL_PROMPT` to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor` with the updated prompt you want to try. + +([Code](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb)) + +### 2B. For tools + +Add the constant `EXPERIMENTAL_TOOL_DESCRIPTION` in the `Executor` class of the tool you want to try. +For example, here is the class for the `IssueReader`: + +([Code](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb)) + +### Cleaning up after finishing experiment + +1. Change the `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#experimental_feature_flag` method to return `nil`. +1. Remove the `ZERO_SHOT_EXPERIMENTAL_PROMPT` or `EXPERIMENTAL_TOOL_DESCRIPTION` constant defined in step 2 above. + ## How a Chat prompt is constructed All Chat requests are resolved with the GitLab GraphQL API. And, for now, -- GitLab From 9498e7a4153e4329f770caeee1e71035f68d1a9d Mon Sep 17 00:00:00 2001 From: lesley Date: Tue, 28 May 2024 16:02:48 -0500 Subject: [PATCH 3/6] Do not use dynamic feature flag --- doc/development/ai_features/duo_chat.md | 13 ++++++++++--- .../gitlab/llm/chain/agents/zero_shot/executor.rb | 13 +++---------- .../llm/chain/agents/zero_shot/executor_spec.rb | 15 --------------- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index bb47cd7ba4f835..ccce96cbc0edb0 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -428,9 +428,16 @@ Please, see the video ([internal link](https://drive.google.com/file/d/1X6CARf0g Only one experiment may be conducted on production at a time. If you go to merge your change and there is a conflict, you must wait for the current experiment to finish. Because of this, it is important that follow the cleanup instructions below as promptly as possible after rolling out your feature flag. -### 1. Add feature flag to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#experimental_feature_flag` method +### 1. Add feature flag to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#use_experimental_prompt?` method -Return a symbol with the name of the feature flag you want to use. +Change the method to the following: + +```ruby +def use_experimental_prompt? + ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(, + context.current_user) +end +``` ### 2A. For Zero Shot Executor @@ -447,7 +454,7 @@ For example, here is the class for the `IssueReader`: ### Cleaning up after finishing experiment -1. Change the `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#experimental_feature_flag` method to return `nil`. +1. Change the `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#use_experimental_prompt?` method to return `false`. 1. Remove the `ZERO_SHOT_EXPERIMENTAL_PROMPT` or `EXPERIMENTAL_TOOL_DESCRIPTION` constant defined in step 2 above. ## How a Chat prompt is constructed diff --git a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb index e3d7575e340514..b5a2dc85d86329 100644 --- a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb +++ b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb @@ -85,13 +85,6 @@ def execute end traceable :execute, name: 'Run ReAct' - # Returns the current feature flag that is being tested as a prompt experiment - # Set this back to nil after rolling out a feature flag experiment fully - # @return [Symbol, nil] The current state or nil - def experimental_feature_flag - :prevent_issue_epic_search - end - private def execute_streamed_request @@ -164,10 +157,10 @@ def zero_shot_prompt use_experimental_prompt? ? ZERO_SHOT_EXPERIMENTAL_PROMPT : ZERO_SHOT_PROMPT end + # Set this method to return false if not using experimental prompt + # See `duo_chat.md` to see how to enable experimental prompts and how to clean up afterwards def use_experimental_prompt? - return false unless experimental_feature_flag.present? - - ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(experimental_feature_flag, + ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(:prevent_issue_epic_search, context.current_user) end diff --git a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb index 3cfaea17bb82b0..22291845418e78 100644 --- a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb @@ -237,7 +237,6 @@ let(:prompt_options) { { zero_shot_prompt: described_class::ZERO_SHOT_PROMPT } } before do - allow(agent).to receive(:experimental_feature_flag).and_return(:prevent_issue_epic_search) stub_const("#{described_class.name}::ZERO_SHOT_PROMPT", 'I am not an experimental prompt.') stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", "") end @@ -267,7 +266,6 @@ context 'when experimental prompt is provided' do before do - allow(agent).to receive(:experimental_feature_flag).and_return(:prevent_issue_epic_search) stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", 'I am an experimental prompt.') end @@ -296,19 +294,6 @@ agent.prompt end end - - context 'when experimental feature flag is nil' do - before do - allow(agent).to receive(:experimental_feature_flag).and_return(nil) - end - - it 'includes the default prompt options' do - expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) - .to receive(:prompt).once.with(a_hash_including(prompt_options)) - - agent.prompt - end - end end context 'when duo_chat_display_source feature flag is enabled' do -- GitLab From 88a2eeba3252534f2a2e33edf4fdc2bcd7ae7186 Mon Sep 17 00:00:00 2001 From: Lesley Razzaghian Date: Wed, 29 May 2024 22:37:53 +0000 Subject: [PATCH 4/6] Make documentation suggestions from MR --- doc/development/ai_features/duo_chat.md | 1 + ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index ccce96cbc0edb0..e5d1e735fde543 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -425,6 +425,7 @@ Please, see the video ([internal link](https://drive.google.com/file/d/1X6CARf0g ## How to test prompt changes on production using a feature flag +It's not allowed to add changes to prompts outside of the experiment process. Only one experiment may be conducted on production at a time. If you go to merge your change and there is a conflict, you must wait for the current experiment to finish. Because of this, it is important that follow the cleanup instructions below as promptly as possible after rolling out your feature flag. diff --git a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb index b5a2dc85d86329..6b6f03516ef435 100644 --- a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb +++ b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb @@ -157,8 +157,8 @@ def zero_shot_prompt use_experimental_prompt? ? ZERO_SHOT_EXPERIMENTAL_PROMPT : ZERO_SHOT_PROMPT end - # Set this method to return false if not using experimental prompt - # See `duo_chat.md` to see how to enable experimental prompts and how to clean up afterwards + # See `https://docs.gitlab.com/ee/development/ai_features/duo_chat.html#how_to_test_prompt_changes_on_production_using_a_feature_flag` + # to see how to enable experimental prompts and how to clean up afterwards def use_experimental_prompt? ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(:prevent_issue_epic_search, context.current_user) -- GitLab From 868a669e8daf5b91c6b62f592175099c7b9917b2 Mon Sep 17 00:00:00 2001 From: lesley Date: Thu, 30 May 2024 14:25:07 -0500 Subject: [PATCH 5/6] Simplify MR --- doc/development/ai_features/duo_chat.md | 35 ------------------- .../llm/chain/agents/zero_shot/executor.rb | 5 +-- .../chain/agents/zero_shot/executor_spec.rb | 33 +---------------- 3 files changed, 2 insertions(+), 71 deletions(-) diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index e5d1e735fde543..a74f8f4b316974 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -423,41 +423,6 @@ the single source of truth and should be the most up-to-date. Please, see the video ([internal link](https://drive.google.com/file/d/1X6CARf0gebFYX4Rc9ULhcfq9LLLnJ_O-)) that covers the full setup. -## How to test prompt changes on production using a feature flag - -It's not allowed to add changes to prompts outside of the experiment process. -Only one experiment may be conducted on production at a time. If you go to merge your change and there is a conflict, you must wait for the current experiment to finish. -Because of this, it is important that follow the cleanup instructions below as promptly as possible after rolling out your feature flag. - -### 1. Add feature flag to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#use_experimental_prompt?` method - -Change the method to the following: - -```ruby -def use_experimental_prompt? - ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(, - context.current_user) -end -``` - -### 2A. For Zero Shot Executor - -Add the constant `ZERO_SHOT_EXPERIMENTAL_PROMPT` to `Gitlab::Llm::Chain::Agents::ZeroShot::Executor` with the updated prompt you want to try. - -([Code](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb)) - -### 2B. For tools - -Add the constant `EXPERIMENTAL_TOOL_DESCRIPTION` in the `Executor` class of the tool you want to try. -For example, here is the class for the `IssueReader`: - -([Code](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/chain/tools/issue_reader/executor.rb)) - -### Cleaning up after finishing experiment - -1. Change the `Gitlab::Llm::Chain::Agents::ZeroShot::Executor#use_experimental_prompt?` method to return `false`. -1. Remove the `ZERO_SHOT_EXPERIMENTAL_PROMPT` or `EXPERIMENTAL_TOOL_DESCRIPTION` constant defined in step 2 above. - ## How a Chat prompt is constructed All Chat requests are resolved with the GitLab GraphQL API. And, for now, diff --git a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb index 6b6f03516ef435..85c46fdd7a59e7 100644 --- a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb +++ b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb @@ -157,11 +157,8 @@ def zero_shot_prompt use_experimental_prompt? ? ZERO_SHOT_EXPERIMENTAL_PROMPT : ZERO_SHOT_PROMPT end - # See `https://docs.gitlab.com/ee/development/ai_features/duo_chat.html#how_to_test_prompt_changes_on_production_using_a_feature_flag` - # to see how to enable experimental prompts and how to clean up afterwards def use_experimental_prompt? - ZERO_SHOT_EXPERIMENTAL_PROMPT.present? && Feature.enabled?(:prevent_issue_epic_search, - context.current_user) + Feature.enabled?(:prevent_issue_epic_search, context.current_user) end def last_conversation diff --git a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb index 22291845418e78..7029fa62c38478 100644 --- a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb @@ -233,38 +233,7 @@ agent.prompt end - context 'when experimental prompt is not provided' do - let(:prompt_options) { { zero_shot_prompt: described_class::ZERO_SHOT_PROMPT } } - - before do - stub_const("#{described_class.name}::ZERO_SHOT_PROMPT", 'I am not an experimental prompt.') - stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", "") - end - - context 'when experimental feature flag is enabled' do - it 'includes the default prompt options' do - expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) - .to receive(:prompt).once.with(a_hash_including(prompt_options)) - - agent.prompt - end - end - - context 'when experimental feature flag is not enabled' do - before do - stub_feature_flags(prevent_issue_epic_search: false) - end - - it 'includes the default prompt options' do - expect(Gitlab::Llm::Chain::Agents::ZeroShot::Prompts::Anthropic) - .to receive(:prompt).once.with(a_hash_including(prompt_options)) - - agent.prompt - end - end - end - - context 'when experimental prompt is provided' do + context 'with the `prevent_issue_epic_search` feature flag' do before do stub_const("#{described_class.name}::ZERO_SHOT_EXPERIMENTAL_PROMPT", 'I am an experimental prompt.') end -- GitLab From 26c070255c83ff575fa0f6737aad925c3dac3e4d Mon Sep 17 00:00:00 2001 From: lesley Date: Fri, 31 May 2024 15:01:14 -0500 Subject: [PATCH 6/6] Put experimental short description under feature flag --- ee/app/models/ai/ai_resource/epic.rb | 6 ++++++ ee/app/models/ai/ai_resource/issue.rb | 6 ++++++ .../llm/chain/agents/zero_shot/executor.rb | 6 +++++- ee/lib/gitlab/llm/chain/gitlab_context.rb | 4 ++++ .../llm/chain/agents/zero_shot/executor_spec.rb | 17 +++++++++++++++++ ee/spec/models/ai/ai_resource/epic_spec.rb | 10 ++++++++++ ee/spec/models/ai/ai_resource/issue_spec.rb | 10 ++++++++++ 7 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ee/app/models/ai/ai_resource/epic.rb b/ee/app/models/ai/ai_resource/epic.rb index dd5ab3b7b08e0e..7eaef1cbabffa3 100644 --- a/ee/app/models/ai/ai_resource/epic.rb +++ b/ee/app/models/ai/ai_resource/epic.rb @@ -22,6 +22,12 @@ def current_page_sentence end def current_page_short_description + <<~SENTENCE + The user is currently on a page that displays an epic with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the epic is '#{resource.title}'. Remember to use the 'EpicReader' tool if they ask a question about the epic. + SENTENCE + end + + def current_page_experimental_short_description <<~SENTENCE The user is currently on a page that displays an epic with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the epic is '#{resource.title}'. SENTENCE diff --git a/ee/app/models/ai/ai_resource/issue.rb b/ee/app/models/ai/ai_resource/issue.rb index f91380d5635ee2..abe078ff90b929 100644 --- a/ee/app/models/ai/ai_resource/issue.rb +++ b/ee/app/models/ai/ai_resource/issue.rb @@ -22,6 +22,12 @@ def current_page_sentence end def current_page_short_description + <<~SENTENCE + The user is currently on a page that displays an issue with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the issue is '#{resource.title}'. Remember to use the 'IssueReader' tool if they ask a question about the issue. + SENTENCE + end + + def current_page_experimental_short_description <<~SENTENCE The user is currently on a page that displays an issue with a description, comments, etc., which the user might refer to, for example, as 'current', 'this' or 'that'. The title of the issue is '#{resource.title}'. SENTENCE diff --git a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb index 85c46fdd7a59e7..4d17025361d63e 100644 --- a/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb +++ b/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb @@ -198,7 +198,11 @@ def prompt_options end def current_resource - context.current_page_short_description + if use_experimental_prompt? + context.current_page_experimental_short_description + else + context.current_page_short_description + end rescue ArgumentError "" end diff --git a/ee/lib/gitlab/llm/chain/gitlab_context.rb b/ee/lib/gitlab/llm/chain/gitlab_context.rb index e5970f0e81efb9..241aa468ef1805 100644 --- a/ee/lib/gitlab/llm/chain/gitlab_context.rb +++ b/ee/lib/gitlab/llm/chain/gitlab_context.rb @@ -30,6 +30,10 @@ def current_page_short_description authorized_resource&.current_page_short_description end + def current_page_experimental_short_description + authorized_resource&.current_page_experimental_short_description + end + def resource_serialized(content_limit:) return '' unless authorized_resource diff --git a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb index 7029fa62c38478..84f762595691ee 100644 --- a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb @@ -329,6 +329,10 @@ XML end + before do + stub_feature_flags(prevent_issue_epic_search: false) + end + let(:prompt_resource) do <<~CONTEXT @@ -338,6 +342,7 @@ end let(:short_description) { 'short description' } + let(:experimental_short_description) { 'experimental short description' } it 'does not include the current resource metadata' do expect(context).not_to receive(:resource_serialized) @@ -348,6 +353,18 @@ expect(context).to receive(:current_page_short_description).and_return(short_description) expect(system_prompt(agent)).to include(short_description) end + + context 'when the `prevent_issue_epic_search` is enabled' do + before do + stub_feature_flags(prevent_issue_epic_search: true) + end + + it 'returns experimental short description' do + expect(context).to receive(:current_page_experimental_short_description) + .and_return(experimental_short_description) + expect(system_prompt(agent)).to include(experimental_short_description) + end + end end context 'when the resource is an issue' do diff --git a/ee/spec/models/ai/ai_resource/epic_spec.rb b/ee/spec/models/ai/ai_resource/epic_spec.rb index 79e29df0956567..7883b62cee0a8a 100644 --- a/ee/spec/models/ai/ai_resource/epic_spec.rb +++ b/ee/spec/models/ai/ai_resource/epic_spec.rb @@ -32,6 +32,16 @@ describe '#current_page_short_description' do it 'returns prompt' do expect(wrapped_epic.current_page_short_description).to include("The title of the epic is '#{epic.title}'.") + expect(wrapped_epic.current_page_short_description).to include("Remember to use the 'EpicReader' tool") + end + end + + describe '#current_page_experimental_short_description' do + it 'returns experimental short description' do + expect(wrapped_epic.current_page_experimental_short_description) + .to include("The title of the epic is '#{epic.title}'.") + expect(wrapped_epic.current_page_experimental_short_description) + .not_to include("Remember to use the 'EpicReader' tool") end end end diff --git a/ee/spec/models/ai/ai_resource/issue_spec.rb b/ee/spec/models/ai/ai_resource/issue_spec.rb index 5e3b6da73b0cb2..d9861b9fc82543 100644 --- a/ee/spec/models/ai/ai_resource/issue_spec.rb +++ b/ee/spec/models/ai/ai_resource/issue_spec.rb @@ -31,6 +31,16 @@ describe '#current_page_short_description' do it 'returns prompt' do expect(wrapped_issue.current_page_short_description).to include("The title of the issue is '#{issue.title}'.") + expect(wrapped_issue.current_page_short_description).to include("Remember to use the 'IssueReader' tool") + end + end + + describe '#current_page_experimental_short_description' do + it 'returns experimental short description' do + expect(wrapped_issue.current_page_experimental_short_description) + .to include("The title of the issue is '#{issue.title}'.") + expect(wrapped_issue.current_page_experimental_short_description) + .not_to include("Remember to use the 'IssueReader' tool") end end end -- GitLab