From 7dcfe12877755478ae89414ca109aa94f17918f3 Mon Sep 17 00:00:00 2001 From: GitLab Duo Date: Tue, 7 Oct 2025 15:47:02 +0000 Subject: [PATCH] Duo Workflow: Resolve issue #568763 --- .../quick_start.md | 10 ++ .../cli/flows/event_definer.rb | 50 +++++++++ .../event_definer_examples.yml | 25 +++++ .../events/ai_event_with_identifiers.yml | 14 +++ .../cli/flows/event_definer_spec.rb | 101 ++++++++++++++++++ spec/scripts/internal_events/cli_spec.rb | 66 ++++++++++++ .../internal_events_cli_shared_examples.rb | 14 +++ test_ai_detection.rb | 0 test_ai_guidance.rb | 0 9 files changed, 280 insertions(+) create mode 100644 spec/fixtures/scripts/internal_events/events/ai_event_with_identifiers.yml create mode 100644 test_ai_detection.rb create mode 100644 test_ai_guidance.rb diff --git a/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md b/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md index 91143e00238dfa..1054677f283eed 100644 --- a/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md +++ b/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md @@ -26,6 +26,16 @@ This CLI will help you create the correct definition files based on your specifi Events should be named in the format of `__`, valid examples are `create_ci_build` or `click_previous_blame_on_blob_page`. +### AI tracking events + +The CLI generator automatically detects AI-related events and provides additional guidance for proper setup. When creating events with names that match AI patterns (such as `code_suggestion_*`, `duo_*`, `ai_*`, etc.) or events with AI-specific properties (like `model_name`, `model_engine`, `unique_tracking_id`), the CLI will: + +1. Display a link to the [AI usage tracking documentation](../../ai_features/usage_tracking.md#adding-new-event-for-tracking) +2. Provide guidance on additional setup steps required for AI events +3. Remind you about the need for `extra_trackers` configuration and `Gitlab::Tracking::AiTracking` registration + +This ensures that AI usage tracking events are properly configured for in-product analytics and external tracking systems. + ## Trigger events Triggering an event and thereby updating a metric is slightly different on backend and frontend. Refer to the relevant section below. diff --git a/scripts/internal_events/cli/flows/event_definer.rb b/scripts/internal_events/cli/flows/event_definer.rb index 347292a641297f..18bf9f63734f8b 100644 --- a/scripts/internal_events/cli/flows/event_definer.rb +++ b/scripts/internal_events/cli/flows/event_definer.rb @@ -246,6 +246,8 @@ def display_result(outcome) Typical flow: Define event > Define metric > Instrument app code > Merge/Deploy MR > Verify data in Tableau/Snowflake TEXT + + display_ai_tracking_guidance if ai_tracking_event? end def prompt_for_next_steps @@ -282,6 +284,54 @@ def prompt_for_next_steps cli.say feedback_notice end end + + private + + def ai_tracking_event? + return false unless event.action + + ai_event_patterns = [ + /^code_suggestion_/, + /^duo_/, + /^ai_/, + /_ai_/, + /_duo_/, + /agent_platform/, + /gitlab_duo/, + /llm_/, + /chat_/ + ] + + ai_event_patterns.any? { |pattern| event.action.match?(pattern) } || + event.additional_properties&.key?('model_name') || + event.additional_properties&.key?('model_engine') || + event.additional_properties&.key?('unique_tracking_id') + end + + def display_ai_tracking_guidance + cli.say <<~TEXT + + #{divider} + #{format_info('AI TRACKING DETECTED!')} + + This event appears to be related to AI usage tracking. For AI events, you'll need additional setup: + + #{format_selection('📖 DOCUMENTATION:')} + For complete AI tracking setup instructions, see: + #{format_help('https://docs.gitlab.com/development/ai_features/usage_tracking/#adding-new-event-for-tracking')} + + #{format_selection('🔧 NEXT STEPS:')} + 1. Register your event in Gitlab::Tracking::AiTracking + 2. Add extra_trackers configuration to your event definition + 3. Implement proper transformation blocks for metadata + 4. Test your AI tracking integration + + #{format_warning('⚠️ IMPORTANT:')} + AI events require special handling for in-product analytics and external tracking. + Make sure to follow the complete setup process in the documentation above. + + TEXT + end end end end diff --git a/spec/fixtures/scripts/internal_events/event_definer_examples.yml b/spec/fixtures/scripts/internal_events/event_definer_examples.yml index e3c08c6a30ca06..d813b6d532bb1b 100644 --- a/spec/fixtures/scripts/internal_events/event_definer_examples.yml +++ b/spec/fixtures/scripts/internal_events/event_definer_examples.yml @@ -308,3 +308,28 @@ files: - path: config/events/internal_events_cli_used.yml content: spec/fixtures/scripts/internal_events/events/event_with_feature_enabled_by_namespace_ids_identifier.yml + +- description: AI tracking event displays guidance and documentation link + inputs: + keystrokes: + - "1\n" # Enum-select: New Event -- start tracking when an action or scenario occurs on gitlab instances + - "Code suggestion accepted in IDE\n" # Submit description + - "code_suggestion_accepted_in_ide\n" # Submit action name + - "1\n" # Select: [namespace, project, user] + - "\n" # Select (add props): None! Continue to next step! + - "\n" # Skip MR URL + - "instrumentation" # Filters to the analytics instrumentation group + - "\n" # Accept analytics:monitor:analytics_instrumentation + - " \n" # Select product category + - "2\n" # Select: [premium, ultimate] + - "y\n" # Create file + - "4\n" # Exit + outputs: + files: + - path: ee/config/events/code_suggestion_accepted_in_ide.yml + content: spec/fixtures/scripts/internal_events/events/ai_event_with_identifiers.yml + expected_cli_output: + - "AI TRACKING DETECTED!" + - "https://docs.gitlab.com/development/ai_features/usage_tracking/#adding-new-event-for-tracking" + - "Register your event in Gitlab::Tracking::AiTracking" + - "Add extra_trackers configuration" diff --git a/spec/fixtures/scripts/internal_events/events/ai_event_with_identifiers.yml b/spec/fixtures/scripts/internal_events/events/ai_event_with_identifiers.yml new file mode 100644 index 00000000000000..591bee0d363d4b --- /dev/null +++ b/spec/fixtures/scripts/internal_events/events/ai_event_with_identifiers.yml @@ -0,0 +1,14 @@ +--- +description: Code suggestion accepted in IDE +internal_events: true +action: code_suggestion_accepted_in_ide +identifiers: +- user +- project +- namespace +product_group: analytics_instrumentation +milestone: '18.5' +introduced_by_url: TODO +tiers: +- premium +- ultimate \ No newline at end of file diff --git a/spec/scripts/internal_events/cli/flows/event_definer_spec.rb b/spec/scripts/internal_events/cli/flows/event_definer_spec.rb index d6eec6054d0f58..dae39a4bdc648a 100644 --- a/spec/scripts/internal_events/cli/flows/event_definer_spec.rb +++ b/spec/scripts/internal_events/cli/flows/event_definer_spec.rb @@ -40,4 +40,105 @@ end end end + + describe 'AI tracking detection' do + let(:event_definer) { InternalEventsCli::Flows::EventDefiner.new(prompt) } + + context 'when event name matches AI patterns' do + it 'detects code_suggestion_ events' do + event_definer.instance_variable_get(:@event).action = 'code_suggestion_accepted_in_ide' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects duo_ events' do + event_definer.instance_variable_get(:@event).action = 'duo_chat_response_received' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects ai_ events' do + event_definer.instance_variable_get(:@event).action = 'ai_response_generated' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects _ai_ events' do + event_definer.instance_variable_get(:@event).action = 'user_ai_interaction_completed' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects _duo_ events' do + event_definer.instance_variable_get(:@event).action = 'user_duo_feature_used' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects agent_platform events' do + event_definer.instance_variable_get(:@event).action = 'agent_platform_session_created' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects gitlab_duo events' do + event_definer.instance_variable_get(:@event).action = 'submit_gitlab_duo_question' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects llm_ events' do + event_definer.instance_variable_get(:@event).action = 'llm_completion_requested' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects chat_ events' do + event_definer.instance_variable_get(:@event).action = 'chat_message_sent' + expect(event_definer.send(:ai_tracking_event?)).to be true + end + end + + context 'when event has AI-related additional properties' do + it 'detects events with model_name property' do + event_definer.instance_variable_get(:@event).action = 'regular_event_name' + event_definer.instance_variable_get(:@event).additional_properties = { 'model_name' => { 'description' => 'AI model used' } } + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects events with model_engine property' do + event_definer.instance_variable_get(:@event).action = 'regular_event_name' + event_definer.instance_variable_get(:@event).additional_properties = { 'model_engine' => { 'description' => 'AI engine used' } } + expect(event_definer.send(:ai_tracking_event?)).to be true + end + + it 'detects events with unique_tracking_id property' do + event_definer.instance_variable_get(:@event).action = 'regular_event_name' + event_definer.instance_variable_get(:@event).additional_properties = { 'unique_tracking_id' => { 'description' => 'Unique tracking ID' } } + expect(event_definer.send(:ai_tracking_event?)).to be true + end + end + + context 'when event is not AI-related' do + it 'does not detect regular events' do + event_definer.instance_variable_get(:@event).action = 'user_clicked_button' + expect(event_definer.send(:ai_tracking_event?)).to be false + end + + it 'does not detect events without action' do + event_definer.instance_variable_get(:@event).action = nil + expect(event_definer.send(:ai_tracking_event?)).to be false + end + end + end + + describe 'AI tracking guidance display' do + let(:event_definer) { InternalEventsCli::Flows::EventDefiner.new(prompt) } + + before do + allow(event_definer).to receive(:ai_tracking_event?).and_return(true) + allow(event_definer).to receive(:divider).and_return('---') + allow(event_definer).to receive(:format_info).and_return('INFO') + allow(event_definer).to receive(:format_selection).and_return('SELECTION') + allow(event_definer).to receive(:format_help).and_return('HELP') + allow(event_definer).to receive(:format_warning).and_return('WARNING') + end + + it 'displays AI tracking guidance when AI event is detected' do + expect(event_definer).to receive(:cli).and_return(double(say: nil)) + event_definer.send(:display_ai_tracking_guidance) + end + end end diff --git a/spec/scripts/internal_events/cli_spec.rb b/spec/scripts/internal_events/cli_spec.rb index 73c388c591b52c..f03e30ca0acbd8 100644 --- a/spec/scripts/internal_events/cli_spec.rb +++ b/spec/scripts/internal_events/cli_spec.rb @@ -138,4 +138,70 @@ let(:output_files) { [{ 'path' => event2_filepath, 'content' => event2_content }] } end end + + context 'when creating AI tracking events' do + it 'displays AI tracking guidance for AI-related event names' do + queue_cli_inputs([ + "1\n", # Enum-select: New Event -- start tracking when an action or scenario occurs on gitlab instances + "Code suggestion accepted in IDE\n", # Submit description + "code_suggestion_accepted_in_ide\n", # Submit action name + "1\n", # Select: [project, namespace, user] + "\n", # Select: None! Continue to next section! + "\n", # Skip MR URL + "instrumentation\n", # Filter & select group + " \n", # Select product category + "2\n", # Select [premium, ultimate] + "y\n", # Create file + "4\n" # Exit + ]) + + with_cli_thread do + expect { prompt.output.string }.to eventually_include_cli_text('AI TRACKING DETECTED!') + expect { prompt.output.string }.to eventually_include_cli_text('https://docs.gitlab.com/development/ai_features/usage_tracking/#adding-new-event-for-tracking') + expect { prompt.output.string }.to eventually_include_cli_text('Register your event in Gitlab::Tracking::AiTracking') + expect { prompt.output.string }.to eventually_include_cli_text('Add extra_trackers configuration') + end + end + + it 'displays AI tracking guidance for duo events' do + queue_cli_inputs([ + "1\n", # Enum-select: New Event -- start tracking when an action or scenario occurs on gitlab instances + "Duo chat response received\n", # Submit description + "duo_chat_response_received\n", # Submit action name + "2\n", # Select: [namespace, user] + "\n", # Select: None! Continue to next section! + "\n", # Skip MR URL + "instrumentation\n", # Filter & select group + " \n", # Select product category + "2\n", # Select [premium, ultimate] + "y\n", # Create file + "4\n" # Exit + ]) + + with_cli_thread do + expect { prompt.output.string }.to eventually_include_cli_text('AI TRACKING DETECTED!') + expect { prompt.output.string }.to eventually_include_cli_text('https://docs.gitlab.com/development/ai_features/usage_tracking/#adding-new-event-for-tracking') + end + end + + it 'does not display AI tracking guidance for regular events' do + queue_cli_inputs([ + "1\n", # Enum-select: New Event -- start tracking when an action or scenario occurs on gitlab instances + "User clicked button\n", # Submit description + "user_clicked_button\n", # Submit action name + "3\n", # Select: [user] + "\n", # Select: None! Continue to next section! + "\n", # Skip MR URL + "instrumentation\n", # Filter & select group + " \n", # Select product category + "1\n", # Select [free, premium, ultimate] + "y\n", # Create file + "4\n" # Exit + ]) + + with_cli_thread do + expect { prompt.output.string }.not_to eventually_include_cli_text('AI TRACKING DETECTED!') + end + end + end end diff --git a/spec/support/shared_examples/internal_events_cli_shared_examples.rb b/spec/support/shared_examples/internal_events_cli_shared_examples.rb index 5d7eb0f9be0d69..fff5817de433d5 100644 --- a/spec/support/shared_examples/internal_events_cli_shared_examples.rb +++ b/spec/support/shared_examples/internal_events_cli_shared_examples.rb @@ -13,6 +13,7 @@ # files: [Array] # - path: [String] # expected path for where the new file should be created # content: [String] # fixure file which contains the expected file contents +# expected_cli_output: [Array] # expected text to appear in CLI output # } # # Note: @@ -23,6 +24,7 @@ let(:keystrokes) { test_case.dig('inputs', 'keystrokes') || [] } let(:input_files) { test_case.dig('inputs', 'files') || [] } let(:output_files) { test_case.dig('outputs', 'files') || [] } + let(:expected_cli_output) { test_case.dig('outputs', 'expected_cli_output') || [] } let(:timeout_error) { 'Internal Events CLI timed out while awaiting completion.' } # Script execution should stop without a reduced timeout @@ -38,6 +40,9 @@ # Check that script exited gracefully as a result of user input expect(plain_last_lines(10)).to include('Thanks for using the Internal Events CLI!') + + # Check expected CLI output if specified + check_expected_cli_output end private @@ -72,4 +77,13 @@ def wait_for_cli_completion wait_for(timeout_error) { !thread.alive? } end end + + def check_expected_cli_output + return if expected_cli_output.empty? + + cli_output = prompt.output.string + expected_cli_output.each do |expected_text| + expect(cli_output).to include(expected_text) + end + end end diff --git a/test_ai_detection.rb b/test_ai_detection.rb new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/test_ai_guidance.rb b/test_ai_guidance.rb new file mode 100644 index 00000000000000..e69de29bb2d1d6 -- GitLab