diff --git a/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/ai_agent_platform_flow_metrics.js b/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/ai_agent_platform_flow_metrics.js new file mode 100644 index 0000000000000000000000000000000000000000..d0792effc67a597b24f5e8ac5888a0c10589ce1f --- /dev/null +++ b/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/ai_agent_platform_flow_metrics.js @@ -0,0 +1,56 @@ +import { orderBy } from 'lodash'; +import { secondsToMinutes } from '~/lib/utils/datetime/date_calculation_utility'; +import AiAgentPlatformFlowMetricsQuery from 'ee/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metrics.query.graphql'; +import { + DATE_RANGE_OPTION_LAST_30_DAYS, + DATE_RANGE_OPTIONS, +} from 'ee/analytics/analytics_dashboards/components/filters/constants'; +import { extractQueryResponseFromNamespace } from '~/analytics/shared/utils'; +import { defaultClient } from '../graphql/client'; + +const MAX_VISIBLE_NODES = 10; + +const requestFlowMetrics = async ({ namespace, startDate, endDate, sortBy, sortDesc = false }) => { + const result = await defaultClient.query({ + query: AiAgentPlatformFlowMetricsQuery, + variables: { + fullPath: namespace, + startDate, + endDate, + }, + }); + + const { + agentPlatform: { flowMetrics }, + } = extractQueryResponseFromNamespace({ + result, + resultKey: 'aiMetrics', + }); + + const nodes = sortBy ? orderBy(flowMetrics, sortBy, sortDesc ? 'desc' : 'asc') : flowMetrics; + return nodes.slice(0, MAX_VISIBLE_NODES).map(({ medianExecutionTime, ...rest }) => ({ + medianExecutionTime: secondsToMinutes(medianExecutionTime), + ...rest, + })); +}; + +export default async function fetch({ + namespace, + query: { dateRange = DATE_RANGE_OPTION_LAST_30_DAYS } = {}, + queryOverrides: { dateRange: dateRangeOverride = null, ...overridesRest } = {}, +}) { + // Default to 30 days if an invalid date range is given + const dateRangeKey = dateRangeOverride ?? dateRange; + const { startDate, endDate } = DATE_RANGE_OPTIONS[dateRangeKey] + ? DATE_RANGE_OPTIONS[dateRangeKey] + : DATE_RANGE_OPTIONS[DATE_RANGE_OPTION_LAST_30_DAYS]; + + const nodes = await requestFlowMetrics({ + startDate, + endDate, + namespace, + ...overridesRest, + }); + + return { nodes }; +} diff --git a/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/index.js b/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/index.js index 1eb2af66df9511a56bce60edd5f4693cf536c164..a64564bf649146914eb16d664599b2cd621b5b3e 100644 --- a/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/index.js +++ b/ee/app/assets/javascripts/analytics/analytics_dashboards/data_sources/index.js @@ -40,6 +40,7 @@ export default { usage_count: () => import('./usage_count'), dora_metrics: () => import('./dora_metrics'), dora_metrics_by_project: () => import('./dora_metrics_by_project'), + ai_agent_platform_flow_metrics: () => import('./ai_agent_platform_flow_metrics'), ai_impact_over_time: () => import('./ai_impact_over_time'), contributions: () => import('./contributions'), namespace_metadata: () => import('./namespace_metadata'), diff --git a/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metric_item.fragment.graphql b/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metric_item.fragment.graphql new file mode 100644 index 0000000000000000000000000000000000000000..790bd8d3bfcab13bf2e395c1e0f36820da5fbb63 --- /dev/null +++ b/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metric_item.fragment.graphql @@ -0,0 +1,11 @@ +fragment AiAgentPlatformFlowMetricItem on AiMetrics { + agentPlatform { + flowMetrics { + flowType + medianExecutionTime + sessionsCount + usersCount + completionRate + } + } +} diff --git a/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metrics.query.graphql b/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metrics.query.graphql new file mode 100644 index 0000000000000000000000000000000000000000..4a92fa972dfbc09a4fd03e1b479b81bf4556e61a --- /dev/null +++ b/ee/app/assets/javascripts/analytics/dashboards/ai_impact/graphql/ai_agent_platform_flow_metrics.query.graphql @@ -0,0 +1,16 @@ +#import "./ai_agent_platform_flow_metric_item.fragment.graphql" + +query aiAgentPlatformFlowMetricsQuery($fullPath: ID!, $startDate: Date!, $endDate: Date!) { + project(fullPath: $fullPath) { + id + aiMetrics(startDate: $startDate, endDate: $endDate) { + ...AiAgentPlatformFlowMetricItem + } + } + group(fullPath: $fullPath) { + id + aiMetrics(startDate: $startDate, endDate: $endDate) { + ...AiAgentPlatformFlowMetricItem + } + } +} diff --git a/ee/app/models/analytics/dashboards/visualization.rb b/ee/app/models/analytics/dashboards/visualization.rb index 7d366c714a2cf9e9e628b2007a3b9f1892186fbe..468e171f0cc0b9138f299ef82108b9ba76467b2e 100644 --- a/ee/app/models/analytics/dashboards/visualization.rb +++ b/ee/app/models/analytics/dashboards/visualization.rb @@ -68,6 +68,7 @@ class Visualization duo_chat_usage_rate_over_time duo_usage_rate_over_time pipeline_metrics_table + duo_flow_metrics_table code_suggestions_acceptance_rate_by_language_chart code_generation_volume_trends_chart duo_code_review_usage_by_user diff --git a/ee/app/validators/json_schemas/analytics_visualization.json b/ee/app/validators/json_schemas/analytics_visualization.json index 17456a0b2acc7d0b624989f32c2f5fe094b6226c..a895b9c6e275e4dc59a335e315e9e3909dd7093e 100644 --- a/ee/app/validators/json_schemas/analytics_visualization.json +++ b/ee/app/validators/json_schemas/analytics_visualization.json @@ -73,6 +73,7 @@ "usage_count", "dora_metrics", "dora_metrics_by_project", + "ai_agent_platform_flow_metrics", "ai_impact_over_time", "contributions", "namespace_metadata", diff --git a/ee/lib/gitlab/analytics/ai_impact_dashboard/dashboard.yaml b/ee/lib/gitlab/analytics/ai_impact_dashboard/dashboard.yaml index dbda6339d151e92d30993ed6c4f37d40d2a601d9..a8e93747f8e64b53581e1523a614259276b44397 100644 --- a/ee/lib/gitlab/analytics/ai_impact_dashboard/dashboard.yaml +++ b/ee/lib/gitlab/analytics/ai_impact_dashboard/dashboard.yaml @@ -90,24 +90,32 @@ panels: width: 12 height: 3 options: {} + - title: 'Flow usage (Last 30 days)' + visualization: duo_flow_metrics_table + gridAttributes: + yPos: 20 + xPos: 0 + width: 12 + height: 3 + options: {} - title: 'GitLab Duo Code Suggestions usage by user (Last 30 days)' visualization: duo_code_suggestions_by_user gridAttributes: - yPos: 20 + yPos: 23 xPos: 0 width: 6 height: 4 - title: 'GitLab Duo Code Review usage by user (Last 30 days)' visualization: duo_code_review_usage_by_user gridAttributes: - yPos: 20 + yPos: 23 xPos: 6 width: 6 height: 4 - title: 'GitLab Duo Root Cause Analysis usage by user (Last 30 days)' visualization: duo_rca_usage_by_user gridAttributes: - yPos: 24 + yPos: 27 xPos: 0 width: 6 height: 4 diff --git a/ee/lib/gitlab/analytics/ai_impact_dashboard/visualizations/duo_flow_metrics_table.yaml b/ee/lib/gitlab/analytics/ai_impact_dashboard/visualizations/duo_flow_metrics_table.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b37b703f79ffafde6ac8caf3407c2ba68f5c9b9 --- /dev/null +++ b/ee/lib/gitlab/analytics/ai_impact_dashboard/visualizations/duo_flow_metrics_table.yaml @@ -0,0 +1,30 @@ +--- +version: 1 +type: DataTable +data: + type: ai_agent_platform_flow_metrics + query: {} +options: + refetchOnSort: true + fields: + - key: 'flowType' + label: 'Flow' + - key: 'sessionsCount' + label: 'Number of sessions' + sortable: true + thAlignRight: true + tdClass: "gl-text-right" + - key: 'medianExecutionTime' + label: 'Median duration (min)' + sortable: true + thAlignRight: true + tdClass: "gl-text-right" + - key: 'usersCount' + label: 'Unique users' + sortable: true + thAlignRight: true + tdClass: "gl-text-right" + - key: 'completionRate' + label: 'Completion rate (%)' + thAlignRight: true + tdClass: "gl-text-right" diff --git a/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/__snapshots__/ai_agent_platform_flow_metrics_spec.js.snap b/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/__snapshots__/ai_agent_platform_flow_metrics_spec.js.snap new file mode 100644 index 0000000000000000000000000000000000000000..996355442fde0a4090c27f122b1cadbd56479bf7 --- /dev/null +++ b/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/__snapshots__/ai_agent_platform_flow_metrics_spec.js.snap @@ -0,0 +1,234 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: medianExecutionTime, sortDesc: false) 1`] = ` +[ + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, +] +`; + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: medianExecutionTime, sortDesc: true) 1`] = ` +[ + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, +] +`; + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: sessionsCount, sortDesc: false) 1`] = ` +[ + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, +] +`; + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: sessionsCount, sortDesc: true) 1`] = ` +[ + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, +] +`; + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: usersCount, sortDesc: false) 1`] = ` +[ + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, +] +`; + +exports[`AI Agent platform flow metrics data source applies correct sorting for (sortBy: usersCount, sortDesc: true) 1`] = ` +[ + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, +] +`; + +exports[`AI Agent platform flow metrics data source returns the flow metrics as nodes on success 1`] = ` +{ + "nodes": [ + { + "completionRate": 92.5, + "flowType": "AGENT_FLOW", + "medianExecutionTime": 20.833333333333332, + "sessionsCount": 145, + "usersCount": 32, + }, + { + "completionRate": 88.3, + "flowType": "TOOL_FLOW", + "medianExecutionTime": 14.166666666666666, + "sessionsCount": 203, + "usersCount": 48, + }, + { + "completionRate": 95.1, + "flowType": "CUSTOM_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + { + "completionRate": 95.1, + "flowType": "BONUS_FLOW", + "medianExecutionTime": 35, + "sessionsCount": 87, + "usersCount": 19, + }, + ], +} +`; diff --git a/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/ai_agent_platform_flow_metrics_spec.js b/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/ai_agent_platform_flow_metrics_spec.js new file mode 100644 index 0000000000000000000000000000000000000000..fdb3d38bb3c8ab8b3e0c3710386d7311b696695a --- /dev/null +++ b/ee/spec/frontend/analytics/analytics_dashboards/components/data_sources/ai_agent_platform_flow_metrics_spec.js @@ -0,0 +1,110 @@ +import fetch from 'ee/analytics/analytics_dashboards/data_sources/ai_agent_platform_flow_metrics'; +import { defaultClient } from 'ee/analytics/analytics_dashboards/graphql/client'; +import { DATE_RANGE_OPTION_LAST_7_DAYS } from 'ee/analytics/analytics_dashboards/components/filters/constants'; + +const mockFlowMetrics = [ + { + flowType: 'AGENT_FLOW', + medianExecutionTime: 1250, + sessionsCount: 145, + usersCount: 32, + completionRate: 92.5, + }, + { + flowType: 'TOOL_FLOW', + medianExecutionTime: 850, + sessionsCount: 203, + usersCount: 48, + completionRate: 88.3, + }, + { + flowType: 'CUSTOM_FLOW', + medianExecutionTime: 2100, + sessionsCount: 87, + usersCount: 19, + completionRate: 95.1, + }, + { + flowType: 'BONUS_FLOW', + medianExecutionTime: 2100, + sessionsCount: 87, + usersCount: 19, + completionRate: 95.1, + }, +]; + +const mockTooManyFlowMetrics = [...mockFlowMetrics, ...mockFlowMetrics, ...mockFlowMetrics]; + +describe('AI Agent platform flow metrics data source', () => { + const namespace = 'namespace'; + + const mockResolvedQuery = (flowMetrics = mockFlowMetrics) => + jest.spyOn(defaultClient, 'query').mockResolvedValueOnce({ + data: { + group: { id: 'gid://gitlab/Group/1', aiMetrics: { agentPlatform: { flowMetrics } } }, + }, + }); + + it('returns the flow metrics as nodes on success', async () => { + mockResolvedQuery(); + + const result = await fetch({ namespace }); + expect(result).toMatchSnapshot(); + }); + + it('uses 30 days as the default date range', async () => { + mockResolvedQuery(); + await fetch({ namespace }); + + expect(defaultClient.query).toHaveBeenCalledWith( + expect.objectContaining({ + variables: expect.objectContaining({ + fullPath: namespace, + startDate: new Date('2020-06-06'), + endDate: new Date('2020-07-06'), + }), + }), + ); + }); + + it('uses a custom date range when defined', async () => { + mockResolvedQuery(); + await fetch({ + namespace, + queryOverrides: { dateRange: DATE_RANGE_OPTION_LAST_7_DAYS }, + }); + + expect(defaultClient.query).toHaveBeenCalledWith( + expect.objectContaining({ + variables: expect.objectContaining({ + fullPath: namespace, + startDate: new Date('2020-06-29'), + endDate: new Date('2020-07-06'), + }), + }), + ); + }); + + it('limits to 10 returned nodes', async () => { + mockResolvedQuery(mockTooManyFlowMetrics); + const result = await fetch({ namespace }); + expect(result.nodes).toHaveLength(10); + }); + + it.each([ + ['sessionsCount', true], + ['sessionsCount', false], + ['medianExecutionTime', true], + ['medianExecutionTime', false], + ['usersCount', true], + ['usersCount', false], + ])('applies correct sorting for (sortBy: %s, sortDesc: %s)', async (sortBy, sortDesc) => { + mockResolvedQuery(); + const result = await fetch({ + namespace, + queryOverrides: { sortBy, sortDesc }, + }); + + expect(result.nodes).toMatchSnapshot(); + }); +}); diff --git a/ee/spec/models/analytics/dashboards/visualization_spec.rb b/ee/spec/models/analytics/dashboards/visualization_spec.rb index 8a37e9d1d8318494c15a223530d2ef541373a114..13dec22ff6d1abfc6b37bf85fb08a02aa720e2b3 100644 --- a/ee/spec/models/analytics/dashboards/visualization_spec.rb +++ b/ee/spec/models/analytics/dashboards/visualization_spec.rb @@ -49,6 +49,7 @@ duo_chat_usage_rate_over_time duo_usage_rate_over_time pipeline_metrics_table + duo_flow_metrics_table code_suggestions_acceptance_rate_by_language_chart code_generation_volume_trends_chart duo_code_review_usage_by_user diff --git a/ee/spec/requests/api/graphql/analytics/dashboards/visualizations_spec.rb b/ee/spec/requests/api/graphql/analytics/dashboards/visualizations_spec.rb index 7b3a50d7690f3b1621c38a06612a376f67c0ecee..7e7df98012759b837b0ec805abee0fb12f90cc59 100644 --- a/ee/spec/requests/api/graphql/analytics/dashboards/visualizations_spec.rb +++ b/ee/spec/requests/api/graphql/analytics/dashboards/visualizations_spec.rb @@ -76,9 +76,10 @@ 8 | 'AiImpactTable' | 'GitLab Duo usage metrics for the %{namespaceName} %{namespaceType}' 9 | 'AiImpactTable' | 'Development metrics for the %{namespaceName} %{namespaceType}' 10 | 'AiImpactTable' | 'Pipeline metrics for the %{namespaceName} %{namespaceType}' - 11 | 'DataTable' | 'GitLab Duo Code Suggestions usage by user (Last 30 days)' - 12 | 'DataTable' | 'GitLab Duo Code Review usage by user (Last 30 days)' - 13 | 'DataTable' | 'GitLab Duo Root Cause Analysis usage by user (Last 30 days)' + 11 | 'DataTable' | 'Flow usage (Last 30 days)' + 12 | 'DataTable' | 'GitLab Duo Code Suggestions usage by user (Last 30 days)' + 13 | 'DataTable' | 'GitLab Duo Code Review usage by user (Last 30 days)' + 14 | 'DataTable' | 'GitLab Duo Root Cause Analysis usage by user (Last 30 days)' end with_them do