AI: provide network summary to Perf Insights agent

Someone found a bug where their site broke the agent because it had 180
requests that the agent asked for details of and this quickly ate up the
available context window. This CL changes the architecture so the LLM is
expected to first get a summary of the network requests before then
being allowed to enquire about specific requests.

There is definitely more room to experiment here; we also could provide
the LLM with a list of "critical" requests based on heuristics we have,
rather than providing a list of all requests. This needs more
experimentation.

Fixed: 400909537
Change-Id: I0d224834010dad050411a0024d226ca1e22f3664
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6324575
Auto-Submit: Jack Franklin <[email protected]>
Reviewed-by: Alex Rudenko <[email protected]>
Commit-Queue: Jack Franklin <[email protected]>

Author: jackfranklin

front_end/panels/ai_assistance/AiAssistancePanel.ts

@@ -253,7 +253,7 @@ function getEmptyStateSuggestions(conversationType: ConversationType): string[]
       ];
     case ConversationType.PERFORMANCE_INSIGHT:
       // TODO(b/393061683): Define these.
-      return ['Placeholder', 'Suggestions', 'For now'];
+      return ['Help me optimize my LCP', 'Suggestions', 'For now'];
   }
 }
 

front_end/panels/ai_assistance/agents/PerformanceInsightsAgent.test.ts

@@ -95,14 +95,15 @@ What is this?`;
   });
 
   describe('function calls', () => {
-    it('calls getNetworkActivity', async function() {
+    it('calls getNetworkActivitySummary', async function() {
       const {parsedTrace, insights} = await TraceLoader.traceEngine(this, 'lcp-images.json.gz');
       assert.isOk(insights);
       const [firstNav] = parsedTrace.Meta.mainFrameNavigations;
       const lcpPhases = getInsightOrError('LCPPhases', insights, firstNav);
       const agent = new PerformanceInsightsAgent({
-        aidaClient: mockAidaClient(
-            [[{explanation: '', functionCalls: [{name: 'getNetworkActivity', args: {}}]}], [{explanation: 'done'}]])
+        aidaClient: mockAidaClient([
+          [{explanation: '', functionCalls: [{name: 'getNetworkActivitySummary', args: {}}]}], [{explanation: 'done'}]
+        ])
       });
       const activeInsight = new TimelineUtils.InsightAIContext.ActiveInsight(lcpPhases, parsedTrace);
       const context = new InsightContext(activeInsight);
@@ -124,8 +125,43 @@ What is this?`;
         return match;
       });
 
-      const expectedRequestsOutput = requests.map(r => TraceEventFormatter.networkRequest(r, parsedTrace));
+      const expectedRequestsOutput =
+          requests.map(r => TraceEventFormatter.networkRequest(r, parsedTrace, {verbose: false}));
       const expectedOutput = JSON.stringify({requests: expectedRequestsOutput});
+      const titleResponse = responses.find(response => response.type === ResponseType.TITLE);
+      assert.exists(titleResponse);
+      assert.strictEqual(titleResponse.title, 'Investigating network activity…');
+
+      assert.exists(action);
+      assert.deepEqual(
+          action, {type: 'action' as ActionResponse['type'], output: expectedOutput, code: undefined, canceled: false});
+    });
+
+    it('can call getNetworkRequestDetail to get detail about a single request', async function() {
+      const {parsedTrace, insights} = await TraceLoader.traceEngine(this, 'lcp-images.json.gz');
+      assert.isOk(insights);
+      const [firstNav] = parsedTrace.Meta.mainFrameNavigations;
+      const lcpPhases = getInsightOrError('LCPPhases', insights, firstNav);
+      const requestUrl = 'https://chromedevtools.github.io/performance-stories/lcp-large-image/app.css';
+      const agent = new PerformanceInsightsAgent({
+        aidaClient: mockAidaClient([
+          [{explanation: '', functionCalls: [{name: 'getNetworkRequestDetail', args: {url: requestUrl}}]}],
+          [{explanation: 'done'}]
+        ])
+      });
+      const activeInsight = new TimelineUtils.InsightAIContext.ActiveInsight(lcpPhases, parsedTrace);
+      const context = new InsightContext(activeInsight);
+
+      const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+      const titleResponse = responses.find(response => response.type === ResponseType.TITLE);
+      assert.exists(titleResponse);
+      assert.strictEqual(titleResponse.title, `Investigating network request ${requestUrl}…`);
+      const action = responses.find(response => response.type === ResponseType.ACTION);
+      const request = parsedTrace.NetworkRequests.byTime.find(r => r.args.data.url === requestUrl);
+      assert.isOk(request);
+
+      const expectedRequestOutput = TraceEventFormatter.networkRequest(request, parsedTrace, {verbose: true});
+      const expectedOutput = JSON.stringify({request: expectedRequestOutput});
 
       assert.exists(action);
       assert.deepEqual(
@@ -145,14 +181,17 @@ What is this?`;
       const context = new InsightContext(activeInsight);
 
       const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+      const titleResponse = responses.find(response => response.type === ResponseType.TITLE);
+      assert.exists(titleResponse);
+      assert.strictEqual(titleResponse.title, 'Investigating main thread activity…');
+
       const action = responses.find(response => response.type === ResponseType.ACTION);
+      assert.exists(action);
 
       const expectedTree = TimelineUtils.InsightAIContext.AIQueries.mainThreadActivity(lcpPhases, parsedTrace);
       assert.isOk(expectedTree);
-
       const expectedOutput = JSON.stringify({activity: expectedTree.serialize()});
 
-      assert.exists(action);
       assert.deepEqual(
           action, {type: 'action' as ActionResponse['type'], output: expectedOutput, code: undefined, canceled: false});
     });

front_end/panels/ai_assistance/agents/PerformanceInsightsAgent.ts

@@ -8,6 +8,7 @@ import type * as Lit from '../../../ui/lit/lit.js';
 import * as TimelineUtils from '../../timeline/utils/utils.js';
 import * as PanelUtils from '../../utils/utils.js';
 import {PerformanceInsightFormatter, TraceEventFormatter} from '../data_formatters/PerformanceInsightFormatter.js';
+import {debugLog} from '../debug.js';
 
 import {
   type AgentOptions as BaseAgentOptions,
@@ -25,7 +26,7 @@ const UIStringsNotTranslated = {
   /**
    *@description Shown when the agent is investigating network activity
    */
-  networkActivity: 'Investigating network activity…',
+  networkActivitySummary: 'Investigating network activity…',
   /**
    *@description Shown when the agent is investigating main thread activity
    */
@@ -48,12 +49,14 @@ You will also be provided with external resources. Use these to ensure you give
 
 - Think about what the user wants.
 - Call any of the available functions to help you gather more information to inform your suggestions.
+- Ensure that you call all relevant functions to receive full information about relevant network requests.
 - Make suggestions that you are confident will improve the performance of the page.
 
 ## General considerations
 
 - *CRITICAL* never make the same function call twice.
 - *CRITICAL* make sure you are thorough and call the functions you have access to to give yourself the most information possible to make accurate recommendations.
+- *CRITICAL* your text output should NEVER mention the functions that you called. These are an implementation detail and not important for the user to be aware of.
 `;
 /* clang-format on */
 
@@ -134,18 +137,20 @@ export class PerformanceInsightsAgent extends AiAgent<TimelineUtils.InsightAICon
 
     this.declareFunction<Record<never, unknown>, {
       requests: string[],
-    }>('getNetworkActivity', {
-      description: 'Returns relevant network requests for the selected insight',
+    }>('getNetworkActivitySummary', {
+      description:
+          'Returns a summary of network activity for the selected insight. If you want to get more detailed information on a network request, you can pass the URL of a request into `getNetworkRequestDetail`.',
       parameters: {
         type: Host.AidaClient.ParametersTypes.OBJECT,
         description: '',
         nullable: true,
         properties: {},
       },
       displayInfoFromArgs: () => {
-        return {title: lockedString(UIStringsNotTranslated.networkActivity)};
+        return {title: lockedString(UIStringsNotTranslated.networkActivitySummary)};
       },
       handler: async () => {
+        debugLog('Function call: getNetworkActivitySummary');
         if (!this.#insight) {
           return {error: 'No insight available'};
         }
@@ -154,11 +159,46 @@ export class PerformanceInsightsAgent extends AiAgent<TimelineUtils.InsightAICon
             activeInsight.insight,
             activeInsight.parsedTrace,
         );
-        const formatted = requests.map(r => TraceEventFormatter.networkRequest(r, activeInsight.parsedTrace));
+        const formatted =
+            requests.map(r => TraceEventFormatter.networkRequest(r, activeInsight.parsedTrace, {verbose: false}));
         return {result: {requests: formatted}};
       },
     });
 
+    this.declareFunction<Record<'url', string>, {
+      request: string,
+    }>('getNetworkRequestDetail', {
+      description: 'Returns detailed debugging information about a specific network request',
+      parameters: {
+        type: Host.AidaClient.ParametersTypes.OBJECT,
+        description: '',
+        nullable: true,
+        properties: {
+          url: {
+            type: Host.AidaClient.ParametersTypes.STRING,
+            description: 'The URL of the network request',
+            nullable: false,
+          }
+        },
+      },
+      displayInfoFromArgs: params => {
+        return {title: lockedString(`Investigating network request ${params.url}…`)};
+      },
+      handler: async params => {
+        debugLog('Function call: getNetworkRequestDetail', params);
+        if (!this.#insight) {
+          return {error: 'No insight available'};
+        }
+        const activeInsight = this.#insight.getItem();
+        const request = TimelineUtils.InsightAIContext.AIQueries.networkRequest(activeInsight.parsedTrace, params.url);
+        if (!request) {
+          return {error: 'Request not found'};
+        }
+        const formatted = TraceEventFormatter.networkRequest(request, activeInsight.parsedTrace, {verbose: true});
+        return {result: {request: formatted}};
+      },
+    });
+
     this.declareFunction<Record<never, unknown>, {activity: string}>('getMainThreadActivity', {
       description: `Returns the main thread activity for the selected insight.
 The tree is represented as a call frame with a root task and a series of children.
@@ -191,6 +231,7 @@ The fields are:
         return {title: lockedString(UIStringsNotTranslated.mainThreadActivity)};
       },
       handler: async () => {
+        debugLog('Function call: getMainThreadActivity');
         if (!this.#insight) {
           return {error: 'No insight available'};
         }

front_end/panels/ai_assistance/data_formatters/PerformanceInsightFormatter.test.ts

@@ -73,20 +73,22 @@ We can break this time down into the 2 phases that combine to make up the LCP ti
   });
 
   describe('Formatting TraceEvents', () => {
-    it('formats network requests', async function() {
+    it('formats network requests in verbose mode', async function() {
       const {parsedTrace} = await TraceLoader.traceEngine(this, 'lcp-images.json.gz');
       const requestUrl = 'https://fonts.googleapis.com/css2?family=Poppins:ital,wght@1,800';
       const request = parsedTrace.NetworkRequests.byTime.find(r => r.args.data.url === requestUrl);
       assert.isOk(request);
-      const output = TraceEventFormatter.networkRequest(request, parsedTrace);
+      const output = TraceEventFormatter.networkRequest(request, parsedTrace, {verbose: true});
       const expected = `## Network request: https://fonts.googleapis.com/css2?family=Poppins:ital,wght@1,800
 Timings:
 - Start time: 37.62 ms
 - Queued at: 43.24 ms
 - Request sent at: 41.71 ms
 - Download complete at: 48.04 ms
-- Fully completed at: 51.55 ms
-- Total request duration: 13.93 ms
+- Completed at: 51.55 ms
+Durations:
+- Main thread processing duration: 3.51 ms
+- Total duration: 13.93 ms
 Status code: 200
 MIME Type: text/css
 Priority:
@@ -111,6 +113,20 @@ Response headers
 - x-xss-protection: 0
 - expires: Thu, 07 Mar 2024 21:17:02 GMT`;
 
+      assert.strictEqual(output, expected);
+    });
+    it('formats network requests in non-verbose mode', async function() {
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'lcp-images.json.gz');
+      const requestUrl = 'https://fonts.googleapis.com/css2?family=Poppins:ital,wght@1,800';
+      const request = parsedTrace.NetworkRequests.byTime.find(r => r.args.data.url === requestUrl);
+      assert.isOk(request);
+      const output = TraceEventFormatter.networkRequest(request, parsedTrace, {verbose: false});
+      const expected = `## Network request: https://fonts.googleapis.com/css2?family=Poppins:ital,wght@1,800
+- Start time: 37.62 ms
+- Duration: 13.93 ms
+- MIME type: text/css
+- This request was render blocking`;
+
       assert.strictEqual(output, expected);
     });
   });

front_end/panels/ai_assistance/data_formatters/PerformanceInsightFormatter.ts

@@ -155,6 +155,10 @@ ${phaseBulletPoints.map(phase => `- ${phase.name}: ${phase.value}`).join('\n')}`
   }
 }
 
+export interface NetworkRequestFormatOptions {
+  verbose: boolean;
+}
+
 export class TraceEventFormatter {
   /**
    * This is the data passed to a network request when the Performance Insights
@@ -165,7 +169,8 @@ export class TraceEventFormatter {
    * talk to jacktfranklin@.
    */
   static networkRequest(
-      request: Trace.Types.Events.SyntheticNetworkRequest, parsedTrace: Trace.Handlers.Types.ParsedTrace): string {
+      request: Trace.Types.Events.SyntheticNetworkRequest, parsedTrace: Trace.Handlers.Types.ParsedTrace,
+      options: NetworkRequestFormatOptions): string {
     const {url, statusCode, initialPriority, priority, fromServiceWorker, mimeType, responseHeaders, syntheticData} =
         request.args.data;
 
@@ -187,19 +192,30 @@ export class TraceEventFormatter {
       requestSent: syntheticData.sendStartTime - baseTime,
       downloadComplete: syntheticData.finishTime - baseTime,
       processingComplete: request.ts + request.dur - baseTime,
-
     } as const;
 
+    const mainThreadProcessingDuration =
+        startTimesForLifecycle.processingComplete - startTimesForLifecycle.downloadComplete;
+
     const renderBlocking = Trace.Helpers.Network.isSyntheticNetworkRequestEventRenderBlocking(request);
 
+    if (!options.verbose) {
+      return `## Network request: ${url}
+- Start time: ${formatMicro(startTimesForLifecycle.start)}
+- Duration: ${formatMicro(request.dur)}
+- MIME type: ${mimeType}${renderBlocking ? '\n- This request was render blocking' : ''}`;
+    }
+
     return `## Network request: ${url}
 Timings:
 - Start time: ${formatMicro(startTimesForLifecycle.start)}
 - Queued at: ${formatMicro(startTimesForLifecycle.queueing)}
 - Request sent at: ${formatMicro(startTimesForLifecycle.requestSent)}
 - Download complete at: ${formatMicro(startTimesForLifecycle.downloadComplete)}
-- Fully completed at: ${formatMicro(startTimesForLifecycle.processingComplete)}
-- Total request duration: ${formatMicro(request.dur)}
+- Completed at: ${formatMicro(startTimesForLifecycle.processingComplete)}
+Durations:
+- Main thread processing duration: ${formatMicro(mainThreadProcessingDuration)}
+- Total duration: ${formatMicro(request.dur)}
 Status code: ${statusCode}
 MIME Type: ${mimeType}
 Priority:

front_end/panels/timeline/utils/InsightAIContext.ts

@@ -64,6 +64,18 @@ export class AIQueries {
     return matchedRequests;
   }
 
+  /**
+   * Returns the single network request. We do not check to filter this by the
+   * bounds of the insight, because the only way that the LLM has found this
+   * request is by first inspecting a summary of relevant network requests for
+   * the given insight. So if it then looks up a request by URL, we know that
+   * is a valid and relevant request.
+   */
+  static networkRequest(parsedTrace: Trace.Handlers.Types.ParsedTrace, url: string):
+      Trace.Types.Events.SyntheticNetworkRequest|null {
+    return parsedTrace.NetworkRequests.byTime.find(r => r.args.data.url === url) ?? null;
+  }
+
   /**
    * Returns an AI Call Tree representing the activity on the main thread for
    * the relevant time range of the given insight.