AI: define getNetworkActivity for Perf Insights agent

This CL allows the LLM to call the function to get the network activity
for a given insight.

In the design doc I explored using time ranges, but so far I am going
for an approach where we query based on the active insight. I think this
reduces the chance of the AI incorrectly passing invalid or incorrect
times, but we can explore and experiment.

Bug: 394552594
Change-Id: I2d5c054391d5802ce46f5c149ee7ce194630fa68
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6276941
Auto-Submit: Jack Franklin <[email protected]>
Reviewed-by: Nikolay Vitkov <[email protected]>
Commit-Queue: Nikolay Vitkov <[email protected]>
Commit-Queue: Jack Franklin <[email protected]>

Author: jackfranklin

front_end/panels/ai_assistance/agents/PerformanceInsightsAgent.test.ts

@@ -6,12 +6,17 @@ import type * as Common from '../../../core/common/common.js';
 import type * as Host from '../../../core/host/host.js';
 import * as Trace from '../../../models/trace/trace.js';
 import {mockAidaClient} from '../../../testing/AiAssistanceHelpers.js';
+import {describeWithEnvironment} from '../../../testing/EnvironmentHelpers.js';
+import {getInsightOrError} from '../../../testing/InsightHelpers.js';
+import {TraceLoader} from '../../../testing/TraceLoader.js';
 import * as TimelineUtils from '../../timeline/utils/utils.js';
 import {
+  type ActionResponse,
   InsightContext,
   PerformanceInsightFormatter,
   PerformanceInsightsAgent,
   ResponseType,
+  TraceEventFormatter,
 } from '../ai_assistance.js';
 
 const FAKE_LCP_MODEL = {
@@ -24,7 +29,7 @@ const FAKE_LCP_MODEL = {
 } as const;
 const FAKE_PARSED_TRACE = {} as unknown as Trace.Handlers.Types.ParsedTrace;
 
-describe('PerformanceInsightsAgent', () => {
+describeWithEnvironment('PerformanceInsightsAgent', () => {
   describe('handleContextDetails', () => {
     it('outputs the right context for the initial query from the user', async () => {
       const mockInsight = new TimelineUtils.InsightAIContext.ActiveInsight(FAKE_LCP_MODEL, FAKE_PARSED_TRACE);
@@ -67,6 +72,7 @@ describe('PerformanceInsightsAgent', () => {
       ]);
     });
   });
+
   describe('enhanceQuery', () => {
     it('adds the context to the query from the user', async () => {
       const agent = new PerformanceInsightsAgent({
@@ -86,4 +92,42 @@ What is this?`;
       assert.strictEqual(finalQuery, expected);
     });
   });
+
+  describe('function calls', () => {
+    it('calls getNetworkActivity', async function() {
+      const {parsedTrace, insights} = await TraceLoader.traceEngine(this, 'lcp-images.json.gz');
+      assert.isOk(insights);
+      const [firstNav] = parsedTrace.Meta.mainFrameNavigations;
+      const lcpPhases = getInsightOrError('LCPPhases', insights, firstNav);
+      const agent = new PerformanceInsightsAgent({
+        aidaClient: mockAidaClient(
+            [[{explanation: '', functionCalls: [{name: 'getNetworkActivity', args: {}}]}], [{explanation: 'done'}]])
+      });
+      const activeInsight = new TimelineUtils.InsightAIContext.ActiveInsight(lcpPhases, parsedTrace);
+      const context = new InsightContext(activeInsight);
+
+      const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+      const action = responses.find(response => response.type === ResponseType.ACTION);
+
+      // Find the requests we expect the handler to have returned.
+      const expectedRequestUrls = [
+        'https://chromedevtools.github.io/performance-stories/lcp-large-image/index.html',
+        'https://fonts.googleapis.com/css2?family=Poppins:ital,wght@1,800',
+        'https://chromedevtools.github.io/performance-stories/lcp-large-image/app.css',
+        'https://via.placeholder.com/50.jpg', 'https://via.placeholder.com/2000.jpg'
+      ];
+
+      const requests = expectedRequestUrls.map(url => {
+        const match = parsedTrace.NetworkRequests.byTime.find(r => r.args.data.url === url);
+        assert.isOk(match, `no request found for ${url}`);
+        return match;
+      });
+
+      const expectedRequestsOutput = requests.map(r => TraceEventFormatter.networkRequest(r, parsedTrace));
+      const expectedOutput = JSON.stringify({requests: expectedRequestsOutput});
+
+      assert.exists(action);
+      assert.deepEqual(action, {type: 'action' as ActionResponse['type'], output: expectedOutput, canceled: false});
+    });
+  });
 });

front_end/panels/ai_assistance/agents/PerformanceInsightsAgent.ts

@@ -4,9 +4,9 @@
 
 import * as Host from '../../../core/host/host.js';
 import type * as Lit from '../../../ui/lit/lit.js';
-import type * as TimelineUtils from '../../timeline/utils/utils.js';
+import * as TimelineUtils from '../../timeline/utils/utils.js';
 import * as PanelUtils from '../../utils/utils.js';
-import {PerformanceInsightFormatter} from '../data_formatters/PerformanceInsightFormatter.js';
+import {PerformanceInsightFormatter, TraceEventFormatter} from '../data_formatters/PerformanceInsightFormatter.js';
 
 import {
   type AgentOptions as BaseAgentOptions,
@@ -79,8 +79,6 @@ export class InsightContext extends ConversationContext<TimelineUtils.InsightAIC
 }
 
 export class PerformanceInsightsAgent extends AiAgent<TimelineUtils.InsightAIContext.ActiveInsight> {
-  // TODO: make use of the Insight.
-  // eslint-disable-next-line no-unused-private-class-members
   #insight: ConversationContext<TimelineUtils.InsightAIContext.ActiveInsight>|undefined;
 
   override async *
@@ -113,7 +111,30 @@ export class PerformanceInsightsAgent extends AiAgent<TimelineUtils.InsightAICon
 
   constructor(opts: BaseAgentOptions) {
     super(opts);
-    // TODO: define the set of functions for the LLM.
+
+    this.declareFunction<Record<never, unknown>, {
+      requests: string[],
+    }>('getNetworkActivity', {
+      description: 'Returns relevant network requests for the selected insight',
+      parameters: {
+        type: Host.AidaClient.ParametersTypes.OBJECT,
+        description: '',
+        nullable: true,
+        properties: {},
+      },
+      handler: async () => {
+        if (!this.#insight) {
+          return {error: 'No insight available'};
+        }
+        const activeInsight = this.#insight.getItem();
+        const requests = TimelineUtils.InsightAIContext.AIQueries.networkRequests(
+            activeInsight.insight,
+            activeInsight.parsedTrace,
+        );
+        const formatted = requests.map(r => TraceEventFormatter.networkRequest(r, activeInsight.parsedTrace));
+        return {result: {requests: formatted}};
+      },
+    });
   }
 
   override async enhanceQuery(

front_end/panels/timeline/utils/InsightAIContext.ts

@@ -13,7 +13,6 @@ import * as Trace from '../../../models/trace/trace.js';
  */
 export class ActiveInsight {
   #insight: Trace.Insights.Types.InsightModel<{}, {}>;
-  // eslint-disable-next-line no-unused-private-class-members
   #parsedTrace: Trace.Handlers.Types.ParsedTrace;
 
   constructor(insight: Trace.Insights.Types.InsightModel<{}, {}>, parsedTrace: Trace.Handlers.Types.ParsedTrace) {
@@ -24,6 +23,9 @@ export class ActiveInsight {
   get insight(): Readonly<Trace.Insights.Types.InsightModel<{}, {}>> {
     return this.#insight;
   }
+  get parsedTrace(): Trace.Handlers.Types.ParsedTrace {
+    return this.#parsedTrace;
+  }
 
   title(): string {
     return this.#insight.title;