RPP: add auto-annotation label generation

Fixed: 405061899
Change-Id: Idf06927221664dfa45a1ba3215fb5f2a6ad99084
Also-by: [email protected]
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6381853
Reviewed-by: Ergün Erdoğmuş <[email protected]>
Commit-Queue: Jack Franklin <[email protected]>
Auto-Submit: Jack Franklin <[email protected]>

Author: jackfranklin

front_end/models/ai_assistance/BUILD.gn

@@ -56,6 +56,7 @@ devtools_entrypoint("bundle") {
     ":*",
     "../../entrypoints/main/*",
     "../../panels/ai_assistance/*",
+    "../../panels/timeline/*",
   ]
 
   visibility += devtools_models_visibility

front_end/models/ai_assistance/agents/PerformanceAgent.test.ts

@@ -226,4 +226,22 @@ self: 3
       assert.isFalse(enhancedQuery3.includes(mockAiCallTree.serialize()));
     });
   });
+
+  describe('generating an AI entry label', () => {
+    it('generates a label from the final answer and trims newlines', async function() {
+      const agent = new PerformanceAgent({
+        aidaClient: mockAidaClient([[{
+          explanation: 'hello world\n',
+        }]]),
+      });
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'web-dev-with-commit.json.gz');
+      const evalScriptEvent = parsedTrace.Renderer.allTraceEntries.find(
+          event => event.name === Trace.Types.Events.Name.EVALUATE_SCRIPT && event.ts === 122411195649);
+      assert.exists(evalScriptEvent);
+      const aiCallTree = TimelineUtils.AICallTree.AICallTree.fromEvent(evalScriptEvent, parsedTrace);
+      assert.isOk(aiCallTree);
+      const label = await agent.generateAIEntryLabel(aiCallTree);
+      assert.strictEqual(label, 'hello world');
+    });
+  });
 });

front_end/models/ai_assistance/agents/PerformanceAgent.ts

@@ -245,4 +245,22 @@ export class PerformanceAgent extends AiAgent<TimelineUtils.AICallTree.AICallTre
     const perfEnhancementQuery = treeStr ? `${treeStr}\n\n# User request\n\n` : '';
     return `${perfEnhancementQuery}${query}`;
   }
+
+  /**
+   * Used in the Performance panel to automatically generate a label for a selected entry.
+   */
+  async generateAIEntryLabel(callTree: TimelineUtils.AICallTree.AICallTree): Promise<string> {
+    const context = new CallTreeContext(callTree);
+    const response = await Array.fromAsync(this.run(AI_LABEL_GENERATION_PROMPT, {selected: context}));
+    const lastResponse = response.at(-1);
+    if (lastResponse && lastResponse.type === ResponseType.ANSWER && lastResponse.complete === true) {
+      return lastResponse.text.trim();
+    }
+    throw new Error('Failed to generate AI entry label');
+  }
 }
+
+const AI_LABEL_GENERATION_PROMPT =
+    `Generate a very short label for the selected callframe of only a few words describing what the callframe is broadly doing, but provide the most important information for debugging performance.
+
+Important: Describe selected callframe in just 1 sentence under 80 characters without line breaks. We will use your response for this callframe annotation so start with the sentence directly with what the callframe is doing and do not return any other text.`;

front_end/panels/settings/AISettingsTab.ts

@@ -342,7 +342,7 @@ export class AISettingsTab extends LegacyWrapper.LegacyWrapper.WrappableComponen
           text: noLogging ? i18nString(UIStrings.generatedAiAnnotationsSendDataNoLogging) :
                             i18nString(UIStrings.generatedAiAnnotationsSendData)
         }],
-        // TODO: Add a relevant link
+        // TODO(b/405316456): Add a relevant link here once we have written the documentation.
         learnMoreLink: {url: '', linkJSLogContext: 'learn-more.ai-annotations'},
         settingExpandState: {
           isSettingExpanded: false,

front_end/panels/timeline/TimelineFlameChartView.ts

@@ -273,6 +273,9 @@ export class TimelineFlameChartView extends Common.ObjectWrapper.eventMixin<Even
         networkProvider: this.networkDataProvider,
       },
       entryQueries: {
+        parsedTrace: () => {
+          return this.#parsedTrace;
+        },
         isEntryCollapsedByUser: (entry: Trace.Types.Events.Event): boolean => {
           return ModificationsManager.activeManager()?.getEntriesFilter().entryIsInvisible(entry) ?? false;
         },

front_end/panels/timeline/overlays/BUILD.gn

@@ -10,12 +10,14 @@ devtools_module("overlays") {
   sources = [ "OverlaysImpl.ts" ]
 
   deps = [
+    "../../../core/common:bundle",
+    "../../../core/i18n:bundle",
     "../../../core/platform:bundle",
     "../../../models/trace:bundle",
     "../../../services/trace_bounds:bundle",
     "../../../ui/legacy/components/perf_ui:bundle",
-    "../../../ui/lit:bundle",
-    "../../timeline/utils:bundle",
+    "../../../ui/visual_logging:bundle",
+    "../utils:bundle",
     "./components:bundle",
   ]
 }

front_end/panels/timeline/overlays/OverlaysImpl.test.ts

@@ -3,7 +3,9 @@
 // found in the LICENSE file.
 
 import * as Common from '../../../core/common/common.js';
+import * as AiAssistanceModels from '../../../models/ai_assistance/ai_assistance.js';
 import * as Trace from '../../../models/trace/trace.js';
+import {mockAidaClient} from '../../../testing/AiAssistanceHelpers.js';
 import {cleanTextContent, dispatchClickEvent} from '../../../testing/DOMHelpers.js';
 import {describeWithEnvironment, updateHostConfig} from '../../../testing/EnvironmentHelpers.js';
 import {
@@ -22,6 +24,9 @@ import * as Components from './components/components.js';
 import * as Overlays from './overlays.js';
 
 const FAKE_OVERLAY_ENTRY_QUERIES: Overlays.Overlays.OverlayEntryQueries = {
+  parsedTrace() {
+    return null;
+  },
   isEntryCollapsedByUser() {
     return false;
   },
@@ -277,7 +282,12 @@ describeWithEnvironment('Overlays', () => {
           network: networkFlameChartsContainer,
         },
         charts,
-        entryQueries: FAKE_OVERLAY_ENTRY_QUERIES,
+        entryQueries: {
+          ...FAKE_OVERLAY_ENTRY_QUERIES,
+          parsedTrace() {
+            return parsedTrace;
+          },
+        },
       });
       const currManager = Timeline.ModificationsManager.ModificationsManager.activeManager();
       // The Annotations Overlays are added through the ModificationsManager listener
@@ -322,6 +332,7 @@ describeWithEnvironment('Overlays', () => {
       inputField: HTMLElement,
       overlays: Overlays.Overlays.Overlays,
       event: Trace.Types.Events.Event,
+      component: Components.EntryLabelOverlay.EntryLabelOverlay,
     }> {
       updateHostConfig({
         devToolsAiGeneratedTimelineLabels: {
@@ -341,6 +352,7 @@ describeWithEnvironment('Overlays', () => {
         label: label ?? '',
       });
       await overlays.update();
+      await RenderCoordinator.done();
 
       // Ensure that the overlay was created.
       const overlayDOM = container.querySelector<HTMLElement>('.overlay-type-ENTRY_LABEL');
@@ -353,7 +365,7 @@ describeWithEnvironment('Overlays', () => {
       const inputField = elementsWrapper.querySelector<HTMLElement>('.input-field');
       assert.isOk(inputField);
 
-      return {elementsWrapper, inputField, overlays, event};
+      return {elementsWrapper, inputField, overlays, event, component};
     }
 
     it('can render an entry selected overlay', async function() {
@@ -483,6 +495,7 @@ describeWithEnvironment('Overlays', () => {
 
          // Double click on the label box to make it editable and focus on it
          inputField.dispatchEvent(new FocusEvent('dblclick', {bubbles: true}));
+         await RenderCoordinator.done();
 
          const aiLabelButtonWrapper =
              elementsWrapper.querySelector<HTMLElement>('.ai-label-button-wrapper') as HTMLSpanElement;
@@ -493,6 +506,7 @@ describeWithEnvironment('Overlays', () => {
          // This dialog should not be visible unless the `generate annotation` button is clicked
          assert.isFalse(showFreDialogStub.called, 'Expected FreDialog to be not shown but it\'s shown');
          aiButton.dispatchEvent(new FocusEvent('click', {bubbles: true}));
+         await RenderCoordinator.done();
 
          // This dialog should be visible
          assert.isTrue(showFreDialogStub.called, 'Expected FreDialog to be shown but it\'s not shown');
@@ -558,6 +572,35 @@ describeWithEnvironment('Overlays', () => {
       assert.strictEqual(inputField?.innerText, 'entry label');
     });
 
+    it('generates a label when the user clicks "Generate" if the setting is enabled', async function() {
+      const {elementsWrapper, inputField, component} = await createAnnotationsLabelElement(this, 'web-dev.json.gz', 50);
+      Common.Settings.moduleSetting('ai-annotations-enabled').set(true);
+
+      const generateButton = elementsWrapper.querySelector<HTMLElement>('.ai-label-button');
+      assert.isOk(generateButton, 'could not find "Generate label" button');
+      assert.isTrue(generateButton.classList.contains('enabled'));
+      const agent = new AiAssistanceModels.PerformanceAgent({
+        aidaClient: mockAidaClient([[{
+          explanation: 'This is an interesting entry',
+          metadata: {
+            rpcGlobalId: 123,
+          }
+        }]])
+      });
+      component.overrideAIAgentForTest(agent);
+
+      // The Agent call is async, so wait for the change event on the label to ensure the UI is updated.
+      const changeEvent = new Promise<void>(resolve => {
+        component.addEventListener(
+            Components.EntryLabelOverlay.EntryLabelChangeEvent.eventName, () => resolve(), {once: true});
+      });
+      dispatchClickEvent(generateButton);
+      await RenderCoordinator.done();
+      await changeEvent;
+
+      assert.strictEqual(inputField.innerHTML, 'This is an interesting entry');
+    });
+
     it('Correct security tooltip on the `generate ai label` info icon hover for the users with logging enabled',
        async function() {
          const {elementsWrapper, inputField} =

front_end/panels/timeline/overlays/OverlaysImpl.ts

@@ -7,7 +7,7 @@ import * as Platform from '../../../core/platform/platform.js';
 import * as Trace from '../../../models/trace/trace.js';
 import type * as PerfUI from '../../../ui/legacy/components/perf_ui/perf_ui.js';
 import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js';
-import {EntryStyles} from '../../timeline/utils/utils.js';
+import * as Utils from '../utils/utils.js';
 
 import * as Components from './components/components.js';
 
@@ -386,6 +386,7 @@ export interface TimelineCharts {
 }
 
 export interface OverlayEntryQueries {
+  parsedTrace: () => Trace.Handlers.Types.ParsedTrace | null;
   isEntryCollapsedByUser: (entry: Trace.Types.Events.Event) => boolean;
   firstVisibleParentForEntry: (entry: Trace.Types.Events.Event) => Trace.Types.Events.Event | null;
 }
@@ -1528,6 +1529,11 @@ export class Overlays extends EventTarget {
       case 'ENTRY_LABEL': {
         const shouldDrawLabelBelowEntry = Trace.Types.Events.isLegacyTimelineFrame(overlay.entry);
         const component = new Components.EntryLabelOverlay.EntryLabelOverlay(overlay.label, shouldDrawLabelBelowEntry);
+        // Generate the AI Call Tree for the AI Auto-Annotation feature.
+        const parsedTrace = this.#queries.parsedTrace();
+        const callTree = parsedTrace ? Utils.AICallTree.AICallTree.fromEvent(overlay.entry, parsedTrace) : null;
+        component.callTree = callTree;
+
         component.addEventListener(Components.EntryLabelOverlay.EmptyEntryLabelRemoveEvent.eventName, () => {
           this.dispatchEvent(new AnnotationOverlayActionEvent(overlay, 'Remove'));
         });
@@ -1602,7 +1608,7 @@ export class Overlays extends EventTarget {
         return overlayElement;
       }
       case 'TIMINGS_MARKER': {
-        const {color} = EntryStyles.markerDetailsForEvent(overlay.entries[0]);
+        const {color} = Utils.EntryStyles.markerDetailsForEvent(overlay.entries[0]);
         const markersComponent = this.#createTimingsMarkerElement(overlay);
         overlayElement.appendChild(markersComponent);
         overlayElement.style.backgroundColor = color;
@@ -1671,7 +1677,7 @@ export class Overlays extends EventTarget {
     const markers = document.createElement('div');
     markers.classList.add('markers');
     for (const entry of overlay.entries) {
-      const {color, title} = EntryStyles.markerDetailsForEvent(entry);
+      const {color, title} = Utils.EntryStyles.markerDetailsForEvent(entry);
       const marker = document.createElement('div');
       marker.classList.add('marker-title');
       marker.textContent = title;

front_end/panels/timeline/overlays/components/BUILD.gn

@@ -25,14 +25,21 @@ devtools_module("components") {
   ]
 
   deps = [
+    "../../../../core/host:bundle",
     "../../../../core/i18n:bundle",
+    "../../../../core/platform:bundle",
+    "../../../../core/root:bundle",
+    "../../../../models/ai_assistance:bundle",
     "../../../../models/trace:bundle",
     "../../../../panels/common:bundle",
     "../../../../ui/components/helpers:bundle",
     "../../../../ui/components/icon_button:bundle",
     "../../../../ui/components/tooltips:bundle",
+    "../../../../ui/legacy/theme_support:bundle",
     "../../../../ui/lit:bundle",
     "../../../../ui/visual_logging:bundle",
+    "../../../common:bundle",
+    "../../utils:bundle",
   ]
 }