[AiAssistance] Send multimodal input prompt as part of user query

Bug: 393036441
Change-Id: I19aff5affd7f45943b2d6981908d0c569ea16bb7
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6262874
Auto-Submit: Samiya Caur <[email protected]>
Commit-Queue: Samiya Caur <[email protected]>
Reviewed-by: Alex Rudenko <[email protected]>

Author: Samiya Caur

front_end/panels/ai_assistance/agents/AiAgent.ts

@@ -244,7 +244,7 @@ export abstract class AiAgent<T> {
     this.confirmSideEffect = opts.confirmSideEffectForTest ?? (() => Promise.withResolvers());
   }
 
-  async enhanceQuery(query: string, selected: ConversationContext<T>|null): Promise<string>;
+  async enhanceQuery(query: string, selected: ConversationContext<T>|null, hasImageInput?: boolean): Promise<string>;
   async enhanceQuery(query: string): Promise<string> {
     return query;
   }
@@ -362,8 +362,7 @@ export abstract class AiAgent<T> {
       this.#context = options.selected;
     }
 
-    const enhancedQuery = await this.enhanceQuery(initialQuery, options.selected);
-
+    const enhancedQuery = await this.enhanceQuery(initialQuery, options.selected, Boolean(imageInput));
     Host.userMetrics.freestylerQueryLength(enhancedQuery.length);
 
     let query: Host.AidaClient.Part|Host.AidaClient.Part[];

front_end/panels/ai_assistance/agents/StylingAgent.test.ts

@@ -17,13 +17,14 @@ const {StylingAgent, ErrorType} = AiAssistance;
 describeWithEnvironment('StylingAgent', () => {
   function mockHostConfig(
       modelId?: string, temperature?: number, userTier?: string,
-      executionMode?: Root.Runtime.HostConfigFreestylerExecutionMode) {
+      executionMode?: Root.Runtime.HostConfigFreestylerExecutionMode, multimodal?: boolean) {
     updateHostConfig({
       devToolsFreestyler: {
         modelId,
         temperature,
         userTier,
         executionMode,
+        multimodal,
       },
     });
   }
@@ -1225,6 +1226,67 @@ STOP
     });
   });
 
+  describe('enhanceQuery', () => {
+    const agent = new StylingAgent({
+      aidaClient: mockAidaClient(),
+    });
+
+    beforeEach(() => {
+      element.simpleSelector.returns('div#myElement');
+      element.getChildNodesPromise.resolves(null);
+    });
+
+    it('does not add multimodal input evaluation prompt when multimodal is disabled', async () => {
+      mockHostConfig('test model');
+      const enhancedQuery = await agent.enhanceQuery('test query', new AiAssistance.NodeContext(element), true);
+
+      assert.strictEqual(
+          enhancedQuery,
+          '# Inspected element\n\n* Its selector is `div#myElement`\n\n# User request\n\nQUERY: test query',
+      );
+    });
+
+    it('does not add multimodal input evaluation prompt when multimodal is enabled but hasImageInput is false',
+       async () => {
+         mockHostConfig('test model', 1, 'PUBLIC', Root.Runtime.HostConfigFreestylerExecutionMode.NO_SCRIPTS, true);
+         const enhancedQuery = await agent.enhanceQuery('test query', new AiAssistance.NodeContext(element), false);
+
+         assert.strictEqual(
+             enhancedQuery,
+             '# Inspected element\n\n* Its selector is `div#myElement`\n\n# User request\n\nQUERY: test query',
+         );
+       });
+
+    it('adds multimodal input evaluation prompt when multimodal is enabled and hasImageInput is true', async () => {
+      mockHostConfig('test model', 1, 'PUBLIC', Root.Runtime.HostConfigFreestylerExecutionMode.NO_SCRIPTS, true);
+      const enhancedQuery = await agent.enhanceQuery('test query', new AiAssistance.NodeContext(element), true);
+
+      assert.strictEqual(
+          enhancedQuery,
+          `The user has provided you a screenshot of the page (as visible in the viewport) in base64-encoded format. You SHOULD use it while answering user's queries.
+
+# Considerations for evaluating image:
+* Pay close attention to the spatial details as well as the visual appearance of the selected element in the image, particularly in relation to layout, spacing, and styling.
+* Try to connect the screenshot to actual DOM elements in the page.
+* Analyze the image to identify the layout structure surrounding the element, including the positioning of neighboring elements.
+* Extract visual information from the image, such as colors, fonts, spacing, and sizes, that might be relevant to the user's query.
+* If the image suggests responsiveness issues (e.g., cropped content, overlapping elements), consider those in your response.
+* Consider the surrounding elements and overall layout in the image, but prioritize the selected element's styling and positioning.
+
+* As part of THOUGHT, evaluate the image to gather data that might be needed to answer the question.
+In case query is related to the image, ALWAYS first use image evaluation to get all details from the image. ONLY after you have all data needed from image, you should move to other steps.
+
+# Inspected element
+
+* Its selector is \`div#myElement\`
+
+# User request
+
+QUERY: test query`,
+      );
+    });
+  });
+
   describe('HostConfigFreestylerExecutionMode', () => {
     function getMockClient() {
       return mockAidaClient([

front_end/panels/ai_assistance/agents/StylingAgent.ts

@@ -140,6 +140,21 @@ OBSERVATION: {"elementStyles":{"display":"block","visibility":"visible","positio
 
 ANSWER: Even though the popup itself has a z-index of 3, its parent container has position: relative and z-index: 1. This creates a new stacking context for the popup. Because the "background" div has a z-index of 2, which is higher than the stacking context of the popup, it is rendered on top, obscuring the popup.
 SUGGESTIONS: ["What is a stacking context?", "How can I change the stacking order?"]
+`;
+
+const promptForMultimodalInputEvaluation = `The user has provided you a screenshot of the page (as visible in the viewport) in base64-encoded format. You SHOULD use it while answering user's queries.
+
+# Considerations for evaluating image:
+* Pay close attention to the spatial details as well as the visual appearance of the selected element in the image, particularly in relation to layout, spacing, and styling.
+* Try to connect the screenshot to actual DOM elements in the page.
+* Analyze the image to identify the layout structure surrounding the element, including the positioning of neighboring elements.
+* Extract visual information from the image, such as colors, fonts, spacing, and sizes, that might be relevant to the user's query.
+* If the image suggests responsiveness issues (e.g., cropped content, overlapping elements), consider those in your response.
+* Consider the surrounding elements and overall layout in the image, but prioritize the selected element's styling and positioning.
+
+* As part of THOUGHT, evaluate the image to gather data that might be needed to answer the question.
+In case query is related to the image, ALWAYS first use image evaluation to get all details from the image. ONLY after you have all data needed from image, you should move to other steps.
+
 `;
 /* clang-format on */
 
@@ -265,6 +280,11 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
     };
   }
 
+  get multimodalInputEnabled(): boolean {
+    const {hostConfig} = Root.Runtime;
+    return Boolean(hostConfig.devToolsFreestyler?.multimodal);
+  }
+
   override parseResponse(response: Host.AidaClient.AidaResponse): ParsedResponse {
     if (response.functionCalls) {
       throw new Error('Function calling not supported yet');
@@ -727,13 +747,16 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
     };
   }
 
-  override async enhanceQuery(query: string, selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null):
-      Promise<string> {
-    const elementEnchantmentQuery = selectedElement ?
+  override async enhanceQuery(
+      query: string, selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null,
+      hasImageInput?: boolean): Promise<string> {
+    const elementEnchancementQuery = selectedElement ?
         `# Inspected element\n\n${
             await StylingAgent.describeElement(selectedElement.getItem())}\n\n# User request\n\n` :
         '';
-    return `${elementEnchantmentQuery}QUERY: ${query}`;
+    const multimodalInputEnhancementQuery =
+        this.multimodalInputEnabled && hasImageInput ? promptForMultimodalInputEvaluation : '';
+    return `${multimodalInputEnhancementQuery}${elementEnchancementQuery}QUERY: ${query}`;
   }
 
   override formatParsedAnswer({answer}: ParsedAnswer): string {