Add new test target for multimodal support for Elements entrypoint

Bug: 407692633
Change-Id: I573bcb707485eb46c6eb30906910e397ebc3665e
Fixed: 407692633
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6438591
Auto-Submit: Samiya Caur <[email protected]>
Reviewed-by: Ergün Erdoğmuş <[email protected]>
Commit-Queue: Ergün Erdoğmuş <[email protected]>

Author: Samiya Caur

scripts/ai_assistance/README.md

@@ -37,6 +37,7 @@ node scripts/ai_assistance/auto-run.js --label title-change --example-urls <exam
 ## `--test-target` values
 
 * `elements`: tests the entrypoint via right clicking on an element in the Elements panel.
+* `elements-multimodal`: tests the multimodal support for Elements entrypoint by providing screenshot input with the prompt.
 * `performance-main-thread`: tests the entrypoint via right clicking on an event in the Performance panel main thread.
 * `performance-insights`: tests the entrypoint via the "Ask AI" button shown on an individual Insight in the Performance panel sidebar.
 

scripts/ai_assistance/auto-run.js

@@ -21,32 +21,33 @@ function formatElapsedTime() {
   return `${numberFormatter.format((performance.now() - startTime) / 1000)}s`;
 }
 
-const yargsInput = yargs(hideBin(process.argv))
-                       .option('example-urls', {
-                         string: true,
-                         type: 'array',
-                         demandOption: true,
-                       })
-                       .option('parallel', {
-                         boolean: true,
-                         default: true,
-                       })
-                       .option('times', {
-                         describe: 'How many times do you want to run an example?',
-                         number: true,
-                         default: 1,
-                       })
-                       .option('label', {string: true, default: 'run'})
-                       .option('include-follow-up', {
-                         boolean: true,
-                         default: false,
-                       })
-                       .option('test-target', {
-                         describe: 'Which panel do you want to run the examples against?',
-                         choices: ['elements', 'performance-main-thread', 'performance-insights'],
-                         demandOption: true,
-                       })
-                       .parseSync();
+const yargsInput =
+    yargs(hideBin(process.argv))
+        .option('example-urls', {
+          string: true,
+          type: 'array',
+          demandOption: true,
+        })
+        .option('parallel', {
+          boolean: true,
+          default: true,
+        })
+        .option('times', {
+          describe: 'How many times do you want to run an example?',
+          number: true,
+          default: 1,
+        })
+        .option('label', {string: true, default: 'run'})
+        .option('include-follow-up', {
+          boolean: true,
+          default: false,
+        })
+        .option('test-target', {
+          describe: 'Which panel do you want to run the examples against?',
+          choices: ['elements', 'performance-main-thread', 'performance-insights', 'elements-multimodal'],
+          demandOption: true,
+        })
+        .parseSync();
 
 // Map the args to a more accurate interface for better type safety.
 const userArgs = /** @type {import('./types.d.ts').YargsInput} **/ (yargsInput);
@@ -409,7 +410,7 @@ class Example {
     await devtoolsPage.keyboard.press('Escape');
     await devtoolsPage.keyboard.press('Escape');
 
-    if (userArgs.testTarget === 'elements') {
+    if (userArgs.testTarget === 'elements' || userArgs.testTarget === 'elements-multimodal') {
       await devtoolsPage.locator(':scope >>> #tab-elements').setTimeout(5000).click();
       this.log('[Info]: Opened Elements panel');
 
@@ -513,6 +514,7 @@ class Example {
   #getLocator() {
     switch (userArgs.testTarget) {
       case 'elements':
+      case 'elements-multimodal':
         return 'aria/Ask a question about the selected element';
       case 'performance-main-thread':
         return 'aria/Ask a question about the selected item and its call tree';
@@ -551,6 +553,10 @@ class Example {
       }
       const devtoolsPage = this.#devtoolsPage;
 
+      if (userArgs.testTarget === 'elements-multimodal') {
+        await devtoolsPage.locator('aria/Take screenshot').click();
+      }
+
       const inputSelector = this.#getLocator();
       await devtoolsPage.locator(inputSelector).click();
       await devtoolsPage.locator(inputSelector).fill(query);

scripts/ai_assistance/types.d.ts

@@ -50,7 +50,7 @@ export interface YargsInput {
   times: number;
   testTarget: TestTarget;
 }
-export type TestTarget = 'elements'|'performance-main-thread'|'performance-insights';
+export type TestTarget = 'elements'|'performance-main-thread'|'performance-insights'|'elements-multimodal';
 
 // Clang cannot handle the Record<> syntax over multiple lines, it seems.
 /* clang-format off */