get xpath by coordinate (#952)

* feat(web-integration): get xpath by coordinate like (x,y)

* test(shared): add tests

* feat(shared): add support for SVG elements and improve XPath generation for text nodes

Author: quanru

packages/android/src/page/index.ts

@@ -783,6 +783,13 @@ ${Object.keys(size)
     throw new Error('Not implemented');
   }
 
+  async getXpathsByPoint(
+    point: Point,
+    isOrderSensitive: boolean,
+  ): Promise<string[]> {
+    throw new Error('Not implemented');
+  }
+
   async getElementInfoByXpath(xpath: string): Promise<ElementInfo> {
     throw new Error('Not implemented');
   }

packages/core/src/ai-model/inspect.ts

@@ -76,6 +76,7 @@ export async function AiLocateElement<
   rawResponse: string;
   elementById: ElementById;
   usage?: AIUsageInfo;
+  isOrderSensitive?: boolean;
 }> {
   const { context, targetElementDescription, callAI } = options;
   const { screenshotBase64 } = context;
@@ -208,6 +209,12 @@ export async function AiLocateElement<
     rawResponse,
     elementById,
     usage: res.usage,
+    isOrderSensitive:
+      typeof res.content === 'object' &&
+      res.content !== null &&
+      'isOrderSensitive' in res.content
+        ? (res.content as any).isOrderSensitive
+        : undefined,
   };
 }
 

packages/core/src/ai-model/prompt/llm-locator.ts

@@ -14,31 +14,47 @@ You are an expert in software testing.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Give the coordinates of the element that matches the user's description best in the screenshot.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Output Format:
 \`\`\`json
 {
   "bbox": [number, number, number, number],  // ${bboxComment}
-  "errors"?: string[]
+  "errors"?: string[],
+  "isOrderSensitive": boolean // Whether the targetElementDescription is order-sensitive (true/false)
 }
 \`\`\`
 
 Fields:
 * \`bbox\` is the bounding box of the element that matches the user's description best in the screenshot
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
 * \`errors\` is an optional array of error messages (if any)
 
-For example, when an element is found:
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
+
+For example, when an element is found and the description is order-sensitive:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
 
-When no element is found:
+When no element is found and the description is not order-sensitive:
 \`\`\`json
 {
   "bbox": [],
+  "isOrderSensitive": false,
   "errors": ["I can see ..., but {some element} is not found"]
 }
 \`\`\`
@@ -52,6 +68,7 @@ You are an expert in software page image (2D) and page element text analysis.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Return JSON data containing the selection reason and element ID.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Skills:
 - Image analysis and recognition
@@ -63,6 +80,7 @@ You are an expert in software page image (2D) and page element text analysis.
 2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.
 3. Found the required number of elements
 4. Return JSON data containing the selection reason and element ID.
+5. Judge whether the user's description is order-sensitive (see below for definition and examples).
 
 ## Constraints:
 - Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
@@ -72,6 +90,10 @@ You are an expert in software page image (2D) and page element text analysis.
 - The returned data must conform to the specified JSON format.
 - The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)
 
+## Order-Sensitive Definition:
+- If the description contains phrases like "the third item in the list", "the last button", "the first input box", "the second row", etc., it is order-sensitive (isOrderSensitive = true).
+- If the description is like "confirm button", "search box", "password input", etc., it is not order-sensitive (isOrderSensitive = false).
+
 ## Output Format:
 
 Please return the result in JSON format as follows:
@@ -87,6 +109,7 @@ Please return the result in JSON format as follows:
     }
     // More elements...
   ],
+  "isOrderSensitive": true, // or false, depending on the user's description
   "errors": [] // Array of strings containing any error messages
 }
 \`\`\`
@@ -175,6 +198,7 @@ Output Example:
       "id": "1231"
     }
   ],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
@@ -213,6 +237,11 @@ export const locatorSchema: ResponseFormatJSONSchema = {
           },
           description: 'List of found elements',
         },
+        isOrderSensitive: {
+          type: 'boolean',
+          description:
+            'Whether the targetElementDescription is order-sensitive (true/false)',
+        },
         errors: {
           type: 'array',
           items: {
@@ -221,7 +250,7 @@ export const locatorSchema: ResponseFormatJSONSchema = {
           description: 'List of error messages, if any',
         },
       },
-      required: ['elements', 'errors'],
+      required: ['elements', 'isOrderSensitive', 'errors'],
       additionalProperties: false,
     },
   },

packages/core/src/insight/index.ts

@@ -140,13 +140,19 @@ export default class Insight<
     }
 
     const startTime = Date.now();
-    const { parseResult, rect, elementById, rawResponse, usage } =
-      await AiLocateElement({
-        callAI: callAI || this.aiVendorFn,
-        context,
-        targetElementDescription: queryPrompt,
-        searchConfig: searchAreaResponse,
-      });
+    const {
+      parseResult,
+      rect,
+      elementById,
+      rawResponse,
+      usage,
+      isOrderSensitive,
+    } = await AiLocateElement({
+      callAI: callAI || this.aiVendorFn,
+      context,
+      targetElementDescription: queryPrompt,
+      searchConfig: searchAreaResponse,
+    });
 
     const timeCost = Date.now() - startTime;
     const taskInfo: InsightTaskInfo = {
@@ -219,6 +225,7 @@ export default class Insight<
           rect: elements[0]!.rect,
           xpaths: elements[0]!.xpaths || [],
           attributes: elements[0]!.attributes,
+          isOrderSensitive,
         },
         rect,
       };

packages/core/src/types.ts

@@ -65,11 +65,13 @@ export interface AIElementLocatorResponse {
     xpaths?: string[];
   }[];
   bbox?: [number, number, number, number];
+  isOrderSensitive?: boolean;
   errors?: string[];
 }
 
 export interface AIElementCoordinatesResponse {
   bbox: [number, number, number, number];
+  isOrderSensitive?: boolean;
   errors?: string[];
 }
 
@@ -156,6 +158,7 @@ export type LocateResultElement = {
     nodeType: NodeType;
     [key: string]: string;
   };
+  isOrderSensitive?: boolean;
 };
 
 export interface LocateResult {

packages/core/tests/unit-test/executor/__snapshots__/index.test.ts.snap

@@ -12,6 +12,7 @@ exports[`executor > insight - basic run 1`] = `
     ],
     "id": "0",
     "indexId": undefined,
+    "isOrderSensitive": undefined,
     "rect": {
       "height": 100,
       "left": 200,

packages/core/tests/unit-test/prompt/__snapshots__/prompt.test.ts.snap

@@ -112,6 +112,7 @@ You are an expert in software page image (2D) and page element text analysis.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Return JSON data containing the selection reason and element ID.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Skills:
 - Image analysis and recognition
@@ -123,6 +124,7 @@ You are an expert in software page image (2D) and page element text analysis.
 2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.
 3. Found the required number of elements
 4. Return JSON data containing the selection reason and element ID.
+5. Judge whether the user's description is order-sensitive (see below for definition and examples).
 
 ## Constraints:
 - Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
@@ -132,6 +134,10 @@ You are an expert in software page image (2D) and page element text analysis.
 - The returned data must conform to the specified JSON format.
 - The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)
 
+## Order-Sensitive Definition:
+- If the description contains phrases like "the third item in the list", "the last button", "the first input box", "the second row", etc., it is order-sensitive (isOrderSensitive = true).
+- If the description is like "confirm button", "search box", "password input", etc., it is not order-sensitive (isOrderSensitive = false).
+
 ## Output Format:
 
 Please return the result in JSON format as follows:
@@ -147,6 +153,7 @@ Please return the result in JSON format as follows:
     }
     // More elements...
   ],
+  "isOrderSensitive": true, // or false, depending on the user's description
   "errors": [] // Array of strings containing any error messages
 }
 \`\`\`
@@ -235,6 +242,7 @@ Output Example:
       "id": "1231"
     }
   ],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
@@ -250,31 +258,47 @@ You are an expert in software testing.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Give the coordinates of the element that matches the user's description best in the screenshot.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Output Format:
 \`\`\`json
 {
   "bbox": [number, number, number, number],  // 2d bounding box as [ymin, xmin, ymax, xmax]
-  "errors"?: string[]
+  "errors"?: string[],
+  "isOrderSensitive": boolean // Whether the targetElementDescription is order-sensitive (true/false)
 }
 \`\`\`
 
 Fields:
 * \`bbox\` is the bounding box of the element that matches the user's description best in the screenshot
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
 * \`errors\` is an optional array of error messages (if any)
 
-For example, when an element is found:
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
+
+For example, when an element is found and the description is order-sensitive:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
 
-When no element is found:
+When no element is found and the description is not order-sensitive:
 \`\`\`json
 {
   "bbox": [],
+  "isOrderSensitive": false,
   "errors": ["I can see ..., but {some element} is not found"]
 }
 \`\`\`
@@ -289,31 +313,47 @@ You are an expert in software testing.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Give the coordinates of the element that matches the user's description best in the screenshot.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Output Format:
 \`\`\`json
 {
   "bbox": [number, number, number, number],  // 2d bounding box as [xmin, ymin, xmax, ymax]
-  "errors"?: string[]
+  "errors"?: string[],
+  "isOrderSensitive": boolean // Whether the targetElementDescription is order-sensitive (true/false)
 }
 \`\`\`
 
 Fields:
 * \`bbox\` is the bounding box of the element that matches the user's description best in the screenshot
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
 * \`errors\` is an optional array of error messages (if any)
 
-For example, when an element is found:
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
+
+For example, when an element is found and the description is order-sensitive:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
 
-When no element is found:
+When no element is found and the description is not order-sensitive:
 \`\`\`json
 {
   "bbox": [],
+  "isOrderSensitive": false,
   "errors": ["I can see ..., but {some element} is not found"]
 }
 \`\`\`

packages/shared/src/extractor/dom-util.ts

@@ -23,6 +23,12 @@ export function isAElement(
   return node instanceof HTMLElement && node.tagName.toLowerCase() === 'a';
 }
 
+export function isSvgElement(
+  node: globalThis.Node,
+): node is globalThis.SVGSVGElement {
+  return node instanceof SVGElement;
+}
+
 export function isImgElement(
   node: globalThis.Node,
 ): node is globalThis.HTMLImageElement {

packages/shared/src/extractor/index.ts

@@ -39,6 +39,7 @@ export { setNodeHashCacheListOnWindow, getNodeFromCacheList } from './util';
 
 export {
   getXpathsById,
+  getXpathsByPoint,
   getNodeInfoByXpath,
   getElementInfoByXpath,
 } from './locator';