diff --git a/packages/core/src/ai-model/prompt/describe.ts b/packages/core/src/ai-model/prompt/describe.ts index 4e0c2fe971..7ea8ff57fa 100644 --- a/packages/core/src/ai-model/prompt/describe.ts +++ b/packages/core/src/ai-model/prompt/describe.ts @@ -1,8 +1,34 @@ import { getPreferredLanguage } from '@midscene/shared/env'; +const examplesMap: Record = { + Chinese: [ + '"登录表单中的"登录"按钮"', + '"搜索输入框,placeholder 为"请输入关键词""', + '"顶部导航栏中文字为"首页"的链接"', + '"联系表单中的提交按钮"', + '"aria-label 为"打开菜单"的菜单图标"', + ], + English: [ + '"Login button with text \'Sign In\'"', + '"Search input with placeholder \'Enter keywords\'"', + '"Navigation link with text \'Home\' in header"', + '"Submit button in contact form"', + '"Menu icon with aria-label \'Open menu\'"', + ], +}; + +const getExamples = (language: string) => { + const examples = examplesMap[language] || examplesMap.English; + return examples.map((e) => `- ${e}`).join('\n'); +}; + export const elementDescriberInstruction = () => { + const preferredLanguage = getPreferredLanguage(); + return ` -Describe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}. +Describe the element in the red rectangle for precise identification. + +IMPORTANT: You MUST write the description in ${preferredLanguage}. CRITICAL REQUIREMENTS: 1. UNIQUENESS: The description must uniquely identify this element on the current page @@ -28,13 +54,10 @@ GUIDELINES: - Avoid page-specific or temporary content - Don't mention the red rectangle or selection box - Focus on stable, reusable characteristics +- **Write the description in ${preferredLanguage}** EXAMPLES: -- "Login button with text 'Sign In'" -- "Search input with placeholder 'Enter keywords'" -- "Navigation link with text 'Home' in header" -- "Submit button in contact form" -- "Menu icon with aria-label 'Open menu'" +${getExamples(preferredLanguage)} Return JSON: { diff --git a/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap b/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap index 5e3b42111e..86664eb012 100644 --- a/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap +++ b/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap @@ -2,7 +2,9 @@ exports[`elementDescriberInstruction > should return the correct instruction 1`] = ` " -Describe the element in the red rectangle for precise identification. Use English. +Describe the element in the red rectangle for precise identification. + +IMPORTANT: You MUST write the description in English. CRITICAL REQUIREMENTS: 1. UNIQUENESS: The description must uniquely identify this element on the current page @@ -28,6 +30,7 @@ GUIDELINES: - Avoid page-specific or temporary content - Don't mention the red rectangle or selection box - Focus on stable, reusable characteristics +- **Write the description in English** EXAMPLES: - "Login button with text 'Sign In'"