feat(core): support config VQA related model

Author: EAGzzyCSL

packages/core/src/ai-model/common.ts

@@ -2,6 +2,7 @@ import type {
   AIUsageInfo,
   BaseElement,
   ElementTreeNode,
+  IModelPreferences,
   MidsceneYamlFlowItem,
   PlanningAction,
   PlanningActionParamInputOrKeyPress,
@@ -45,10 +46,12 @@ export enum AIActionType {
 export async function callAiFn<T>(
   msgs: AIArgs,
   AIActionTypeValue: AIActionType,
+  modelPreferences?: IModelPreferences,
 ): Promise<{ content: T; usage?: AIUsageInfo }> {
   const { content, usage } = await callToGetJSONObject<T>(
     msgs,
     AIActionTypeValue,
+    modelPreferences,
   );
   return { content, usage };
 }

packages/core/src/ai-model/inspect.ts

@@ -7,6 +7,7 @@ import type {
   AIUsageInfo,
   BaseElement,
   ElementById,
+  IModelPreferences,
   InsightExtractOption,
   Rect,
   ReferenceImage,
@@ -385,8 +386,15 @@ export async function AiExtractElementInfo<
   multimodalPrompt?: TMultimodalPrompt;
   context: UIContext<ElementType>;
   extractOption?: InsightExtractOption;
+  modelPreferences?: IModelPreferences;
 }) {
-  const { dataQuery, context, extractOption, multimodalPrompt } = options;
+  const {
+    dataQuery,
+    context,
+    extractOption,
+    multimodalPrompt,
+    modelPreferences,
+  } = options;
   const systemPrompt = systemPromptToExtract();
 
   const { screenshotBase64 } = context;
@@ -445,6 +453,7 @@ export async function AiExtractElementInfo<
   const result = await callAiFn<AIDataExtractionResponse<T>>(
     msgs,
     AIActionType.EXTRACT_DATA,
+    modelPreferences,
   );
   return {
     parseResult: result.content,

packages/core/src/ai-model/prompt/playwright-generator.ts

@@ -206,7 +206,7 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
 
   if (options.stream && options.onChunk) {
     // Use streaming
-    return await callAi(prompt, AIActionType.EXTRACT_DATA, undefined, {
+    return await callAi(prompt, AIActionType.EXTRACT_DATA, {
       stream: true,
       onChunk: options.onChunk,
     });

packages/core/src/ai-model/prompt/yaml-generator.ts

@@ -425,7 +425,7 @@ Respond with YAML only, no explanations.`,
 
     if (options.stream && options.onChunk) {
       // Use streaming
-      return await callAi(prompt, AIActionType.EXTRACT_DATA, undefined, {
+      return await callAi(prompt, AIActionType.EXTRACT_DATA, {
         stream: true,
         onChunk: options.onChunk,
       });