Skip to content

Commit 6f20686

Browse files
EAGzzyCSLEAGzzyCSL
committed
feat(core): enable specific modelConfig when create agent
1 parent 55c9f70 commit 6f20686

File tree

5 files changed

+286
-82
lines changed

5 files changed

+286
-82
lines changed

‎packages/core/src/ai-model/service-caller/utils.ts

Lines changed: 155 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { IModelPreferences } from '@/types';
1+
import type { IModelConfigForVQA, IModelPreferences } from '@/types';
22
import {
33
ANTHROPIC_API_KEY,
44
AZURE_OPENAI_API_VERSION,
@@ -114,40 +114,52 @@ interface IModelConfigForCreateLLMClient {
114114
}
115115

116116
const createAssert =
117-
(modelNameKey: string, modelName: string) =>
117+
(
118+
modelNameKey: string,
119+
modelName: string,
120+
provider: 'process.env' | 'modelConfig',
121+
) =>
118122
(value: string | undefined, key: string, modelVendorFlag?: string) => {
119123
if (modelVendorFlag) {
120124
assert(
121125
value,
122-
`The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} and ${modelVendorFlag} has also been specified, but got: ${value}\nPlease check your config.`,
126+
`The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} and ${modelVendorFlag} has also been specified in ${provider}, but got: ${value}\nPlease check your config.`,
123127
);
124128
} else {
125129
assert(
126130
value,
127-
`The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName}, but got: ${value}\nPlease check your config.`,
131+
`The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} in ${provider}, but got: ${value}\nPlease check your config.`,
128132
);
129133
}
130134
};
131135

132-
const getModelConfigFromEnv = (
133-
modelName: string,
136+
const getModelConfigFromProvider = ({
137+
modelName,
138+
keys,
139+
valueAssert,
140+
getStringConfig,
141+
getJsonConfig,
142+
}: {
143+
modelName: string;
134144
keys: Record<
135145
Exclude<keyof IModelConfigForCreateLLMClient, 'modelName'>,
136146
Parameters<typeof getAIConfig>[0]
137-
>,
147+
>;
138148
valueAssert: (
139149
value: string | undefined,
140150
key: string,
141151
modelVendorFlag?: string,
142-
) => void,
143-
): IModelConfigForCreateLLMClient => {
144-
const socksProxy = getAIConfig(keys.socksProxy);
145-
const httpProxy = getAIConfig(keys.httpProxy);
146-
147-
if (getAIConfig(keys.openaiUseAzureDeprecated)) {
148-
const openaiBaseURL = getAIConfig(keys.openaiBaseURL);
149-
const openaiApiKey = getAIConfig(keys.openaiApiKey);
150-
const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
152+
) => void;
153+
getStringConfig: (key?: string) => string | undefined;
154+
getJsonConfig: (key?: string) => Record<string, unknown> | undefined;
155+
}): IModelConfigForCreateLLMClient => {
156+
const socksProxy = getStringConfig(keys.socksProxy);
157+
const httpProxy = getStringConfig(keys.httpProxy);
158+
159+
if (getStringConfig(keys.openaiUseAzureDeprecated)) {
160+
const openaiBaseURL = getStringConfig(keys.openaiBaseURL);
161+
const openaiApiKey = getStringConfig(keys.openaiApiKey);
162+
const openaiExtraConfig = getJsonConfig(keys.openaiExtraConfig);
151163

152164
valueAssert(
153165
openaiBaseURL,
@@ -165,16 +177,16 @@ const getModelConfigFromEnv = (
165177
openaiBaseURL,
166178
openaiExtraConfig,
167179
};
168-
} else if (getAIConfig(keys.useAzureOpenai)) {
169-
const azureOpenaiScope = getAIConfig(keys.azureOpenaiScope);
180+
} else if (getStringConfig(keys.useAzureOpenai)) {
181+
const azureOpenaiScope = getStringConfig(keys.azureOpenaiScope);
170182

171-
const azureOpenaiApiKey = getAIConfig(keys.azureOpenaiApiKey);
172-
const azureOpenaiEndpoint = getAIConfig(keys.azureOpenaiEndpoint);
173-
const azureOpenaiDeployment = getAIConfig(keys.azureOpenaiDeployment);
174-
const azureOpenaiApiVersion = getAIConfig(keys.azureOpenaiApiVersion);
183+
const azureOpenaiApiKey = getStringConfig(keys.azureOpenaiApiKey);
184+
const azureOpenaiEndpoint = getStringConfig(keys.azureOpenaiEndpoint);
185+
const azureOpenaiDeployment = getStringConfig(keys.azureOpenaiDeployment);
186+
const azureOpenaiApiVersion = getStringConfig(keys.azureOpenaiApiVersion);
175187

176-
const azureExtraConfig = getAIConfigInJson(keys.azureExtraConfig);
177-
const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
188+
const azureExtraConfig = getJsonConfig(keys.azureExtraConfig);
189+
const openaiExtraConfig = getJsonConfig(keys.openaiExtraConfig);
178190

179191
valueAssert(azureOpenaiApiKey, keys.azureOpenaiApiKey, keys.useAzureOpenai);
180192

@@ -191,8 +203,8 @@ const getModelConfigFromEnv = (
191203
azureExtraConfig,
192204
openaiExtraConfig,
193205
};
194-
} else if (getAIConfig(keys.useAnthropicSdk)) {
195-
const anthropicApiKey = getAIConfig(keys.anthropicApiKey);
206+
} else if (getStringConfig(keys.useAnthropicSdk)) {
207+
const anthropicApiKey = getStringConfig(keys.anthropicApiKey);
196208
valueAssert(anthropicApiKey, keys.anthropicApiKey, keys.useAnthropicSdk);
197209

198210
return {
@@ -203,9 +215,9 @@ const getModelConfigFromEnv = (
203215
anthropicApiKey,
204216
};
205217
} else {
206-
const openaiBaseURL = getAIConfig(keys.openaiBaseURL);
207-
const openaiApiKey = getAIConfig(keys.openaiApiKey);
208-
const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
218+
const openaiBaseURL = getStringConfig(keys.openaiBaseURL);
219+
const openaiApiKey = getStringConfig(keys.openaiApiKey);
220+
const openaiExtraConfig = getJsonConfig(keys.openaiExtraConfig);
209221

210222
valueAssert(openaiBaseURL, keys.openaiBaseURL);
211223
valueAssert(openaiApiKey, keys.openaiApiKey);
@@ -253,6 +265,36 @@ const maskConfig = (config: IModelConfigForCreateLLMClient) => {
253265
);
254266
};
255267

268+
const vqaModelConfigKeys = {
269+
/**
270+
* proxy
271+
*/
272+
socksProxy: MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
273+
httpProxy: MIDSCENE_VQA_OPENAI_HTTP_PROXY,
274+
/**
275+
* OpenAI
276+
*/
277+
openaiBaseURL: MIDSCENE_VQA_OPENAI_BASE_URL,
278+
openaiApiKey: MIDSCENE_VQA_OPENAI_API_KEY,
279+
openaiExtraConfig: MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
280+
/**
281+
* Azure
282+
*/
283+
openaiUseAzureDeprecated: MIDSCENE_VQA_OPENAI_USE_AZURE,
284+
useAzureOpenai: MIDSCENE_VQA_USE_AZURE_OPENAI,
285+
azureOpenaiScope: MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
286+
azureOpenaiApiKey: MIDSCENE_VQA_AZURE_OPENAI_KEY,
287+
azureOpenaiEndpoint: MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
288+
azureOpenaiApiVersion: MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
289+
azureOpenaiDeployment: MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
290+
azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
291+
/**
292+
* Anthropic
293+
*/
294+
useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
295+
anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY,
296+
} as const;
297+
256298
/**
257299
* get and validate model config for model client
258300
*/
@@ -267,59 +309,91 @@ export const decideModelConfig = (
267309

268310
const isVQAIntent = modelPreferences?.intent === 'VQA';
269311

312+
const vqaModelCallback = modelPreferences?.modelConfigByIntent?.VQA;
270313
const vqaModelName = getAIConfig(MIDSCENE_VQA_MODEL_NAME);
271314

272-
if (isVQAIntent && vqaModelName) {
273-
debugLog(
274-
`current action is a VQA action and detected ${MIDSCENE_VQA_MODEL_NAME} ${vqaModelName}, will only read VQA related model config from process.env`,
275-
);
276-
const config = getModelConfigFromEnv(
277-
vqaModelName,
278-
{
279-
/**
280-
* proxy
281-
*/
282-
socksProxy: MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
283-
httpProxy: MIDSCENE_VQA_OPENAI_HTTP_PROXY,
284-
/**
285-
* OpenAI
286-
*/
287-
openaiBaseURL: MIDSCENE_VQA_OPENAI_BASE_URL,
288-
openaiApiKey: MIDSCENE_VQA_OPENAI_API_KEY,
289-
openaiExtraConfig: MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
290-
/**
291-
* Azure
292-
*/
293-
openaiUseAzureDeprecated: MIDSCENE_VQA_OPENAI_USE_AZURE,
294-
useAzureOpenai: MIDSCENE_VQA_USE_AZURE_OPENAI,
295-
azureOpenaiScope: MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
296-
azureOpenaiApiKey: MIDSCENE_VQA_AZURE_OPENAI_KEY,
297-
azureOpenaiEndpoint: MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
298-
azureOpenaiApiVersion: MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
299-
azureOpenaiDeployment: MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
300-
azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
301-
/**
302-
* Anthropic
303-
*/
304-
useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
305-
anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY,
306-
},
307-
createAssert(MIDSCENE_VQA_MODEL_NAME, vqaModelName),
308-
);
315+
const vqaModelConfig = vqaModelCallback?.();
309316

310-
debugLog('got model config for VQA usage:', maskConfig(config));
317+
if (isVQAIntent && (vqaModelConfig || vqaModelName)) {
318+
if (vqaModelConfig) {
319+
debugLog(
320+
'current action is a VQA action and detected VQA declared in modelConfig, will only read VQA related model config from modelConfig.VQA',
321+
);
322+
const modelName = vqaModelConfig[MIDSCENE_VQA_MODEL_NAME];
323+
assert(
324+
modelName,
325+
'The return value of modelConfig.VQA() does not have a valid MIDSCENE_VQA_MODEL_NAME filed.',
326+
);
327+
const config = getModelConfigFromProvider({
328+
modelName,
329+
keys: vqaModelConfigKeys,
330+
valueAssert: createAssert(
331+
MIDSCENE_VQA_MODEL_NAME,
332+
modelName,
333+
'modelConfig',
334+
),
335+
getStringConfig: (key) =>
336+
key ? vqaModelConfig[key as keyof IModelConfigForVQA] : undefined,
337+
getJsonConfig: (key) => {
338+
if (key) {
339+
const content = vqaModelConfig[key as keyof IModelConfigForVQA];
340+
if (content) {
341+
try {
342+
return JSON.parse(content);
343+
} catch (e) {
344+
throw new Error(
345+
`Failed to parse json config: ${key}. ${(e as Error).message}`,
346+
{
347+
cause: e,
348+
},
349+
);
350+
}
351+
}
352+
}
353+
return undefined;
354+
},
355+
});
356+
debugLog(
357+
'got model config for VQA usage from modelConfig.VQA:',
358+
maskConfig(config),
359+
);
311360

312-
return config;
361+
return config;
362+
} else {
363+
debugLog(
364+
`current action is a VQA action and detected ${MIDSCENE_VQA_MODEL_NAME} ${vqaModelName} in process.env, will only read VQA related model config from process.env`,
365+
);
366+
const config = getModelConfigFromProvider({
367+
modelName: vqaModelName!,
368+
keys: vqaModelConfigKeys,
369+
valueAssert: createAssert(
370+
MIDSCENE_VQA_MODEL_NAME,
371+
vqaModelName!,
372+
'process.env',
373+
),
374+
getStringConfig: getAIConfig as (key?: string) => string | undefined,
375+
getJsonConfig: getAIConfigInJson as (
376+
key?: string,
377+
) => Record<string, unknown> | undefined,
378+
});
379+
380+
debugLog(
381+
'got model config for VQA usage from process.env:',
382+
maskConfig(config),
383+
);
384+
385+
return config;
386+
}
313387
} else {
314388
debugLog('read model config from process.env as normal.');
315389
const commonModelName = getAIConfig(MIDSCENE_MODEL_NAME);
316390
assert(
317391
commonModelName,
318392
`${MIDSCENE_MODEL_NAME} is empty, please check your config.`,
319393
);
320-
const config = getModelConfigFromEnv(
321-
commonModelName,
322-
{
394+
const config = getModelConfigFromProvider({
395+
modelName: commonModelName,
396+
keys: {
323397
/**
324398
* proxy
325399
*/
@@ -348,8 +422,16 @@ export const decideModelConfig = (
348422
useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
349423
anthropicApiKey: ANTHROPIC_API_KEY,
350424
},
351-
createAssert(MIDSCENE_MODEL_NAME, commonModelName),
352-
);
425+
valueAssert: createAssert(
426+
MIDSCENE_MODEL_NAME,
427+
commonModelName,
428+
'process.env',
429+
),
430+
getStringConfig: getAIConfig as (key?: string) => string | undefined,
431+
getJsonConfig: getAIConfigInJson as (
432+
key?: string,
433+
) => Record<string, unknown> | undefined,
434+
});
353435

354436
debugLog('got model config for common usage:', maskConfig(config));
355437

‎packages/core/src/types.ts

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -623,10 +623,39 @@ export type TUserPrompt =
623623
prompt: string;
624624
} & Partial<TMultimodalPrompt>);
625625

626+
export interface IModelConfigForVQA {
627+
// model name
628+
MIDSCENE_VQA_MODEL_NAME: string;
629+
// proxy
630+
MIDSCENE_VQA_OPENAI_SOCKS_PROXY?: string;
631+
MIDSCENE_VQA_OPENAI_HTTP_PROXY?: string;
632+
// OpenAI
633+
MIDSCENE_VQA_OPENAI_BASE_URL?: string;
634+
MIDSCENE_VQA_OPENAI_API_KEY?: string;
635+
MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON?: string;
636+
// Azure
637+
MIDSCENE_VQA_OPENAI_USE_AZURE?: string;
638+
MIDSCENE_VQA_USE_AZURE_OPENAI?: string;
639+
MIDSCENE_VQA_AZURE_OPENAI_SCOPE?: string;
640+
MIDSCENE_VQA_AZURE_OPENAI_KEY?: string;
641+
MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT?: string;
642+
MIDSCENE_VQA_AZURE_OPENAI_API_VERSION?: string;
643+
MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT?: string;
644+
MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON?: string;
645+
// Anthropic
646+
MIDSCENE_VQA_USE_ANTHROPIC_SDK?: string;
647+
MIDSCENE_VQA_ANTHROPIC_API_KEY?: string;
648+
}
649+
650+
export interface IModelConfigByIntent {
651+
VQA?: () => IModelConfigForVQA;
652+
}
653+
626654
export interface IModelPreferences {
627655
/**
628656
* - VQA: Visual Question Answering
629657
* - grounding：short for Visual Grounding
630658
*/
631-
intent: 'VQA' | 'planning' | 'grounding';
659+
intent?: 'VQA' | 'planning' | 'grounding';
660+
modelConfigByIntent?: IModelConfigByIntent;
632661
}

0 commit comments

Comments
 (0)