Skip to content

Commit 885fce5

Browse files
axel7083axel7083
authored
fix: adding chatformat to use for inference servers (#868)
* fix: adding chatformat to use for inference servers Signed-off-by: axel7083 <[email protected]> * test: ensuring chatformat propagate the env Signed-off-by: axel7083 <[email protected]> * refactor: move chat format to properties section Signed-off-by: axel7083 <[email protected]> * fix: using camelCase for json properties Signed-off-by: axel7083 <[email protected]> * fix: json properties Signed-off-by: axel7083 <[email protected]> * fix: adding MODEL_ prefix Signed-off-by: axel7083 <[email protected]> * fix: linter&prettier Signed-off-by: axel7083 <[email protected]> * chore: bump container image version Signed-off-by: axel7083 <[email protected]> * fix: prettier Signed-off-by: axel7083 <[email protected]> --------- Signed-off-by: axel7083 <[email protected]>
1 parent 7b96eb9 commit 885fce5

File tree

5 files changed

+92
-6
lines changed

5 files changed

+92
-6
lines changed

‎packages/backend/src/assets/ai.json

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,10 @@
104104
"registry": "Hugging Face",
105105
"license": "Apache-2.0",
106106
"url": "https://huggingface.co/ibm/merlinite-7b-GGUF/resolve/main/merlinite-7b-Q4_K_M.gguf",
107-
"memory": 4370129224
107+
"memory": 4370129224,
108+
"properties": {
109+
"chatFormat": "openchat"
110+
}
108111
},
109112
{
110113
"id": "hf.TheBloke.mistral-7b-codealpaca-lora.Q4_K_M",
@@ -134,7 +137,10 @@
134137
"registry": "Hugging Face",
135138
"license": "Apache-2.0",
136139
"url": "https://huggingface.co/froggeric/Cerebrum-1.0-7b-GGUF/resolve/main/Cerebrum-1.0-7b-Q4_KS.gguf",
137-
"memory": 4144643441
140+
"memory": 4144643441,
141+
"properties": {
142+
"chatFormat": "openchat"
143+
}
138144
},
139145
{
140146
"id": "hf.TheBloke.openchat-3.5-0106.Q4_K_M",
@@ -174,7 +180,10 @@
174180
"registry": "Hugging Face",
175181
"license": "Apache-2.0",
176182
"url": "https://huggingface.co/llmware/dragon-mistral-7b-v0/resolve/main/dragon-mistral-7b-q4_k_m.gguf",
177-
"memory": 4370129224
183+
"memory": 4370129224,
184+
"properties": {
185+
"chatFormat": "openchat"
186+
}
178187
},
179188
{
180189
"id": "hf.MaziyarPanahi.MixTAO-7Bx2-MoE-Instruct-v7.0.Q4_K_M",

‎packages/backend/src/managers/inference/inferenceManager.spec.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,9 @@ describe('Create Inference Server', () => {
309309
);
310310
expect(taskRegistryMock.createTask).toHaveBeenNthCalledWith(
311311
1,
312-
'Pulling ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat:0.2.0.',
312+
expect.stringContaining(
313+
'Pulling ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat:',
314+
),
313315
'loading',
314316
{
315317
trackingId: 'dummyTrackingId',

‎packages/backend/src/utils/inferenceUtils.spec.ts

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,68 @@ describe('generateContainerCreateOptions', () => {
9696
},
9797
});
9898
});
99+
100+
test('model info with chat_format properties', () => {
101+
const result = generateContainerCreateOptions(
102+
{
103+
port: 8888,
104+
providerId: 'test@providerId',
105+
image: INFERENCE_SERVER_IMAGE,
106+
modelsInfo: [
107+
{
108+
id: 'dummyModelId',
109+
file: {
110+
file: 'dummyFile',
111+
path: 'dummyPath',
112+
},
113+
properties: {
114+
chatFormat: 'dummyChatFormat',
115+
},
116+
},
117+
],
118+
} as unknown as InferenceServerConfig,
119+
{
120+
Id: 'dummyImageId',
121+
engineId: 'dummyEngineId',
122+
RepoTags: [INFERENCE_SERVER_IMAGE],
123+
} as unknown as ImageInfo,
124+
);
125+
126+
expect(result.Env).toContain('MODEL_CHAT_FORMAT=dummyChatFormat');
127+
});
128+
129+
test('model info with multiple properties', () => {
130+
const result = generateContainerCreateOptions(
131+
{
132+
port: 8888,
133+
providerId: 'test@providerId',
134+
image: INFERENCE_SERVER_IMAGE,
135+
modelsInfo: [
136+
{
137+
id: 'dummyModelId',
138+
file: {
139+
file: 'dummyFile',
140+
path: 'dummyPath',
141+
},
142+
properties: {
143+
basicProp: 'basicProp',
144+
lotOfCamelCases: 'lotOfCamelCases',
145+
lowercase: 'lowercase',
146+
},
147+
},
148+
],
149+
} as unknown as InferenceServerConfig,
150+
{
151+
Id: 'dummyImageId',
152+
engineId: 'dummyEngineId',
153+
RepoTags: [INFERENCE_SERVER_IMAGE],
154+
} as unknown as ImageInfo,
155+
);
156+
157+
expect(result.Env).toContain('MODEL_BASIC_PROP=basicProp');
158+
expect(result.Env).toContain('MODEL_LOT_OF_CAMEL_CASES=lotOfCamelCases');
159+
expect(result.Env).toContain('MODEL_LOWERCASE=lowercase');
160+
});
99161
});
100162

101163
describe('withDefaultConfiguration', () => {

‎packages/backend/src/utils/inferenceUtils.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ export const SECOND: number = 1_000_000_000;
3434
export const LABEL_INFERENCE_SERVER: string = 'ai-lab-inference-server';
3535

3636
export const INFERENCE_SERVER_IMAGE =
37-
'ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat:0.2.0';
37+
'ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat:0.3.0';
3838

3939
/**
4040
* Return container connection provider
@@ -115,6 +115,16 @@ export function generateContainerCreateOptions(
115115
throw new Error('The model info file provided is undefined');
116116
}
117117

118+
const envs: string[] = [`MODEL_PATH=/models/${modelInfo.file.file}`, 'HOST=0.0.0.0', 'PORT=8000'];
119+
if (modelInfo.properties) {
120+
envs.push(
121+
...Object.entries(modelInfo.properties).map(([key, value]) => {
122+
const formattedKey = key.replace(/[A-Z]/g, m => `_${m}`).toUpperCase();
123+
return `MODEL_${formattedKey}=${value}`;
124+
}),
125+
);
126+
}
127+
118128
return {
119129
Image: imageInfo.Id,
120130
Detach: true,
@@ -147,7 +157,7 @@ export function generateContainerCreateOptions(
147157
...config.labels,
148158
[LABEL_INFERENCE_SERVER]: JSON.stringify(config.modelsInfo.map(model => model.id)),
149159
},
150-
Env: [`MODEL_PATH=/models/${modelInfo.file.file}`, 'HOST=0.0.0.0', 'PORT=8000'],
160+
Env: envs,
151161
Cmd: ['--models-path', '/models', '--context-size', '700', '--threads', '4'],
152162
};
153163
}

‎packages/shared/src/models/IModelInfo.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ export interface ModelInfo {
2929
file?: LocalModelInfo;
3030
state?: 'deleting';
3131
memory?: number;
32+
properties?: {
33+
[key: string]: string;
34+
};
3235
}
3336

3437
export type ModelCheckerContext = 'inference' | 'recipe';

0 commit comments

Comments
 (0)