Skip to content

Commit 865e1b0

Browse files
smalininsmalinin
committed
Update model list
1 parent e14813a commit 865e1b0

File tree

1 file changed

+210
-11
lines changed

1 file changed

+210
-11
lines changed

‎src/index.js‎

Lines changed: 210 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,87 @@ function updateEngineInitProgressCallback(report) {
114114
document.getElementById("download-status").textContent = report.text;
115115
}
116116

117+
const appConfig = {model_list:[]};
118+
119+
for(const m of webllm.prebuiltAppConfig.model_list) {
120+
if (m.model_id.startsWith('Qwen2.5-7B')
121+
|| (m.model_id.startsWith('Llama-3.1-8B-') && !m.model_id.endsWith('-1k'))
122+
|| m.model_id.startsWith('Hermes-3-Llama-3.1'))
123+
appConfig.model_list.push(m);
124+
}
125+
126+
appConfig.model_list.push(
127+
{
128+
model: "https://huggingface.co/smalinin/Llama-3.1-Storm-8B_q4f32_1-MLC",
129+
model_id: "Llama-3.1-Storm-8B_q4f32_1-MLC",
130+
model_lib: "https://huggingface.co/smalinin/Llama-3.1-Storm-8B_q4f32_1-MLC/resolve/main/Llama-3.1-Storm-8B_q4f32_1-webgpu.wasm",
131+
low_resource_required: false,
132+
vram_required_MB: 5750,
133+
overrides: {
134+
context_window_size: 4096,
135+
prefill_chunk_size: 4096
136+
},
137+
});
138+
139+
appConfig.model_list.push(
140+
{
141+
model: "https://huggingface.co/smalinin/gorilla-openfunctions-v2_q4f32_1-MLC",
142+
model_id: "gorilla-openfunctions-v2-q4f32_1-MLC",
143+
model_lib: "https://huggingface.co/smalinin/gorilla-openfunctions-v2_q4f32_1-MLC/resolve/main/gorilla-openfunctions-v2_q4f32_1-webgpu.wasm",
144+
vram_required_MB: 5660.67,
145+
low_resource_required: false,
146+
overrides: {
147+
context_window_size: 4096,
148+
prefill_chunk_size: 4096
149+
},
150+
});
151+
152+
appConfig.model_list.push(
153+
{
154+
model: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q3f16_1-MLC",
155+
model_id: "Qwen2.5-14B-Instruct_q3f16_1-MLC",
156+
model_lib: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q3f16_1-MLC/resolve/main/Qwen2.5-14B-Instruct_q3f16_1-webgpu.wasm",
157+
low_resource_required: false,
158+
vram_required_MB: 9300.0,
159+
required_features: ["shader-f16"],
160+
overrides: {
161+
context_window_size: 4096,
162+
prefill_chunk_size: 2048
163+
},
164+
});
165+
166+
appConfig.model_list.push(
167+
{
168+
model: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q4f16_1-MLC",
169+
model_id: "Qwen2.5-14B-Instruct_q4f16_1-MLC",
170+
model_lib: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q4f16_1-MLC/resolve/main/Qwen2.5-14B-Instruct_q4f16_1-webgpu.wasm",
171+
low_resource_required: false,
172+
vram_required_MB: 10900.0,
173+
required_features: ["shader-f16"],
174+
overrides: {
175+
context_window_size: 4096,
176+
prefill_chunk_size: 2048
177+
},
178+
});
179+
180+
181+
appConfig.model_list.push(
182+
{
183+
model: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q4f32_1-MLC",
184+
model_id: "Qwen2.5-14B-Instruct_q4f32_1-MLC",
185+
model_lib: "https://huggingface.co/smalinin/Qwen2.5-14B-Instruct_q4f32_1-MLC/resolve/main/Qwen2.5-14B-Instruct_q4f32_1-webgpu.wasm",
186+
low_resource_required: false,
187+
vram_required_MB: 12000.0,
188+
overrides: {
189+
context_window_size: 4096,
190+
prefill_chunk_size: 4096
191+
},
192+
});
193+
117194

118195
// Create engine instance
119-
const engine = new webllm.MLCEngine();
196+
const engine = new webllm.MLCEngine({appConfig: appConfig});
197+
120198
engine.setInitProgressCallback(updateEngineInitProgressCallback);
121199

122200
async function initializeWebLLMEngine() {
@@ -194,16 +272,8 @@ async function streamingGenerating(messages, onUpdate, onFinish, onError) {
194272
}
195273

196274
/*************** UI logic ***************/
197-
const availableModels = webllm.prebuiltAppConfig.model_list
198-
.map((m) => m.model_id)
199-
.filter((model_id) => (
200-
model_id.startsWith('Qwen2.5-7B')
201-
|| model_id.startsWith('Hermes-3-Llama-3.1')
202-
|| (model_id.startsWith('Llama-3.1-8B-') && !model_id.endsWith('-1k'))
203-
// || model_id.startsWith('DeepSeek-R1-Distill-Llama-')
204-
));
205-
206-
//let selectedModel = "Llama-3.1-8B-Instruct-q4f16_1-1k";
275+
const availableModels = appConfig.model_list.map((m) => m.model_id);
276+
207277
let selectedModel = "Qwen2.5-7B-Instruct-q4f16_1-MLC";
208278

209279
async function onMessageStop() {
@@ -443,6 +513,23 @@ class ToolHanler {
443513
+'Always do real call of functions, when it is required.\n'
444514
+'Execute only one function per time.\n'
445515

516+
llama31_storm_template =
517+
`You are a function calling AI model. You may call one or more functions to assist with the user query.`
518+
+` Don't make assumptions about what values to plug into function. The user may use the terms function`
519+
+` calling or tool use interchangeably.\n\n`
520+
+`Here are the available functions:\n`
521+
+`<tools>#{functions_list}</tools>\n\n`
522+
+`For each function call return a json object with function name and arguments within <tool_call></tool_call>`
523+
+` XML tags in the format:\n`
524+
+`<tool_call>{"tool_name": <function-name>, "tool_arguments": <args-dict>}</tool_call>`
525+
526+
gorilla_template =
527+
`You are an AI programming assistant, utilizing the Gorilla LLM model, developed by Gorilla LLM,`
528+
+` and you only answer questions related to computer science. For politically sensitive questions,`
529+
+` security and privacy issues, and other non-computer science questions, you will refuse to answer.`
530+
+`### Instruction\n`
531+
+`#{functions_list}\n`
532+
446533

447534
deepseek_template =
448535
'Cutting Knowledge Date: December 2023\n'
@@ -475,12 +562,16 @@ class ToolHanler {
475562
this.mode = 'qwen';
476563
else if (model_id.startsWith('Hermes-3-Llama'))
477564
this.mode = 'hermes3_llama'
565+
else if (model_id.startsWith('Llama-3.1-Storm'))
566+
this.mode = 'llama31_storm'
478567
else if (model_id.startsWith('Llama-3.1-'))
479568
this.mode = 'llama31'
480569
else if (model_id.startsWith('Llama-3.2-'))
481570
this.mode = 'llama32'
482571
else if (model_id.startsWith('DeepSeek-R1-Distill-Llama'))
483572
this.mode = 'deepseek'
573+
else if (model_id.startsWith('gorilla'))
574+
this.mode = 'gorilla'
484575
else
485576
this.mode = 'llama31';
486577
this.tool_call_id=0;
@@ -498,10 +589,14 @@ class ToolHanler {
498589
sys_template = this.hermes2_template.replace('#{functions}', funcs);
499590
else if (this.mode==='llama31')
500591
sys_template = this.llama31_template.replace('#{functions}', funcs);
592+
else if (this.mode==='llama31_storm')
593+
sys_template = this.llama31_storm_template.replace('#{functions_list}', JSON.stringify(tools,'\n', 2));
501594
else if (this.mode==='llama32')
502595
sys_template = this.llama32_template.replace('#{functions}', funcs);
503596
else if (this.mode==='deepseek')
504597
sys_template = this.deepseek_template.replace('#{functions}', funcs);
598+
else if (this.mode==='gorilla')
599+
sys_template = this.deepseek_template.replace('#{functions}', funcs);
505600

506601
return sys_template + `\n\n ${JSON.stringify(rules, '\n', 2)}\n`
507602
}
@@ -533,6 +628,15 @@ class ToolHanler {
533628
is_end = true;
534629
}
535630
}
631+
else if (this.mode==='llama31_storm') {
632+
if (str.startsWith("<tool_call>")) {
633+
tool_call = str.replace("<tool_call>", "").replace("</tool_call>", "");
634+
}
635+
else if (tool_end) {
636+
tool_call = tool_end[0].replace("<tool_call>", "").replace("</tool_call>", "");
637+
is_end = true;
638+
}
639+
}
536640
else if (this.mode==='llama31') {
537641
if (str.startsWith("<function>")) {
538642
tool_call = str.replace("<function>", "").replace("</function>", "");
@@ -552,12 +656,37 @@ class ToolHanler {
552656
is_end = true;
553657
}
554658
}
659+
else if (this.mode==='gorilla') {
660+
if (str.startsWith("<<function>>")) {
661+
tool_call = str.replace("<<function>>", "").trim();
662+
}
663+
else if (function_end) {
664+
tool_call = function_end[0].replace("<<function>>", "").trim();
665+
is_end = true;
666+
}
667+
if (tool_call) {
668+
let i = tool_call.indexOf('(')
669+
if (i!=-1) {
670+
const fname = tool_call.substring(0, i);
671+
const body = this.convertToJSON(tool_call.substring (i))
672+
tool_call = `{"name":"${fname}", "arguments":${body}}`
673+
}
674+
}
675+
console.log(tool_call)
676+
}
555677

556678
if (tool_call) {
557679
try {
558680
const func = JSON.parse(tool_call);
681+
682+
if (func.tool_name)
683+
func["name"] = func.tool_name;
684+
if (func.tool_arguments)
685+
func["arguments"] = func.tool_arguments;
686+
559687
if (func.parameters)
560688
func["arguments"] = func.parameters;
689+
561690
return {func, tool_call, is_end};
562691
} catch(e) {
563692
console.log(e);
@@ -586,8 +715,78 @@ class ToolHanler {
586715
this.tool_call_id++;
587716
return rc;
588717
}
718+
719+
720+
convertToJSON(input) {
721+
// Remove the surrounding parentheses
722+
let content = input.slice(1, -1);
723+
724+
// Initialize an empty object to store the parsed data
725+
let result = {};
726+
let key = '';
727+
let value = '';
728+
let inQuotes = false;
729+
let escapeNext = false;
730+
731+
let i = 0;
732+
while (i < content.length) {
733+
const char = content[i];
734+
735+
if (inQuotes) {
736+
if (char === '"' && !escapeNext) {
737+
inQuotes = false;
738+
} else if (char === '\\' && !escapeNext) {
739+
escapeNext = true;
740+
} else {
741+
value += char;
742+
escapeNext = false;
743+
}
744+
} else if (char === '=') {
745+
key = content.slice(0, i).trim();
746+
value = '';
747+
} else if (char === ',') {
748+
value = value.trim();
749+
if (value.startsWith('"') && value.endsWith('"')) {
750+
value = value.slice(1, -1).replace(/\\"/g, '"');
751+
} else if (value.startsWith("'") && value.endsWith("'")) {
752+
value = value.slice(1, -1).replace(/\\"/g, '"');
753+
} else if (value === 'true' || value === 'false') {
754+
value = value === 'true';
755+
} else if (!isNaN(value)) {
756+
value = Number(value);
757+
}
758+
result[key] = value;
759+
key = '';
760+
value = '';
761+
} else if (char === '"') {
762+
inQuotes = true;
763+
} else {
764+
value += char;
765+
}
766+
767+
i++;
768+
}
769+
770+
// Handle the last key-value pair
771+
value = value.trim();
772+
if (value.startsWith('"') && value.endsWith('"')) {
773+
value = value.slice(1, -1).replace(/\\"/g, '"');
774+
} else if (value.startsWith("'") && value.endsWith("'")) {
775+
value = value.slice(1, -1).replace(/\\"/g, '"');
776+
} else if (value === 'true' || value === 'false') {
777+
value = value === 'true';
778+
} else if (!isNaN(value)) {
779+
value = Number(value);
780+
}
781+
result[key] = value;
782+
783+
return JSON.stringify(result);
784+
}
785+
589786
}
590787

788+
789+
591790
/****** TOOLS code **************************/
592791
async function fetch_wikipedia_content(searchQuery)
593792
{

0 commit comments

Comments
 (0)