Support RedPajama-Chat-3B-V1 (#99)

Author: jinhongyii

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "3rdparty/sentencepiece-js"]
-	path = 3rdparty/sentencepiece-js
-	url = https://github.com/tqchen/sentencepiece-js
 [submodule "mlc-llm"]
 	path = mlc-llm
 	url = https://github.com/mlc-ai/mlc-llm
+[submodule "3rdparty/tokenizers-cpp"]
+	path = 3rdparty/tokenizers-cpp
+	url = https://github.com/mlc-ai/tokenizers-cpp

3rdparty/sentencepiece-js

@@ -23,7 +23,7 @@ cp web/llm_chat.css site/dist/
 
 cp dist/tvmjs_runtime.wasi.js site/dist
 cp dist/tvmjs.bundle.js site/dist
-cp -r dist/sentencepiece site/dist
+cp -r dist/tokenizers-cpp site/dist
 
 if [ -d "$MLC_LLM_HOME/dist/vicuna-v1-7b-q4f32_0/params" ]; then
     mkdir -p site/dist/vicuna-v1-7b-q4f32_0

3rdparty/tokenizers-cpp

@@ -15,15 +15,15 @@ if [[ -z ${TVM_HOME_SET} ]]; then
     export TVM_HOME="${TVM_HOME:-3rdparty/tvm}"
 fi
 
-export SENTENCEPIECE_JS_HOME="3rdparty/sentencepiece-js"
+export TOKENIZERS_CPP_HOME="3rdparty/tokenizers-cpp/web"
 
 mkdir -p dist
 cd ${TVM_HOME}/web && make && npm install && npm run bundle && cd -
 git submodule update --init --recursive
-cd ${SENTENCEPIECE_JS_HOME} && npm install && npm run build && cd -
+cd ${TOKENIZERS_CPP_HOME} && npm install && npm run build && cd -
 git submodule update --init --recursive
-rm -rf dist/sentencepiece
-cp -r ${SENTENCEPIECE_JS_HOME}/dist dist/sentencepiece
+rm -rf dist/tokenizers-cpp
+cp -r ${TOKENIZERS_CPP_HOME}/dist dist/tokenizers-cpp
 
 echo "Exporting tvmjs runtime dist files"
 python -c "from tvm.contrib import tvmjs; tvmjs.export_runtime(\"dist\")"

mlc-llm

@@ -3,31 +3,39 @@
 </script>
 
 <script type="module">
-  async function getTokenizer(url) {
-    const mod = await import("./dist/sentencepiece/index.js");
-    return await mod.sentencePieceProcessor(url);
+  import { Tokenizer } from './dist/tokenizers-cpp/index.js';
+
+  async function tokenizerFromJSON(jsonArrayBuffer) {
+    return await Tokenizer.fromJSON(jsonArrayBuffer);
+  }
+  async function tokenizerFromSentencePiece(modelBuffer) {
+    return await Tokenizer.fromSentencePiece(modelBuffer);
   }
-  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
+  tvmjsGlobalEnv.tokenizerFromJSON = tokenizerFromJSON;
+  tvmjsGlobalEnv.tokenizerFromSentencePiece = tokenizerFromSentencePiece;
+
 </script>
 <script>
   function handleChatUIInputEnter(event) {
     if (event.keyCode === 13) {
       tvmjsGlobalEnv.asyncOnGenerate();
     }
   }
-  async function getTokenizer(url) {
-    const mod = await import("./dist/sentencepiece/index.js");
-    return await mod.sentencePieceProcessor(url);
-  }
-  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
 </script>
-
+<!-- 
 <form>
   <select id="model-name">
     <option selected="selected">vicuna-v1-7b-q4f32_0</option>
-    <!-- <option >dolly-v1-3b-q4f32_0</option> -->
+    <option >RedPajama-INCITE-Chat-3B-v1-q4f32_0</option>
   </select>
-</form>
+</form> -->
+
+<label>Pick a pre-compiled model or load your own model's mlc-chat-config.json:
+  <input list="model-names" name="model" id="model"/></label>
+<datalist id="model-names">
+  <option value="vicuna-v1-7b-q4f32_0" ></option>
+  <option value="RedPajama-INCITE-Chat-3B-v1-q4f32_0"></option>
+</datalist>
 
 <script src="dist/llm_chat.js"></script>
 <link href="dist/llm_chat.css" rel="stylesheet" type="text/css"/>

scripts/build_site.sh

@@ -1,8 +1,10 @@
 {
     "url_dict":{
-        "vicuna-v1-7b-q4f32_0": "https://huggingface.co/hongyij/web-llm-test-model/resolve/main/model_config.json"
+        "vicuna-v1-7b-q4f32_0": "https://huggingface.co/hongyij/mlc-llm-vicuna-7b-v1/resolve/new_config/model_config.json",
+        "RedPajama-INCITE-Chat-3B-v1-q4f32_0": "https://huggingface.co/hongyij/mlc-llm-redpajama/resolve/main/mlc-llm-config.json"
     },
     "model_lib_map":{
-        "vicuna-v1-7b-q4f32_0": "https://raw.githubusercontent.com/jinhongyii/llm-lib-debug/main/vicuna-v1-7b-q4f32_0-webgpu.wasm"
+        "vicuna-v1-7b-q4f32_0": "https://raw.githubusercontent.com/jinhongyii/llm-lib-debug/new_config/vicuna-v1-7b-q4f32_0-webgpu.wasm",
+        "RedPajama-INCITE-Chat-3B-v1-q4f32_0": "https://raw.githubusercontent.com/jinhongyii/llm-lib-debug/new_config/RedPajama-INCITE-Chat-3B-v1-q4f32_0-webgpu.wasm"
     }
 }

scripts/prep_deps.sh

@@ -3,31 +3,39 @@
 </script>
 
 <script type="module">
-  async function getTokenizer(url) {
-    const mod = await import("./dist/sentencepiece/index.js");
-    return await mod.sentencePieceProcessor(url);
+  import { Tokenizer } from './dist/tokenizers-cpp/index.js';
+
+  async function tokenizerFromJSON(jsonArrayBuffer) {
+    return await Tokenizer.fromJSON(jsonArrayBuffer);
+  }
+  async function tokenizerFromSentencePiece(modelBuffer) {
+    return await Tokenizer.fromSentencePiece(modelBuffer);
   }
-  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
+  tvmjsGlobalEnv.tokenizerFromJSON = tokenizerFromJSON;
+  tvmjsGlobalEnv.tokenizerFromSentencePiece = tokenizerFromSentencePiece;
+
 </script>
 <script>
   function handleChatUIInputEnter(event) {
     if (event.keyCode === 13) {
       tvmjsGlobalEnv.asyncOnGenerate();
     }
   }
-  async function getTokenizer(url) {
-    const mod = await import("./dist/sentencepiece/index.js");
-    return await mod.sentencePieceProcessor(url);
-  }
-  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
 </script>
-
+<!-- 
 <form>
   <select id="model-name">
     <option selected="selected">vicuna-v1-7b-q4f32_0</option>
-    <!-- <option >dolly-v1-3b-q4f32_0</option> -->
+    <option >RedPajama-INCITE-Chat-3B-v1-q4f32_0</option>
   </select>
-</form>
+</form> -->
+
+<label>Pick a pre-compiled model or load your own model's mlc-chat-config.json:
+  <input list="model-names" name="model" id="model"/></label>
+<datalist id="model-names">
+  <option value="vicuna-v1-7b-q4f32_0" ></option>
+  <option value="RedPajama-INCITE-Chat-3B-v1-q4f32_0"></option>
+</datalist>
 
 <script src="dist/llm_chat.js"></script>
 <link href="dist/llm_chat.css" rel="stylesheet" type="text/css"/>