test2

Author: winskuo-quic

.ci/scripts/utils.sh

@@ -146,10 +146,14 @@ download_stories_model_artifacts() {
   # Download stories110M.pt and tokenizer from Github
   curl -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
   curl -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
+
+  # Create tokenizer.bin.
+  echo "Creating tokenizer.bin"
+  $PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+
   # Create params.json file
   touch params.json
   echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
-  python -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
 }
 
 do_not_use_nightly_on_ci() {