File tree Expand file tree Collapse file tree 1 file changed +9
-8
lines changed Expand file tree Collapse file tree 1 file changed +9
-8
lines changed Original file line number Diff line number Diff line change @@ -200,14 +200,15 @@ jobs:
200200 TOKENIZER_BIN_FILE=tokenizer.bin
201201 # Fetch the file using a Python one-liner
202202 DOWNLOADED_TOKENIZER_FILE_PATH=$(python -c "
203- from huggingface_hub import hf_hub_download
204- # Download the file from the Hugging Face Hub
205- downloaded_path = hf_hub_download(
206- repo_id='${{ matrix.model }}',
207- filename='${TOKENIZER_FILE}'
208- )
209- print(downloaded_path)
210- " )
203+ from huggingface_hub import hf_hub_download
204+ # Download the file from the Hugging Face Hub
205+ downloaded_path = hf_hub_download(
206+ repo_id='${{ matrix.model }}',
207+ filename='${TOKENIZER_FILE}'
208+ )
209+ print(downloaded_path)
210+ ")
211+
211212 if [ -f "$DOWNLOADED_TOKENIZER_FILE_PATH" ]; then
212213 echo "${TOKENIZER_FILE} downloaded successfully at: $DOWNLOADED_TOKENIZER_FILE_PATH"
213214 python -m extension.llm.tokenizer.tokenizer -t $DOWNLOADED_TOKENIZER_FILE_PATH -o ./${TOKENIZER_BIN_FILE}
You can’t perform that action at this time.
0 commit comments