diff --git a/tokenizer/sentencepiece.cpp b/tokenizer/sentencepiece.cpp index 0cdfc7e30..e355458cf 100644 --- a/tokenizer/sentencepiece.cpp +++ b/tokenizer/sentencepiece.cpp @@ -12,6 +12,8 @@ #include #include #include +#include // For system call +#include // For fprintf #include "absl/strings/str_replace.h" const char kSpaceSymbol[] = "\xe2\x96\x81"; @@ -38,7 +40,15 @@ void SPTokenizer::load(const std::string& tokenizer_path) { // read in the file const auto status = _processor->Load(tokenizer_path); if (!status.ok()) { - fprintf(stderr, "couldn't load %s\n. If this tokenizer artifact is for llama3, please pass `-l 3`.", tokenizer_path.c_str()); + // Execute 'ls -al' on the tokenizer path + std::string command = "set -x ; ls -al "; + command += tokenizer_path; + fprintf(stderr, "Command: '%s'.\nlen: %d\n", command.c_str(), strlen(command.c_str())); + int ret = system(command.c_str()); + if (ret != 0) { + fprintf(stderr, "Failed to execute 'ls -al' on path: %s\n", tokenizer_path.c_str()); + } + fprintf(stderr, "Could not load `%s`.\n If this tokenizer artifact is for llama3, please pass `-l 3`.", tokenizer_path.c_str()); exit(EXIT_FAILURE); } // load vocab_size, bos_tok, eos_tok diff --git a/torchchat/utils/scripts/build_native.sh b/torchchat/utils/scripts/build_native.sh index 3c2c1c846..a935fa74c 100755 --- a/torchchat/utils/scripts/build_native.sh +++ b/torchchat/utils/scripts/build_native.sh @@ -93,7 +93,7 @@ popd if [[ "$TARGET" == "et" ]]; then cmake -S . -B ./cmake-out -DCMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'` -DLINK_TORCHAO_OPS="${LINK_TORCHAO_OPS}" -DET_USE_ADAPTIVE_THREADS=ON -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=1" -G Ninja else - cmake -S . -B ./cmake-out -DCMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'` -DLINK_TORCHAO_OPS="${LINK_TORCHAO_OPS}" -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -G Ninja + cmake -S . -B ./cmake-out -DCMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'` -DLINK_TORCHAO_OPS="${LINK_TORCHAO_OPS}" -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=1" -G Ninja fi cmake --build ./cmake-out --target "${TARGET}"_run