@@ -84,25 +84,6 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen2 ARGS ${CMAKE
8484llama_test(test -tokenizer-0 NAME test -tokenizer-0-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-refact.gguf)
8585llama_test(test -tokenizer-0 NAME test -tokenizer-0-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-starcoder.gguf)
8686
87- # build test-tokenizer-parallel target once and add many tests
88- add_executable (test -tokenizer-parallel test -tokenizer-parallel.cpp)
89- target_link_libraries (test -tokenizer-parallel PRIVATE common)
90- install (TARGETS test -tokenizer-parallel RUNTIME)
91-
92- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-bert-bge ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-bert-bge.gguf)
93- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-command -r ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-command -r.gguf)
94- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-deepseek-coder.gguf)
95- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-deepseek-llm.gguf)
96- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-falcon.gguf)
97- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-gpt-2 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-gpt-2.gguf)
98- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-llama-bpe ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-llama-bpe.gguf)
99- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-llama-spm.gguf)
100- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-mpt ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-mpt.gguf)
101- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-phi-3 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-phi-3.gguf)
102- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-qwen2 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-qwen2.gguf)
103- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-refact.gguf)
104- llama_test(test -tokenizer-parallel NAME test -tokenizer-parallel-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-starcoder.gguf)
105-
10687# build test-tokenizer-1-bpe target once and add many tests
10788add_executable (test -tokenizer-1-bpe test -tokenizer-1-bpe.cpp)
10889target_link_libraries (test -tokenizer-1-bpe PRIVATE common)
0 commit comments