@@ -42,34 +42,6 @@ function(llama_test target)
4242 set_property (TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL} )
4343endfunction ()
4444
45- function (llama_test_cmd target )
46- include (CMakeParseArguments)
47- set (options )
48- set (oneValueArgs NAME LABEL WORKING_DIRECTORY )
49- set (multiValueArgs ARGS)
50- cmake_parse_arguments (LLAMA_TEST "${options} " "${oneValueArgs} " "${multiValueArgs} " ${ARGN} )
51-
52- if (NOT DEFINED LLAMA_TEST_LABEL)
53- set (LLAMA_TEST_LABEL "main" )
54- endif ()
55- if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY)
56- set (LLAMA_TEST_WORKING_DIRECTORY .)
57- endif ()
58- if (DEFINED LLAMA_TEST_NAME)
59- set (TEST_NAME ${LLAMA_TEST_NAME} )
60- else ()
61- set (TEST_NAME ${target} )
62- endif ()
63-
64- add_test (
65- NAME ${TEST_NAME}
66- WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
67- COMMAND ${target}
68- ${LLAMA_TEST_ARGS} )
69-
70- set_property (TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL} )
71- endfunction ()
72-
7345# Builds and runs a test source file.
7446# Optional args:
7547# - NAME: name of the executable & test target (defaults to the source file name without extension)
@@ -111,31 +83,25 @@ endfunction()
11183# build test-tokenizer-0 target once and add many tests
11284llama_build(test -tokenizer-0.cpp)
11385
114- llama_test(test -tokenizer-0 NAME test -tokenizer-0-bert-bge ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-bert-bge.gguf)
115- llama_test(test -tokenizer-0 NAME test -tokenizer-0-command -r ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-command -r.gguf)
116- llama_test(test -tokenizer-0 NAME test -tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-deepseek-coder.gguf)
117- llama_test(test -tokenizer-0 NAME test -tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-deepseek-llm.gguf)
118- llama_test(test -tokenizer-0 NAME test -tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-falcon.gguf)
119- llama_test(test -tokenizer-0 NAME test -tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-gpt-2.gguf)
120- llama_test(test -tokenizer-0 NAME test -tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-llama-bpe.gguf)
121- llama_test(test -tokenizer-0 NAME test -tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-llama-spm.gguf)
122- llama_test(test -tokenizer-0 NAME test -tokenizer-0-mpt ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-mpt.gguf)
123- llama_test(test -tokenizer-0 NAME test -tokenizer-0-phi-3 ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-phi-3.gguf)
124- llama_test(test -tokenizer-0 NAME test -tokenizer-0-qwen2 ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-qwen2.gguf)
125- llama_test(test -tokenizer-0 NAME test -tokenizer-0-refact ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-refact.gguf)
126- llama_test(test -tokenizer-0 NAME test -tokenizer-0-starcoder ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-starcoder.gguf)
127-
128- if (NOT WIN32 )
129- llama_test_cmd(
130- ${CMAKE_CURRENT_SOURCE_DIR} /test -tokenizers-repo.sh
131- NAME test -tokenizers-ggml-vocabs
132- WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
133- ARGS https://huggingface.co/ggml-org/vocabs ${PROJECT_SOURCE_DIR} /models/ggml-vocabs
134- )
135- endif ()
86+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-bert-bge ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-bert-bge.gguf)
87+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-command -r ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-command -r.gguf)
88+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-deepseek-coder.gguf)
89+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-deepseek-llm.gguf)
90+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-falcon.gguf)
91+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-gpt-2 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-gpt-2.gguf)
92+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-llama-bpe ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-llama-bpe.gguf)
93+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-llama-spm.gguf)
94+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-mpt ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-mpt.gguf)
95+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-phi-3 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-phi-3.gguf)
96+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-qwen2 ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-qwen2.gguf)
97+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-refact.gguf)
98+ llama_test(test -tokenizer-0 NAME test -tokenizer-0-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR} /../models/ggml-vocab-starcoder.gguf)
99+
100+ # TODO: missing HF tokenizer for this model in convert_hf_to_gguf_update.py, see https://github.com/ggml-org/llama.cpp/pull/13847
101+ # llama_test(test-tokenizer-0 NAME test-tokenizer-0-nomic-bert-moe ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-nomic-bert-moe.gguf)
136102
137103if (LLAMA_LLGUIDANCE)
138- llama_build_and_test(test -grammar-llguidance.cpp ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-llama-bpe.gguf)
104+ llama_build_and_test(test -grammar-llguidance.cpp ARGS ${CMAKE_CURRENT_SOURCE_DIR} /.. /models/ggml-vocab-llama-bpe.gguf)
139105endif ()
140106
141107if (NOT WIN32 OR NOT BUILD_SHARED_LIBS )
@@ -147,8 +113,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
147113 llama_build_and_test(test -chat.cpp)
148114 # TODO: disabled on loongarch64 because the ggml-ci node lacks Python 3.8
149115 if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64" )
150- llama_build_and_test(test -json-schema-to-grammar.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} )
151- target_include_directories (test -json-schema-to-grammar PRIVATE ${PROJECT_SOURCE_DIR} /tools/server)
116+ llama_build_and_test(test -json-schema-to-grammar.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} /.. )
117+ target_include_directories (test -json-schema-to-grammar PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} /.. /tools/server)
152118 endif ()
153119
154120 if (NOT GGML_BACKEND_DL)
@@ -161,20 +127,20 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
161127 llama_build(test -tokenizer-1-bpe.cpp)
162128
163129 # TODO: disabled due to slowness
164- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-aquila ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-aquila.gguf)
165- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-falcon ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-falcon.gguf)
166- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-2 ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-gpt-2.gguf)
167- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-neox ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-gpt-neox.gguf)
168- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-llama-bpe ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-llama-bpe.gguf --ignore-merges)
169- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-mpt ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-mpt.gguf)
170- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-refact ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-refact.gguf)
171- #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-starcoder ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-starcoder.gguf)
130+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-aquila ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-aquila.gguf)
131+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-falcon.gguf)
132+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-gpt-2.gguf)
133+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-neox ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-gpt-neox.gguf)
134+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-llama-bpe ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-llama-bpe.gguf --ignore-merges)
135+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-mpt ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-mpt.gguf)
136+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-refact.gguf)
137+ #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-starcoder.gguf)
172138
173139 # build test-tokenizer-1-spm target once and add many tests
174140 llama_build(test -tokenizer-1-spm.cpp)
175141
176- llama_test(test -tokenizer-1-spm NAME test -tokenizer-1-llama-spm ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-llama-spm.gguf)
177- #llama_test(test-tokenizer-1-spm NAME test-tokenizer-1-baichuan ARGS ${PROJECT_SOURCE_DIR} /models/ggml-vocab-baichuan.gguf)
142+ llama_test(test -tokenizer-1-spm NAME test -tokenizer-1-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR} /.. /models/ggml-vocab-llama-spm.gguf)
143+ #llama_test(test-tokenizer-1-spm NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/.. /models/ggml-vocab-baichuan.gguf)
178144
179145 # llama_build_and_test(test-double-float.cpp) # SLOW
180146endif ()
@@ -185,8 +151,6 @@ llama_build_and_test(test-json-partial.cpp)
185151llama_build_and_test(test -log .cpp)
186152llama_build_and_test(test -regex -partial.cpp)
187153
188- llama_build_and_test(test -thread-safety.cpp ARGS -hf ggml-org/models -hff tinyllamas/stories15M-q4_0.gguf -ngl 99 -p "The meaning of life is" -n 128 -c 256 -ub 32 -np 4)
189-
190154# this fails on windows (github hosted runner) due to curl DLL not found (exit code 0xc0000135)
191155if (NOT WIN32 )
192156 llama_build_and_test(test -arg-parser.cpp)
0 commit comments