Move models/ggml-vocab-* to models/tests/ggml-vocab-*

Author: MichelleTanPY

.gitignore

@@ -78,7 +78,7 @@ autogen-*.md
 models/*
 models-mnt
 !models/.editorconfig
-!models/ggml-vocab-*.gguf*
+!models/tests/ggml-vocab-*.gguf*
 
 # Zig
 zig-out/

Makefile

@@ -216,14 +216,14 @@ test: $(TEST_TARGETS)
 	@failures=0; \
 	for test_target in $(TEST_TARGETS); do \
 		if [ "$$test_target" = "tests/test-tokenizer-0" ]; then \
-			./$$test_target $(CURDIR)/models/ggml-vocab-llama-spm.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-llama-bpe.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-phi-3.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-falcon.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-bert-bge.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-starcoder.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-gpt-2.gguf; \
-			./$$test_target $(CURDIR)/models/ggml-vocab-refact.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-llama-spm.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-llama-bpe.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-phi-3.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-falcon.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-bert-bge.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-starcoder.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-gpt-2.gguf; \
+			./$$test_target $(CURDIR)/models/tests/ggml-vocab-refact.gguf; \
 		elif [ "$$test_target" = "tests/test-tokenizer-1-spm" ]; then \
 			continue; \
 		elif [ "$$test_target" = "tests/test-tokenizer-1-bpe" ]; then \

convert_hf_to_gguf_update.py

@@ -351,12 +351,12 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")
         continue  # Skip this model and continue with the next one in the loop
 
-    with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
+    with open(f"models/tests/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
         for text in tests:
             f.write(f"{text}")
             f.write("\n__ggml_vocab_test__\n")
 
-    with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
+    with open(f"models/tests/ggml-vocab-{name}.gguf.out", "w") as f:
         for text in tests:
             res = tokenizer.encode(text, add_special_tokens=False)
             for r in res: