Skip to content

Commit 2758fa1

Browse files
danbevdanbev
authored
examples : add model conversion tool/example (ggml-org#15455)
* examples : add model conversion tool/example This commit adds an "example/tool" that is intended to help in the process of converting models to GGUF. Currently it supports normal causal models and embedding models. The readme contains instructions and command to guide through the process. The motivation for this to have a structured and repeatable process for model conversions and hopefully with time improve upon it to make the process easier and more reliable. We have started to use this for new model conversions internally and will continue doing so and improve it as we go along. Perhaps with time this should be placed in a different directory than the examples directory, but for now it seems like a good place to keep it while we are still developing it. * squash! examples : add model conversion tool/example Remove dependency on scikit-learn in model conversion example. * squash! examples : add model conversion tool/example Update transformer dep to use non-dev version. And also import `AutoModelForCausalLM` instead of `AutoModel` to ensure compatibility with the latest version. * squash! examples : add model conversion tool/example Remove the logits requirements file from the all requirements file.
1 parent b108e42 commit 2758fa1

33 files changed

+2230
-0
lines changed

‎examples/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ else()
3434
add_subdirectory(gen-docs)
3535
add_subdirectory(training)
3636
add_subdirectory(diffusion)
37+
add_subdirectory(model-conversion)
3738
if (NOT GGML_BACKEND_DL)
3839
add_subdirectory(convert-llama2c-to-ggml)
3940
# these examples use the backends directly and cannot be built with dynamic loading

‎examples/model-conversion/.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
.model_name
2+
data
3+
ppl

‎examples/model-conversion/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(TARGET llama-logits)
2+
add_executable(${TARGET} logits.cpp)
3+
install(TARGETS ${TARGET} RUNTIME)
4+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
5+
target_compile_features(${TARGET} PRIVATE cxx_std_17)

‎examples/model-conversion/Makefile

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# Validation functions
2+
define validate_model_path
3+
@if [ -z "$(MODEL_PATH)" ]; then \
4+
echo "Error: MODEL_PATH must be provided either as:"; \
5+
echo " 1. Environment variable: export MODEL_PATH=/path/to/model"; \
6+
echo " 2. Command line argument: make $(1) MODEL_PATH=/path/to/model"; \
7+
exit 1; \
8+
fi
9+
endef
10+
11+
define validate_embedding_model_path
12+
@if [ -z "$(EMBEDDING_MODEL_PATH)" ]; then \
13+
echo "Error: EMBEDDING_MODEL_PATH must be provided either as:"; \
14+
echo " 1. Environment variable: export EMBEDDING_MODEL_PATH=/path/to/model"; \
15+
echo " 2. Command line argument: make $(1) EMBEDDING_MODEL_PATH=/path/to/model"; \
16+
exit 1; \
17+
fi
18+
endef
19+
20+
###
21+
### Casual Model targets/recipes
22+
###
23+
causal-convert-model-bf16: OUTTYPE=bf16
24+
causal-convert-model-bf16: causal-convert-model
25+
26+
causal-convert-model:
27+
$(call validate_model_path,causal-convert-model)
28+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
29+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
30+
./scripts/causal/convert-model.sh
31+
32+
causal-run-original-model:
33+
$(call validate_model_path,causal-run-original-model)
34+
@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/run-org-model.py
35+
36+
causal-run-converted-model:
37+
@CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh
38+
39+
causal-verify-logits: causal-run-original-model causal-run-converted-model
40+
@./scripts/causal/compare-logits.py
41+
@MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH}
42+
43+
causal-run-original-embeddings:
44+
@./scripts/causal/run-casual-gen-embeddings-org.sh
45+
46+
causal-run-converted-embeddings:
47+
@./scripts/causal/run-converted-model-embeddings-logits.sh
48+
49+
causal-verify-embeddings: causal-run-original-embeddings causal-run-converted-embeddings
50+
@./scripts/causal/compare-embeddings-logits.sh
51+
52+
causal-inspect-original-model:
53+
@./scripts/utils/inspect-org-model.py
54+
55+
causal-inspect-converted-model:
56+
@./scripts/utils/inspect-converted-model.sh
57+
58+
causal-start-embedding-server:
59+
@./scripts/utils/run-embedding-server.sh ${CONVERTED_MODEL}
60+
61+
causal-curl-embedding-endpoint: causal-run-original-embeddings
62+
@./scripts/utils/curl-embedding-server.sh | ./scripts/causal/compare-embeddings-logits.sh
63+
64+
causal-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
65+
causal-quantize-Q8_0: causal-quantize-model
66+
67+
causal-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
68+
causal-quantize-Q4_0: causal-quantize-model
69+
70+
causal-quantize-model:
71+
@CONVERTED_MODEL="$(CONVERTED_MODEL)" QUANTIZED_TYPE="$(QUANTIZED_TYPE)" ./scripts/utils/quantize.sh ${CONVERTED_MODEL} ${QUANTIZED_TYPE}
72+
@echo "Export the quantized model path to QUANTIZED_MODEL variable in your environment"
73+
74+
causal-run-quantized-model:
75+
@QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/causal/run-converted-model.sh ${QUANTIZED_MODEL}
76+
77+
78+
###
79+
### Embedding Model targets/recipes
80+
###
81+
82+
embedding-convert-model-bf16: OUTTYPE=bf16
83+
embedding-convert-model-bf16: embedding-convert-model
84+
85+
embedding-convert-model:
86+
$(call validate_embedding_model_path,embedding-convert-model)
87+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \
88+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
89+
./scripts/embedding/convert-model.sh
90+
91+
embedding-run-original-model:
92+
$(call validate_embedding_model_path,embedding-run-original-model)
93+
@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/embedding/run-original-model.py
94+
95+
embedding-run-converted-model:
96+
@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/embedding/run-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}
97+
98+
embedding-verify-logits: embedding-run-original-model embedding-run-converted-model
99+
@./scripts/embedding/compare-embeddings-logits.sh
100+
101+
embedding-inspect-original-model:
102+
$(call validate_embedding_model_path,embedding-inspect-original-model)
103+
@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/utils/inspect-org-model.py -m ${EMBEDDING_MODEL_PATH}
104+
105+
embedding-inspect-converted-model:
106+
@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/utils/inspect-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}
107+
108+
embedding-start-embedding-server:
109+
@./scripts/utils/run-embedding-server.sh ${CONVERTED_EMBEDDING_MODEL}
110+
111+
embedding-curl-embedding-endpoint:
112+
@./scripts/utils/curl-embedding-server.sh | ./scripts/embedding/compare-embeddings-logits.sh
113+
114+
embedding-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
115+
embedding-quantize-Q8_0: embedding-quantize-model
116+
117+
embedding-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
118+
embedding-quantize-Q4_0: embedding-quantize-model
119+
120+
embedding-quantize-model:
121+
@./scripts/utils/quantize.sh ${CONVERTED_EMBEDDING_MODEL} ${QUANTIZED_TYPE}
122+
@echo "Export the quantized model path to QUANTIZED_EMBEDDING_MODEL variable in your environment"
123+
124+
embedding-run-quantized-model:
125+
@./scripts/embedding/run-converted-model.sh ${QUANTIZED_EMBEDDING_MODEL}
126+
127+
###
128+
### Perplexity targets/recipes
129+
###
130+
perplexity-data-gen:
131+
CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/utils/perplexity-gen.sh
132+
133+
perplexity-run-full:
134+
QUANTIZED_MODEL="$(QUANTIZED_MODEL)" LOOGITS_FILE="$(LOGITS_FILE)" \
135+
./scripts/utils/perplexity-run.sh
136+
137+
perplexity-run:
138+
QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/utils/perplexity-run-simple.sh
139+
140+
###
141+
### HuggingFace targets/recipes
142+
###
143+
144+
hf-create-model:
145+
@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}"
146+
147+
hf-create-model-private:
148+
@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -p
149+
150+
hf-upload-gguf-to-model:
151+
@./scripts/utils/hf-upload-gguf-model.py -m "${MODEL_PATH}" -r "${REPO_ID}" -o "${NAME_IN_REPO}"
152+
153+
hf-create-collection:
154+
@./scripts/utils/hf-create-collection.py -n "${NAME}" -d "${DESCRIPTION}" -ns "${NAMESPACE}"
155+
156+
hf-add-model-to-collection:
157+
@./scripts/utils/hf-add-model-to-collection.py -c "${COLLECTION}" -m "${MODEL}"
158+
159+
160+
.PHONY: clean
161+
clean:
162+
@${RM} -rf data .converted_embedding_model.txt .converted_model.txt .embedding_model_name.txt .model_name.txt
163+

0 commit comments

Comments
 (0)