Skip to content

Commit 21f1113

Browse files
danbevdanbev
committed
examples : add model conversion tool/example
This commit adds an "example/tool" that is intended to help in the process of converting models to GGUF. Currently it supports normal causal models and embedding models. The readme contains instructions and command to guide through the process. The motivation for this to have a structured and repeatable process for model conversions and hopefully with time improve upon it to make the process easier and more reliable. We have started to use this for new model conversions internally and will continue doing so and improve it as we go along. Perhaps with time this should be placed in a different directory than the examples directory, but for now it seems like a good place to keep it while we are still developing it.
1 parent 657b8a7 commit 21f1113

35 files changed

+2211
-0
lines changed

‎examples/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ else()
3434
add_subdirectory(gen-docs)
3535
add_subdirectory(training)
3636
add_subdirectory(diffusion)
37+
add_subdirectory(model-conversion)
3738
if (NOT GGML_BACKEND_DL)
3839
add_subdirectory(convert-llama2c-to-ggml)
3940
# these examples use the backends directly and cannot be built with dynamic loading

‎examples/model-conversion/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
.model_name
2+
data
3+
ppl

‎examples/model-conversion/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(TARGET llama-logits)
2+
add_executable(${TARGET} logits.cpp)
3+
install(TARGETS ${TARGET} RUNTIME)
4+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
5+
target_compile_features(${TARGET} PRIVATE cxx_std_17)

‎examples/model-conversion/Makefile‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# Validation functions
2+
define validate_model_path
3+
@if [ -z "$(MODEL_PATH)" ]; then \
4+
echo "Error: MODEL_PATH must be provided either as:"; \
5+
echo " 1. Environment variable: export MODEL_PATH=/path/to/model"; \
6+
echo " 2. Command line argument: make $(1) MODEL_PATH=/path/to/model"; \
7+
exit 1; \
8+
fi
9+
endef
10+
11+
define validate_embedding_model_path
12+
@if [ -z "$(EMBEDDING_MODEL_PATH)" ]; then \
13+
echo "Error: EMBEDDING_MODEL_PATH must be provided either as:"; \
14+
echo " 1. Environment variable: export EMBEDDING_MODEL_PATH=/path/to/model"; \
15+
echo " 2. Command line argument: make $(1) EMBEDDING_MODEL_PATH=/path/to/model"; \
16+
exit 1; \
17+
fi
18+
endef
19+
20+
###
21+
### Casual Model targets/recipes
22+
###
23+
causal-convert-model-bf16: OUTTYPE=bf16
24+
causal-convert-model-bf16: causal-convert-model
25+
26+
causal-convert-model:
27+
$(call validate_model_path,causal-convert-model)
28+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
29+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
30+
./scripts/causal/convert-model.sh
31+
32+
causal-run-original-model:
33+
$(call validate_model_path,causal-run-original-model)
34+
@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/run-org-model.py
35+
36+
causal-run-converted-model:
37+
@CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh
38+
39+
causal-verify-logits: causal-run-original-model causal-run-converted-model
40+
@./scripts/causal/compare-logits.py
41+
@MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH}
42+
43+
causal-run-original-embeddings:
44+
@./scripts/causal/run-casual-gen-embeddings-org.sh
45+
46+
causal-run-converted-embeddings:
47+
@./scripts/causal/run-converted-model-embeddings-logits.sh
48+
49+
causal-verify-embeddings: causal-run-original-embeddings causal-run-converted-embeddings
50+
@./scripts/causal/compare-embeddings-logits.sh
51+
52+
causal-inspect-original-model:
53+
@./scripts/utils/inspect-org-model.py
54+
55+
causal-inspect-converted-model:
56+
@./scripts/utils/inspect-converted-model.sh
57+
58+
causal-start-embedding-server:
59+
@./scripts/utils/run-embedding-server.sh ${CONVERTED_MODEL}
60+
61+
causal-curl-embedding-endpoint: causal-run-original-embeddings
62+
@./scripts/utils/curl-embedding-server.sh | ./scripts/causal/compare-embeddings-logits.sh
63+
64+
causal-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
65+
causal-quantize-Q8_0: causal-quantize-model
66+
67+
causal-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
68+
causal-quantize-Q4_0: causal-quantize-model
69+
70+
causal-quantize-model:
71+
@CONVERTED_MODEL="$(CONVERTED_MODEL)" QUANTIZED_TYPE="$(QUANTIZED_TYPE)" ./scripts/utils/quantize.sh ${CONVERTED_MODEL} ${QUANTIZED_TYPE}
72+
@echo "Export the quantized model path to QUANTIZED_MODEL variable in your environment"
73+
74+
causal-run-quantized-model:
75+
@QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/causal/run-converted-model.sh ${QUANTIZED_MODEL}
76+
77+
78+
###
79+
### Embedding Model targets/recipes
80+
###
81+
82+
embedding-convert-model-bf16: OUTTYPE=bf16
83+
embedding-convert-model-bf16: embedding-convert-model
84+
85+
embedding-convert-model:
86+
$(call validate_embedding_model_path,embedding-convert-model)
87+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \
88+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
89+
./scripts/embedding/convert-model.sh
90+
91+
embedding-run-original-model:
92+
$(call validate_embedding_model_path,embedding-run-original-model)
93+
@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/embedding/run-original-model.py
94+
95+
embedding-run-converted-model:
96+
@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/embedding/run-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}
97+
98+
embedding-verify-logits: embedding-run-original-model embedding-run-converted-model
99+
@./scripts/embedding/compare-embeddings-logits.sh
100+
101+
embedding-inspect-original-model:
102+
$(call validate_embedding_model_path,embedding-inspect-original-model)
103+
@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/utils/inspect-org-model.py -m ${EMBEDDING_MODEL_PATH}
104+
105+
embedding-inspect-converted-model:
106+
@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/utils/inspect-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}
107+
108+
embedding-start-embedding-server:
109+
@./scripts/utils/run-embedding-server.sh ${CONVERTED_EMBEDDING_MODEL}
110+
111+
embedding-curl-embedding-endpoint:
112+
@./scripts/utils/curl-embedding-server.sh | ./scripts/embedding/compare-embeddings-logits.sh
113+
114+
embedding-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
115+
embedding-quantize-Q8_0: embedding-quantize-model
116+
117+
embedding-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
118+
embedding-quantize-Q4_0: embedding-quantize-model
119+
120+
embedding-quantize-model:
121+
@./scripts/utils/quantize.sh ${CONVERTED_EMBEDDING_MODEL} ${QUANTIZED_TYPE}
122+
@echo "Export the quantized model path to QUANTIZED_EMBEDDING_MODEL variable in your environment"
123+
124+
embedding-run-quantized-model:
125+
@./scripts/embedding/run-converted-model.sh ${QUANTIZED_EMBEDDING_MODEL}
126+
127+
###
128+
### Perplexity targets/recipes
129+
###
130+
perplexity-data-gen:
131+
CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/utils/perplexity-gen.sh
132+
133+
perplexity-run-full:
134+
QUANTIZED_MODEL="$(QUANTIZED_MODEL)" LOOGITS_FILE="$(LOGITS_FILE)" \
135+
./scripts/utils/perplexity-run.sh
136+
137+
perplexity-run:
138+
QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/utils/perplexity-run-simple.sh
139+
140+
###
141+
### HuggingFace targets/recipes
142+
###
143+
144+
hf-create-model:
145+
@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}"
146+
147+
hf-create-model-private:
148+
@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -p
149+
150+
hf-upload-gguf-to-model:
151+
@./scripts/utils/hf-upload-gguf-model.py -m "${MODEL_PATH}" -r "${REPO_ID}" -o "${NAME_IN_REPO}"
152+
153+
hf-create-collection:
154+
@./scripts/utils/hf-create-collection.py -n "${NAME}" -d "${DESCRIPTION}" -ns "${NAMESPACE}"
155+
156+
hf-add-model-to-collection:
157+
@./scripts/utils/hf-add-model-to-collection.py -c "${COLLECTION}" -m "${MODEL}"
158+
159+
160+
.PHONY: clean
161+
clean:
162+
@${RM} -rf data .converted_embedding_model.txt .converted_model.txt .embedding_model_name.txt .model_name.txt
163+

0 commit comments

Comments
 (0)