Skip to content

Commit 50d5683

Browse files
author
katsu560
committed
load files from model
2 parents 2c3603e + 2210bb0 commit 50d5683

File tree

3 files changed

+210
-0
lines changed

3 files changed

+210
-0
lines changed

‎examples/yolo/gguf-addfile.py‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/usr/bin/env python3
2+
# gguf-addfile.py srcfile dstfile addfiles ...
3+
4+
from __future__ import annotations
5+
6+
import logging
7+
import argparse
8+
import os
9+
import sys
10+
from pathlib import Path
11+
#from typing import Any
12+
from typing import Any, Literal, NamedTuple, TypeVar, Union
13+
14+
import numpy as np
15+
import numpy.typing as npt
16+
17+
# Necessary to load the local gguf package
18+
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
19+
sys.path.insert(0, str(Path(__file__).parent.parent))
20+
21+
from gguf import GGUFReader, GGUFWriter, ReaderField, GGMLQuantizationType, GGUFEndian, GGUFValueType, Keys # noqa: E402
22+
23+
logger = logging.getLogger("gguf-addfile")
24+
25+
26+
def get_file_host_endian(reader: GGUFReader) -> tuple[str, str]:
27+
host_endian = 'LITTLE' if np.uint32(1) == np.uint32(1).newbyteorder("<") else 'BIG'
28+
if reader.byte_order == 'S':
29+
file_endian = 'BIG' if host_endian == 'LITTLE' else 'LITTLE'
30+
else:
31+
file_endian = host_endian
32+
return (host_endian, file_endian)
33+
34+
35+
def get_byteorder(reader: GGUFReader) -> GGUFEndian:
36+
if np.uint32(1) == np.uint32(1).newbyteorder("<"):
37+
# Host is little endian
38+
host_endian = GGUFEndian.LITTLE
39+
swapped_endian = GGUFEndian.BIG
40+
else:
41+
# Sorry PDP or other weird systems that don't use BE or LE.
42+
host_endian = GGUFEndian.BIG
43+
swapped_endian = GGUFEndian.LITTLE
44+
45+
if reader.byte_order == "S":
46+
return swapped_endian
47+
else:
48+
return host_endian
49+
50+
51+
def decode_field(field: ReaderField) -> Any:
52+
if field and field.types:
53+
main_type = field.types[0]
54+
55+
if main_type == GGUFValueType.ARRAY:
56+
sub_type = field.types[-1]
57+
58+
if sub_type == GGUFValueType.STRING:
59+
return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data]
60+
else:
61+
return [pv for idx in field.data for pv in field.parts[idx].tolist()]
62+
if main_type == GGUFValueType.STRING:
63+
return str(bytes(field.parts[-1]), encoding='utf8')
64+
else:
65+
return field.parts[-1][0]
66+
67+
return None
68+
69+
70+
def get_field_data(reader: GGUFReader, key: str) -> Any:
71+
field = reader.get_field(key)
72+
73+
return decode_field(field)
74+
75+
76+
def copy_with_filename(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, str], filename: str[Any]) -> None:
77+
for field in reader.fields.values():
78+
# Suppress virtual fields and fields written by GGUFWriter
79+
if field.name == Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'):
80+
logger.debug(f'Suppressing {field.name}')
81+
continue
82+
83+
# Skip old chat templates if we have new ones
84+
if field.name.startswith(Keys.Tokenizer.CHAT_TEMPLATE) and Keys.Tokenizer.CHAT_TEMPLATE in new_metadata:
85+
logger.debug(f'Skipping {field.name}')
86+
continue
87+
88+
old_val = decode_field(field)
89+
val = new_metadata.get(field.name, old_val)
90+
91+
if field.name in new_metadata:
92+
logger.debug(f'Modifying {field.name}: "{old_val}" -> "{val}"')
93+
del new_metadata[field.name]
94+
elif val is not None:
95+
logger.debug(f'Copying {field.name}')
96+
97+
if val is not None:
98+
writer.add_key(field.name)
99+
writer.add_val(val, field.types[0])
100+
101+
if Keys.Tokenizer.CHAT_TEMPLATE in new_metadata:
102+
logger.debug('Adding chat template(s)')
103+
writer.add_chat_template(new_metadata[Keys.Tokenizer.CHAT_TEMPLATE])
104+
del new_metadata[Keys.Tokenizer.CHAT_TEMPLATE]
105+
106+
# add filenames to kv
107+
writer.add_array(Keys.EMBEDDED_FILES, filename)
108+
109+
for tensor in reader.tensors:
110+
# Dimensions are written in reverse order, so flip them first
111+
shape = np.flipud(tensor.shape)
112+
writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type)
113+
114+
offset_next = 0
115+
len_last = 0
116+
offset_last = 0
117+
for n, tensor in enumerate(reader.tensors, 1):
118+
len_last = tensor.n_bytes
119+
offset_last = tensor.data_offset
120+
offset_next = max(offset_next, writer.ggml_pad(offset_last + int(len_last), writer.data_alignment))
121+
122+
offs = offset_next
123+
# add file info as tensor_info
124+
for path in filename:
125+
logger.debug(f'Adding {path}')
126+
with open(path, "rb") as f:
127+
data = f.read()
128+
data_len = len(data)
129+
dims = [data_len]
130+
raw_dtype = GGMLQuantizationType.I8
131+
writer.add_tensor_info(path, dims, np.float16, data_len, raw_dtype)
132+
133+
writer.write_header_to_file()
134+
writer.write_kv_data_to_file()
135+
writer.write_ti_data_to_file()
136+
137+
for tensor in reader.tensors:
138+
writer.write_tensor_data(tensor.data)
139+
140+
# write file body as tensor data
141+
for path in filename:
142+
logger.debug(f'Adding {path}')
143+
with open(path, "rb") as f:
144+
data = f.read()
145+
data_len = len(data)
146+
# write data with padding
147+
writer.write_data(data)
148+
149+
writer.close()
150+
151+
152+
def main() -> None:
153+
parser = argparse.ArgumentParser(description="Add files to GGUF file metadata")
154+
parser.add_argument("input", type=str, help="GGUF format model input filename")
155+
parser.add_argument("output", type=str, help="GGUF format model output filename")
156+
parser.add_argument("addfiles", type=str, nargs='+', help="add filenames ...")
157+
parser.add_argument("--verbose", action="store_true", help="Increase output verbosity")
158+
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
159+
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
160+
161+
logger.info(f'* Loading: {args.input}')
162+
reader = GGUFReader(args.input, 'r')
163+
arch = get_field_data(reader, Keys.General.ARCHITECTURE)
164+
endianess = get_byteorder(reader)
165+
166+
logger.info(f'* Writing: {args.output}')
167+
writer = GGUFWriter(args.output, arch=arch, endianess=endianess)
168+
169+
alignment = get_field_data(reader, Keys.General.ALIGNMENT)
170+
if alignment is not None:
171+
logger.debug(f'Setting custom alignment: {alignment}')
172+
writer.data_alignment = alignment
173+
174+
logger.info(f'* Adding: {args.addfiles}')
175+
new_metadata = {}
176+
filename = []
177+
for path in args.addfiles:
178+
filename.append(path)
179+
logger.info(f'* Adding: {path}')
180+
#new_metadata[Keys.EMBEDDED_FILES] = path
181+
copy_with_filename(reader, writer, new_metadata, filename)
182+
183+
184+
if __name__ == '__main__':
185+
main()

‎include/ggml/ggml.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2329,12 +2329,22 @@ extern "C" {
23292329
GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
23302330
GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
23312331

2332+
<<<<<<< HEAD
23322333
GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
23332334
GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
23342335
GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
23352336
GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
23362337
GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
23372338
GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int i);
2339+
=======
2340+
GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
2341+
GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
2342+
GGML_API size_t gguf_get_tensor_offset (const struct gguf_context * ctx, int i);
2343+
GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
2344+
GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
2345+
GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int i);
2346+
GGML_API int gguf_find_and_get_tensor(const struct gguf_context * ctx, const char * name, char ** data, size_t * size);
2347+
>>>>>>> 2210bb0ccfdbf848bd4ba52ec0797bea5c1acd8d
23382348

23392349
// removes key if it exists
23402350
GGML_API void gguf_remove_key(struct gguf_context * ctx, const char * key);

‎src/ggml.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21977,6 +21977,21 @@ size_t gguf_get_tensor_size(const struct gguf_context * ctx, int i) {
2197721977
return ctx->infos[i].size;
2197821978
}
2197921979

21980+
<<<<<<< HEAD
21981+
=======
21982+
int gguf_find_and_get_tensor(const struct gguf_context * ctx, const char * name, char ** data, size_t * size) {
21983+
int tensor = gguf_find_tensor(ctx, name);
21984+
if (tensor != -1) {
21985+
const size_t offset = gguf_get_tensor_offset(ctx, tensor);
21986+
char * _data = (char *)gguf_get_data(ctx);
21987+
*data = _data + offset;
21988+
*size = gguf_get_tensor_size(ctx, tensor);
21989+
}
21990+
21991+
return tensor;
21992+
}
21993+
21994+
>>>>>>> 2210bb0ccfdbf848bd4ba52ec0797bea5c1acd8d
2198021995
// returns the index
2198121996
static int gguf_get_or_add_key(struct gguf_context * ctx, const char * key) {
2198221997
const int idx = gguf_find_key(ctx, key);

0 commit comments

Comments
 (0)