Skip to content

quantize: Add UINT type to KV overrides #14182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

quantize: Add UINT type to KV overrides #14182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b44038
Add UINT type to KV overrides
EAddario Jun 13, 2025
b5dded3
Merge branch 'master' into kv_override
EAddario Jun 13, 2025
111448c
Merge branch 'master' into kv_override
EAddario Jun 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions common/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,10 @@ bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_over
sep += 4;
kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
kvo.val_i64 = std::atol(sep);
} else if (strncmp(sep, "uint:", 5) == 0) {
sep += 5;
kvo.tag = LLAMA_KV_OVERRIDE_TYPE_UINT;
kvo.val_u64 = std::atol(sep);
} else if (strncmp(sep, "float:", 6) == 0) {
sep += 6;
kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
Expand Down
10 changes: 6 additions & 4 deletions include/llama.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ extern "C" {
LLAMA_KV_OVERRIDE_TYPE_FLOAT,
LLAMA_KV_OVERRIDE_TYPE_BOOL,
LLAMA_KV_OVERRIDE_TYPE_STR,
LLAMA_KV_OVERRIDE_TYPE_UINT,
};

struct llama_model_kv_override {
Expand All @@ -280,10 +281,11 @@ extern "C" {
char key[128];

union {
int64_t val_i64;
double val_f64;
bool val_bool;
char val_str[128];
int64_t val_i64;
uint64_t val_u64;
double val_f64;
bool val_bool;
char val_str[128];
};
};

Expand Down
8 changes: 8 additions & 0 deletions src/llama-model-loader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ namespace GGUFMeta {
case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";
case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";
case LLAMA_KV_OVERRIDE_TYPE_UINT: return "uint";
}
return "unknown";
}
Expand All @@ -180,6 +181,9 @@ namespace GGUFMeta {
case LLAMA_KV_OVERRIDE_TYPE_INT: {
LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);
} break;
case LLAMA_KV_OVERRIDE_TYPE_UINT: {
LLAMA_LOG_INFO("%" PRIu64 "\n", ovrd->val_u64);
} break;
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {
LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);
} break;
Expand Down Expand Up @@ -216,6 +220,10 @@ namespace GGUFMeta {
target = ovrd->val_i64;
return true;
}
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_UINT, ovrd)) {
target = ovrd->val_u64;
return true;
}
return false;
}

Expand Down
2 changes: 2 additions & 0 deletions src/llama-quant.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
gguf_set_val_f32(ctx_out.get(), o.key, o.val_f64);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {
gguf_set_val_i32(ctx_out.get(), o.key, o.val_i64);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_UINT) {
gguf_set_val_u32(ctx_out.get(), o.key, o.val_u64);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {
gguf_set_val_bool(ctx_out.get(), o.key, o.val_bool);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
Expand Down
Loading