Skip to content

Commit c2557b5

Browse files
NexesenexNexesenex
committed
Merge branch 'master' into esocrok
2 parents 566ec02 + 9fee191 commit c2557b5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2766
-497
lines changed

‎common/arg.cpp‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,13 @@
5656
#endif
5757
#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
5858

59+
// isatty
60+
#if defined(_WIN32)
61+
#include <io.h>
62+
#else
63+
#include <unistd.h>
64+
#endif
65+
5966
using json = nlohmann::ordered_json;
6067

6168
std::initializer_list<enum llama_example> mmproj_examples = {
@@ -102,6 +109,14 @@ static void write_file(const std::string & fname, const std::string & content) {
102109
}
103110
}
104111

112+
static bool is_output_a_tty() {
113+
#if defined(_WIN32)
114+
return _isatty(_fileno(stdout));
115+
#else
116+
return isatty(1);
117+
#endif
118+
}
119+
105120
common_arg & common_arg::set_examples(std::initializer_list<enum llama_example> examples) {
106121
this->examples = std::move(examples);
107122
return *this;
@@ -664,7 +679,11 @@ static std::string show_masked_url(const common_url & parts) {
664679
return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + parts.host + parts.path;
665680
}
666681

667-
static void print_progress(size_t current, size_t total) { // TODO isatty
682+
static void print_progress(size_t current, size_t total) {
683+
if (!is_output_a_tty()) {
684+
return;
685+
}
686+
668687
if (!total) {
669688
return;
670689
}

‎common/chat-parser.cpp‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,35 @@ bool common_chat_msg_parser::add_tool_calls(const json & arr) {
7575
}
7676
return true;
7777
}
78+
79+
bool common_chat_msg_parser::add_tool_call_short_form(const json & tool_call) {
80+
if (!tool_call.is_object() || tool_call.size() != 1) {
81+
return false;
82+
}
83+
84+
// Get the tool name (the single key in the object)
85+
auto it = tool_call.begin();
86+
std::string name = it.key();
87+
88+
if (name.empty()) {
89+
return false;
90+
}
91+
92+
// Get the arguments (the nested object)
93+
const json & args_json = it.value();
94+
std::string arguments = "";
95+
96+
if (args_json.is_object()) {
97+
arguments = args_json.dump();
98+
} else if (args_json.is_string()) {
99+
arguments = args_json;
100+
} else if (!args_json.is_null()) {
101+
// For other types, convert to string representation
102+
arguments = args_json.dump();
103+
}
104+
105+
return add_tool_call(name, "", arguments);
106+
}
78107
void common_chat_msg_parser::finish() {
79108
if (!is_partial_ && pos_ != input_.size()) {
80109
throw std::runtime_error("Unexpected content at end of input");// + input_.substr(pos_));

‎common/chat-parser.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ class common_chat_msg_parser {
6464
// Adds an array of tool calls using their "name", "id" and "arguments" fields.
6565
bool add_tool_calls(const nlohmann::ordered_json & arr);
6666

67+
// Adds a tool call using the short form: { "tool_name": { "arg1": val, "arg2": val } }
68+
bool add_tool_call_short_form(const nlohmann::ordered_json & tool_call);
69+
6770
void finish();
6871

6972
bool consume_spaces();

‎common/chat.cpp‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,7 @@ const char * common_chat_format_name(common_chat_format format) {
638638
case COMMON_CHAT_FORMAT_GPT_OSS: return "GPT-OSS";
639639
case COMMON_CHAT_FORMAT_SEED_OSS: return "Seed-OSS";
640640
case COMMON_CHAT_FORMAT_NEMOTRON_V2: return "Nemotron V2";
641+
case COMMON_CHAT_FORMAT_APERTUS: return "Apertus";
641642
default:
642643
throw std::runtime_error("Unknown chat format");
643644
}
@@ -801,6 +802,7 @@ static std::string apply(
801802
}
802803
tmpl_inputs.add_generation_prompt = inputs.add_generation_prompt;
803804
tmpl_inputs.extra_context = inputs.extra_context;
805+
tmpl_inputs.extra_context["enable_thinking"] = inputs.enable_thinking;
804806
if (additional_context) {
805807
tmpl_inputs.extra_context.merge_patch(*additional_context);
806808
}
@@ -1264,6 +1266,75 @@ static common_chat_params common_chat_params_init_nemotron_v2(const common_chat_
12641266
}
12651267
return data;
12661268
}
1269+
1270+
static common_chat_params common_chat_params_init_apertus(const common_chat_template & tmpl, const struct templates_params & inputs) {
1271+
common_chat_params data;
1272+
1273+
// Generate the prompt using the apply() function with the template
1274+
data.prompt = apply(tmpl, inputs);
1275+
data.format = COMMON_CHAT_FORMAT_APERTUS;
1276+
1277+
// Handle thinking tags appropriately based on inputs.enable_thinking
1278+
if (string_ends_with(data.prompt, "<|inner_prefix|>")) {
1279+
if (!inputs.enable_thinking) {
1280+
data.prompt += "<|inner_suffix|>";
1281+
} else {
1282+
data.thinking_forced_open = true;
1283+
}
1284+
}
1285+
1286+
// When tools are present, build grammar for the <|tools_prefix|> format
1287+
if (!inputs.tools.is_null() && inputs.tools.is_array() && !inputs.tools.empty()) {
1288+
data.grammar_lazy = true;
1289+
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
1290+
auto schemas = json::array();
1291+
foreach_function(inputs.tools, [&](const json & tool) {
1292+
const auto & function = tool.at("function");
1293+
schemas.push_back({
1294+
{ "type", "object" },
1295+
{ "properties",
1296+
{
1297+
{ function.at("name"), function.at("parameters") }
1298+
} },
1299+
{ "required", json::array({ function.at("name") }) },
1300+
});
1301+
});
1302+
auto schema = json{
1303+
{ "type", "array" },
1304+
{ "items", schemas.size() == 1 ? schemas[0] : json{ { "anyOf", schemas } } },
1305+
{ "minItems", 1 },
1306+
};
1307+
if (!inputs.parallel_tool_calls) {
1308+
schema["maxItems"] = 1;
1309+
}
1310+
builder.add_rule("root",
1311+
std::string(data.thinking_forced_open ? "( \"<|inner_suffix|>\" space )? " : "") +
1312+
"\"<|tools_prefix|>\"" + builder.add_schema("tool_calls", schema) + "\"<|tools_suffix|>\"");
1313+
});
1314+
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL,
1315+
// If thinking_forced_open, then we capture the <|inner_suffix|> tag in the grammar,
1316+
// (important for required tool choice) and in the trigger's first capture (decides what is sent to the grammar)
1317+
std::string(data.thinking_forced_open ?
1318+
"[\\s\\S]*?(<\\|inner_suffix\\|>\\s*)" :
1319+
"(?:<\\|inner_prefix\\|>[\\s\\S]*?<\\|inner_suffix\\|>\\s*)?") +
1320+
"(<\\|tools_prefix\\|>)[\\s\\S]*" });
1321+
data.preserved_tokens = {
1322+
"<|system_start|>",
1323+
"<|system_end|>",
1324+
"<|developer_start|>",
1325+
"<|developer_end|>",
1326+
"<|user_start|>",
1327+
"<|user_end|>",
1328+
"<|assistant_start|>",
1329+
"<|assistant_end|>",
1330+
"<|inner_prefix|>",
1331+
"<|inner_suffix|>",
1332+
"<|tools_prefix|>",
1333+
"<|tools_suffix|>",
1334+
};
1335+
}
1336+
return data;
1337+
}
12671338
static void common_chat_parse_llama_3_1(common_chat_msg_parser & builder, bool with_builtin_tools = false) {
12681339
if (!builder.syntax().parse_tool_calls) {
12691340
builder.add_content(builder.consume_rest());
@@ -2323,6 +2394,37 @@ static void common_chat_parse_nemotron_v2(common_chat_msg_parser & builder) {
23232394
builder.add_content(builder.consume_rest());
23242395
}
23252396

2397+
static void common_chat_parse_apertus(common_chat_msg_parser & builder) {
2398+
// Parse thinking tags
2399+
builder.try_parse_reasoning("<|inner_prefix|>", "<|inner_suffix|>");
2400+
if (!builder.syntax().parse_tool_calls) {
2401+
builder.add_content(builder.consume_rest());
2402+
return;
2403+
}
2404+
2405+
// Look for tool calls
2406+
static const common_regex tool_call_regex(regex_escape("<|tools_prefix|>"));
2407+
if (auto res = builder.try_find_regex(tool_call_regex)) {
2408+
builder.move_to(res->groups[0].end);
2409+
2410+
auto tool_calls_data = builder.consume_json();
2411+
if (tool_calls_data.json.is_array()) {
2412+
builder.consume_spaces();
2413+
if (!builder.try_consume_literal("<|tools_suffix|>")) {
2414+
throw common_chat_msg_partial_exception("Incomplete tool call");
2415+
}
2416+
for (const auto & value : tool_calls_data.json) {
2417+
if (value.is_object()) {
2418+
builder.add_tool_call_short_form(value);
2419+
}
2420+
}
2421+
} else {
2422+
throw common_chat_msg_partial_exception("Incomplete tool call");
2423+
}
2424+
}
2425+
builder.add_content(builder.consume_rest());
2426+
}
2427+
23262428
static void common_chat_parse_seed_oss(common_chat_msg_parser & builder) {
23272429
// Parse thinking tags first - this handles the main reasoning content
23282430
builder.try_parse_reasoning("<seed:think>", "</seed:think>");
@@ -2567,6 +2669,11 @@ static common_chat_params common_chat_templates_apply_jinja(
25672669
return common_chat_params_init_nemotron_v2(tmpl, params);
25682670
}
25692671

2672+
// Apertus format detection
2673+
if (src.find("<|system_start|>") != std::string::npos && src.find("<|tools_prefix|>") != std::string::npos) {
2674+
return common_chat_params_init_apertus(tmpl, params);
2675+
}
2676+
25702677
// Use generic handler when mixing tools + JSON schema.
25712678
// TODO: support that mix in handlers below.
25722679
if ((params.tools.is_array() && params.json_schema.is_object())) {
@@ -2734,6 +2841,9 @@ static void common_chat_parse(common_chat_msg_parser & builder) {
27342841
case COMMON_CHAT_FORMAT_NEMOTRON_V2:
27352842
common_chat_parse_nemotron_v2(builder);
27362843
break;
2844+
case COMMON_CHAT_FORMAT_APERTUS:
2845+
common_chat_parse_apertus(builder);
2846+
break;
27372847
default:
27382848
throw std::runtime_error(std::string("Unsupported format: ") + common_chat_format_name(builder.syntax().format));
27392849
}

‎common/chat.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ enum common_chat_format {
114114
COMMON_CHAT_FORMAT_GPT_OSS,
115115
COMMON_CHAT_FORMAT_SEED_OSS,
116116
COMMON_CHAT_FORMAT_NEMOTRON_V2,
117+
COMMON_CHAT_FORMAT_APERTUS,
117118

118119
COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
119120
};

‎convert_hf_to_gguf.py‎

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4255,7 +4255,8 @@ def set_gguf_parameters(self):
42554255
# This logic matches modeling_plamo.py's is_mamba function
42564256
mamba_step = hparams.get("mamba_step", 2)
42574257
mamba_enabled = hparams.get("mamba_enabled", True)
4258-
mamba_layers = []
4258+
num_key_value_heads = []
4259+
num_attention_heads = []
42594260

42604261
if mamba_enabled:
42614262
for i in range(block_count):
@@ -4265,17 +4266,21 @@ def set_gguf_parameters(self):
42654266
else:
42664267
is_mamba = (i % mamba_step) != (mamba_step // 2)
42674268
if is_mamba:
4268-
mamba_layers.append(0)
4269+
num_key_value_heads.append(0)
4270+
num_attention_heads.append(0)
42694271
else:
4270-
mamba_layers.append(hparams.get("num_key_value_heads", 4))
4272+
num_key_value_heads.append(hparams.get("num_key_value_heads", 4))
4273+
num_attention_heads.append(hparams.get("num_attention_heads", 32))
42714274

4272-
if mamba_layers:
4273-
self.gguf_writer.add_head_count_kv(mamba_layers)
4275+
if num_key_value_heads and num_attention_heads:
4276+
self.gguf_writer.add_head_count_kv(num_key_value_heads)
4277+
self.gguf_writer.add_head_count(num_attention_heads)
42744278

42754279
self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 2048))
42764280
self.gguf_writer.add_embedding_length(hparams.get("hidden_size", 4096))
4281+
self.gguf_writer.add_key_length(hparams.get("hidden_size_per_head", 128))
4282+
self.gguf_writer.add_value_length(hparams.get("hidden_size_per_head", 128))
42774283
self.gguf_writer.add_block_count(block_count)
4278-
self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 32))
42794284
self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))
42804285
self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 10000))
42814286

@@ -8945,6 +8950,43 @@ def prepare_tensors(self):
89458950
raise ValueError(f"Unprocessed experts: {experts}")
89468951

89478952

8953+
@ModelBase.register("ApertusForCausalLM")
8954+
class ApertusModel(LlamaModel):
8955+
model_arch = gguf.MODEL_ARCH.APERTUS
8956+
undo_permute = False
8957+
8958+
_alpha_n = {}
8959+
_alpha_p = {}
8960+
_beta = {}
8961+
_eps = {}
8962+
8963+
def modify_tensors(self, data_torch, name, bid):
8964+
# Handle xIELU activation parameters
8965+
n_layers = self.hparams["num_hidden_layers"]
8966+
if name.endswith(".act_fn.alpha_n"):
8967+
self._alpha_n[bid] = data_torch.to("cpu").float().item()
8968+
if (len(self._alpha_n) == n_layers):
8969+
self.gguf_writer.add_xielu_alpha_n([self._alpha_n[k] for k in sorted(self._alpha_n)])
8970+
return []
8971+
if name.endswith(".act_fn.alpha_p"):
8972+
self._alpha_p[bid] = data_torch.to("cpu").float().item()
8973+
if (len(self._alpha_p) == n_layers):
8974+
self.gguf_writer.add_xielu_alpha_p([self._alpha_p[k] for k in sorted(self._alpha_p)])
8975+
return []
8976+
if name.endswith(".act_fn.beta"):
8977+
self._beta[bid] = data_torch.to("cpu").float().item()
8978+
if (len(self._beta) == n_layers):
8979+
self.gguf_writer.add_xielu_beta([self._beta[k] for k in sorted(self._beta)])
8980+
return []
8981+
if name.endswith(".act_fn.eps"):
8982+
self._eps[bid] = data_torch.to("cpu").float().item()
8983+
if (len(self._eps) == n_layers):
8984+
self.gguf_writer.add_xielu_eps([self._eps[k] for k in sorted(self._eps)])
8985+
return []
8986+
8987+
return super().modify_tensors(data_torch, name, bid)
8988+
8989+
89488990
class MistralModel(LlamaModel):
89498991
model_arch = gguf.MODEL_ARCH.LLAMA
89508992
model_name = "Mistral"
@@ -9112,7 +9154,7 @@ def meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: tuple[int, ...]) -
91129154
def from_safetensors_slice(cls, st_slice: Any) -> Tensor:
91139155
dtype = cls._dtype_str_map[st_slice.get_dtype()]
91149156
shape: tuple[int, ...] = tuple(st_slice.get_shape())
9115-
lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(st_slice,), func=lambda s: s[:])
9157+
lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(st_slice,), func=lambda s: s[...] if len(s.get_shape()) == 0 else s[:])
91169158
return cast(torch.Tensor, lazy)
91179159

91189160
@classmethod

‎ggml/include/ggml.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@
237237
#define GGML_EXIT_SUCCESS 0
238238
#define GGML_EXIT_ABORTED 1
239239

240+
// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726
241+
#define GGML_ROPE_TYPE_NORMAL 0
240242
#define GGML_ROPE_TYPE_NEOX 2
241243
#define GGML_ROPE_TYPE_MROPE 8
242244
#define GGML_ROPE_TYPE_VISION 24
@@ -592,6 +594,7 @@ extern "C" {
592594
GGML_UNARY_OP_HARDSIGMOID,
593595
GGML_UNARY_OP_EXP,
594596
GGML_UNARY_OP_GELU_ERF,
597+
GGML_UNARY_OP_XIELU,
595598

596599
GGML_UNARY_OP_COUNT,
597600
};
@@ -1173,6 +1176,18 @@ extern "C" {
11731176
struct ggml_context * ctx,
11741177
struct ggml_tensor * a);
11751178

1179+
// xIELU activation function
1180+
// x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)
1181+
// where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions
1182+
// that constrain the positive and negative source alpha values respectively
1183+
GGML_API struct ggml_tensor * ggml_xielu(
1184+
struct ggml_context * ctx,
1185+
struct ggml_tensor * a,
1186+
float alpha_n,
1187+
float alpha_p,
1188+
float beta,
1189+
float eps);
1190+
11761191
// gated linear unit ops
11771192
// A: n columns, r rows,
11781193
// result is n / 2 columns, r rows,
@@ -1640,6 +1655,13 @@ extern "C" {
16401655
float scale,
16411656
float max_bias);
16421657

1658+
GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(
1659+
struct ggml_context * ctx,
1660+
struct ggml_tensor * a,
1661+
struct ggml_tensor * mask,
1662+
float scale,
1663+
float max_bias);
1664+
16431665
GGML_API void ggml_soft_max_add_sinks(
16441666
struct ggml_tensor * a,
16451667
struct ggml_tensor * sinks);

‎ggml/src/ggml-cpu/ggml-cpu.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3012,6 +3012,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
30123012
case GGML_UNARY_OP_GELU_ERF:
30133013
case GGML_UNARY_OP_GELU_QUICK:
30143014
case GGML_UNARY_OP_SILU:
3015+
case GGML_UNARY_OP_XIELU:
30153016
{
30163017
n_tasks = n_threads;
30173018
} break;

0 commit comments

Comments
 (0)