Skip to content

Commit fdac154

Browse files
author
prima
committed
Merge remote-tracking branch 'origin/concedo_experimental' into remoteManagement
2 parents 414956b + 7e35954 commit fdac154

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1826
-413
lines changed

common/arg.cpp

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1547,10 +1547,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
15471547
}
15481548
).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_RETRIEVAL}));
15491549
add_opt(common_arg(
1550-
{"-fa", "--flash-attn"},
1551-
string_format("enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled"),
1552-
[](common_params & params) {
1553-
params.flash_attn = true;
1550+
{"-fa", "--flash-attn"}, "FA",
1551+
string_format("set Flash Attention use ('on', 'off', or 'auto', default: '%s')", llama_flash_attn_type_name(params.flash_attn_type)),
1552+
[](common_params & params, const std::string & value) {
1553+
if (value == "on" || value == "enabled") {
1554+
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
1555+
} else if (value == "off" || value == "disabled") {
1556+
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
1557+
} else if (value == "auto") {
1558+
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO;
1559+
} else {
1560+
throw std::runtime_error(string_format("error: unkown value for --flash-attn: '%s'\n", value.c_str()));
1561+
}
15541562
}
15551563
).set_env("LLAMA_ARG_FLASH_ATTN"));
15561564
add_opt(common_arg(
@@ -3461,8 +3469,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
34613469
params.model.hf_repo = "ggml-org/Qwen2.5-Coder-1.5B-Q8_0-GGUF";
34623470
params.model.hf_file = "qwen2.5-coder-1.5b-q8_0.gguf";
34633471
params.port = 8012;
3464-
params.n_gpu_layers = 99;
3465-
params.flash_attn = true;
34663472
params.n_ubatch = 1024;
34673473
params.n_batch = 1024;
34683474
params.n_ctx = 0;
@@ -3477,8 +3483,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
34773483
params.model.hf_repo = "ggml-org/Qwen2.5-Coder-3B-Q8_0-GGUF";
34783484
params.model.hf_file = "qwen2.5-coder-3b-q8_0.gguf";
34793485
params.port = 8012;
3480-
params.n_gpu_layers = 99;
3481-
params.flash_attn = true;
34823486
params.n_ubatch = 1024;
34833487
params.n_batch = 1024;
34843488
params.n_ctx = 0;
@@ -3493,8 +3497,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
34933497
params.model.hf_repo = "ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF";
34943498
params.model.hf_file = "qwen2.5-coder-7b-q8_0.gguf";
34953499
params.port = 8012;
3496-
params.n_gpu_layers = 99;
3497-
params.flash_attn = true;
34983500
params.n_ubatch = 1024;
34993501
params.n_batch = 1024;
35003502
params.n_ctx = 0;
@@ -3510,10 +3512,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
35103512
params.model.hf_file = "qwen2.5-coder-7b-q8_0.gguf";
35113513
params.speculative.model.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF";
35123514
params.speculative.model.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf";
3513-
params.speculative.n_gpu_layers = 99;
35143515
params.port = 8012;
3515-
params.n_gpu_layers = 99;
3516-
params.flash_attn = true;
35173516
params.n_ubatch = 1024;
35183517
params.n_batch = 1024;
35193518
params.n_ctx = 0;
@@ -3529,10 +3528,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
35293528
params.model.hf_file = "qwen2.5-coder-14b-q8_0.gguf";
35303529
params.speculative.model.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF";
35313530
params.speculative.model.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf";
3532-
params.speculative.n_gpu_layers = 99;
35333531
params.port = 8012;
3534-
params.n_gpu_layers = 99;
3535-
params.flash_attn = true;
35363532
params.n_ubatch = 1024;
35373533
params.n_batch = 1024;
35383534
params.n_ctx = 0;
@@ -3547,8 +3543,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
35473543
params.model.hf_repo = "ggml-org/Qwen3-Coder-30B-A3B-Instruct-Q8_0-GGUF";
35483544
params.model.hf_file = "qwen3-coder-30b-a3b-instruct-q8_0.gguf";
35493545
params.port = 8012;
3550-
params.n_gpu_layers = 99;
3551-
params.flash_attn = true;
35523546
params.n_ubatch = 1024;
35533547
params.n_batch = 1024;
35543548
params.n_ctx = 0;

common/chat.cpp

Lines changed: 152 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -622,6 +622,7 @@ const char * common_chat_format_name(common_chat_format format) {
622622
case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B";
623623
case COMMON_CHAT_FORMAT_GRANITE: return "Granite";
624624
case COMMON_CHAT_FORMAT_GPT_OSS: return "GPT-OSS";
625+
case COMMON_CHAT_FORMAT_SEED_OSS: return "Seed-OSS";
625626
default:
626627
throw std::runtime_error("Unknown chat format");
627628
}
@@ -2059,6 +2060,94 @@ static void common_chat_parse_granite(common_chat_msg_parser & builder) {
20592060
}
20602061
}
20612062

2063+
static void common_chat_parse_seed_oss(common_chat_msg_parser & builder) {
2064+
// Parse thinking tags first - this handles the main reasoning content
2065+
builder.try_parse_reasoning("<seed:think>", "</seed:think>");
2066+
2067+
if (!builder.syntax().parse_tool_calls) {
2068+
builder.add_content(builder.consume_rest());
2069+
return;
2070+
}
2071+
2072+
// Parse tool calls - Seed-OSS uses <seed:tool_call> format
2073+
static const common_regex tool_call_begin_regex("<seed:tool_call>");
2074+
static const common_regex tool_call_end_regex("</seed:tool_call>");
2075+
static const common_regex function_regex("<function=([^>]+)>");
2076+
static const common_regex param_regex("<parameter=([^>]+)>");
2077+
2078+
while (auto tool_res = builder.try_find_regex(tool_call_begin_regex)) {
2079+
builder.consume_spaces(); // Consume whitespace after <seed:tool_call>
2080+
2081+
// Look for function call inside tool call, ignore any content before it
2082+
if (auto func_res = builder.try_find_regex(function_regex, std::string::npos, false)) {
2083+
auto function_name = builder.str(func_res->groups[1]);
2084+
2085+
// Parse Seed-OSS parameters <parameter=name>value</parameter>
2086+
json args = json::object();
2087+
// Parse all parameters
2088+
while (auto param_res = builder.try_find_regex(param_regex, std::string::npos, false)) {
2089+
// again, ignore noise around parameters
2090+
auto param_name = builder.str(param_res->groups[1]);
2091+
builder.move_to(param_res->groups[0].end);
2092+
builder.consume_spaces(); // Consume whitespace after parameter
2093+
auto savedPos = builder.pos();
2094+
if (auto param_parse = builder.try_find_literal("</parameter>")) {
2095+
auto param = param_parse->prelude;
2096+
builder.move_to(savedPos);
2097+
try {
2098+
if (auto param_res = builder.try_consume_json()) {
2099+
args[param_name] = param_res->json;
2100+
} else {
2101+
args[param_name] = param;
2102+
}
2103+
} catch (json::exception &) {
2104+
args[param_name] = param;
2105+
}
2106+
} else {
2107+
throw common_chat_msg_partial_exception("Incomplete tool parameter");
2108+
}
2109+
}
2110+
// Look for closing function tag
2111+
auto end_func = builder.try_find_literal("</function>");
2112+
if (end_func) {
2113+
builder.move_to(end_func->groups[0].end);
2114+
builder.consume_spaces(); // Consume whitespace after </function>
2115+
2116+
// Add the tool call with parsed arguments, but only if we REALLY got the literal
2117+
auto eaten_fragment = builder.input().substr(end_func->groups[0].begin, end_func->groups[0].end);
2118+
auto funlen = std::string("</function>").length();
2119+
if (eaten_fragment.length() >= funlen && eaten_fragment.substr(0, funlen) == std::string("</function>")) {
2120+
if (!builder.add_tool_call(function_name, "", args.dump())) {
2121+
throw common_chat_msg_partial_exception("Incomplete tool call");
2122+
}
2123+
} else {
2124+
throw common_chat_msg_partial_exception("Incomplete tool call");
2125+
}
2126+
} else {
2127+
throw common_chat_msg_partial_exception("Incomplete tool call");
2128+
}
2129+
// Look for closing tool call tag
2130+
if (auto end_tool = builder.try_find_regex(tool_call_end_regex, std::string::npos, false)) {
2131+
builder.move_to(end_tool->groups[0].end);
2132+
builder.consume_spaces(); // Consume trailing whitespace after tool call
2133+
} else {
2134+
throw common_chat_msg_partial_exception("Incomplete tool call");
2135+
}
2136+
} else {
2137+
// No function found - don't consume content here, let it be handled at the end
2138+
break;
2139+
}
2140+
}
2141+
2142+
// Consume any remaining whitespace after all tool call processing
2143+
builder.consume_spaces();
2144+
auto remaining = builder.consume_rest();
2145+
// If there's any non-whitespace content remaining, add it as content
2146+
if (!string_strip(remaining).empty()) {
2147+
builder.add_content(remaining);
2148+
}
2149+
}
2150+
20622151
static common_chat_params common_chat_params_init_without_tools(const common_chat_template & tmpl, const struct templates_params & inputs) {
20632152
common_chat_params data;
20642153
data.prompt = apply(tmpl, inputs);
@@ -2075,8 +2164,62 @@ static common_chat_params common_chat_params_init_without_tools(const common_cha
20752164
return data;
20762165
}
20772166

2167+
static common_chat_params common_chat_params_init_seed_oss(
2168+
const common_chat_template & tmpl,
2169+
templates_params & params,
2170+
const common_chat_templates_inputs & inputs)
2171+
{
2172+
common_chat_params data;
2173+
data.prompt = apply(tmpl, params);
2174+
data.format = COMMON_CHAT_FORMAT_SEED_OSS;
2175+
if (string_ends_with(data.prompt, "<seed:think>")) {
2176+
if (!inputs.enable_thinking) {
2177+
data.prompt += "</seed:think>";
2178+
} else {
2179+
data.thinking_forced_open = true;
2180+
}
2181+
}
2182+
2183+
if (params.tools.is_array() && !params.tools.empty()) {
2184+
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
2185+
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
2186+
std::vector<std::string> tool_rules;
2187+
foreach_function(params.tools, [&](const json & tool) {
2188+
const auto & function = tool.at("function");
2189+
std::string name = function.at("name");
2190+
auto parameters = function.at("parameters");
2191+
builder.resolve_refs(parameters);
2192+
2193+
// Create rule for Seed-OSS function call format
2194+
std::string param_rules;
2195+
if (parameters.contains("properties")) {
2196+
for (const auto & [key, value] : parameters.at("properties").items()) {
2197+
param_rules += "\"<parameter=" + key + ">\"" + builder.add_schema(name + "-arg-" + key, value) +
2198+
"\"</parameter>\"";
2199+
}
2200+
}
2201+
2202+
tool_rules.push_back(builder.add_rule(name + "-call",
2203+
"\"<seed:tool_call>\" space \"<function=" + name + ">\" space " +
2204+
param_rules +
2205+
" \"</function>\" space \"</seed:tool_call>\""));
2206+
});
2207+
2208+
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<seed:tool_call>" });
2209+
2210+
data.preserved_tokens = {
2211+
"<seed:think>", "</seed:think>", "<seed:tool_call>", "</seed:tool_call>",
2212+
"<function=", "</function>", "<parameter=", "</parameter>",
2213+
};
2214+
2215+
builder.add_rule("root", string_join(tool_rules, " | "));
2216+
});
2217+
}
2218+
return data;
2219+
}
2220+
20782221
static common_chat_params common_chat_templates_apply_jinja(
2079-
const struct common_chat_templates * tmpls,
2222+
const struct common_chat_templates * tmpls,
20802223
const struct common_chat_templates_inputs & inputs)
20812224
{
20822225
templates_params params;
@@ -2145,6 +2288,11 @@ static common_chat_params common_chat_templates_apply_jinja(
21452288
return common_chat_params_init_gpt_oss(tmpl, params);
21462289
}
21472290

2291+
// Seed-OSS
2292+
if (src.find("<seed:think>") != std::string::npos) {
2293+
return common_chat_params_init_seed_oss(tmpl, params, inputs);
2294+
}
2295+
21482296
// Use generic handler when mixing tools + JSON schema.
21492297
// TODO: support that mix in handlers below.
21502298
if ((params.tools.is_array() && params.json_schema.is_object())) {
@@ -2303,6 +2451,9 @@ static void common_chat_parse(common_chat_msg_parser & builder) {
23032451
case COMMON_CHAT_FORMAT_GPT_OSS:
23042452
common_chat_parse_gpt_oss(builder);
23052453
break;
2454+
case COMMON_CHAT_FORMAT_SEED_OSS:
2455+
common_chat_parse_seed_oss(builder);
2456+
break;
23062457
default:
23072458
throw std::runtime_error(std::string("Unsupported format: ") + common_chat_format_name(builder.syntax().format));
23082459
}

common/chat.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ enum common_chat_format {
111111
COMMON_CHAT_FORMAT_COMMAND_R7B,
112112
COMMON_CHAT_FORMAT_GRANITE,
113113
COMMON_CHAT_FORMAT_GPT_OSS,
114+
COMMON_CHAT_FORMAT_SEED_OSS,
114115

115116
COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
116117
};

common/common.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -909,7 +909,8 @@ struct common_init_result common_init_from_params(common_params & params) {
909909

910910
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
911911
if (model == NULL) {
912-
LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
912+
LOG_ERR("%s: failed to load model '%s', try reducing --n-gpu-layers if you're running out of VRAM\n",
913+
__func__, params.model.path.c_str());
913914
return iparams;
914915
}
915916

@@ -919,7 +920,8 @@ struct common_init_result common_init_from_params(common_params & params) {
919920

920921
llama_context * lctx = llama_init_from_model(model, cparams);
921922
if (lctx == NULL) {
922-
LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str());
923+
LOG_ERR("%s: failed to create context with model '%s', try reducing --n-gpu-layers if you're running out of VRAM\n",
924+
__func__, params.model.path.c_str());
923925
llama_model_free(model);
924926
return iparams;
925927
}
@@ -1165,10 +1167,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
11651167
cparams.yarn_orig_ctx = params.yarn_orig_ctx;
11661168
cparams.pooling_type = params.pooling_type;
11671169
cparams.attention_type = params.attention_type;
1170+
cparams.flash_attn_type = params.flash_attn_type;
11681171
cparams.cb_eval = params.cb_eval;
11691172
cparams.cb_eval_user_data = params.cb_eval_user_data;
11701173
cparams.offload_kqv = !params.no_kv_offload;
1171-
cparams.flash_attn = params.flash_attn;
11721174
cparams.no_perf = params.no_perf;
11731175
cparams.op_offload = !params.no_op_offload;
11741176
cparams.swa_full = params.swa_full;

common/common.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ struct common_params {
308308
enum llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
309309
enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings
310310
enum llama_attention_type attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; // attention type for embeddings
311+
enum llama_flash_attn_type flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; // whether to use Flash Attention
311312

312313
struct common_params_sampling sampling;
313314
struct common_params_speculative speculative;
@@ -371,7 +372,6 @@ struct common_params {
371372
bool multiline_input = false; // reverse the usage of `\`
372373
bool simple_io = false; // improves compatibility with subprocesses and limited consoles
373374
bool cont_batching = true; // insert new sequences for decoding on-the-fly
374-
bool flash_attn = false; // flash attention
375375
bool no_perf = false; // disable performance metrics
376376
bool ctx_shift = false; // context shift on infinite text generation
377377
bool swa_full = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)

0 commit comments

Comments
 (0)