move various places to batch.add_text

Author: ngxson

common/speculative.cpp

@@ -149,8 +149,6 @@ llama_tokens common_speculative_gen_draft(
 
     const int i_start = std::max<int>(0, (int) prompt_tgt.size() - n_ctx);
 
-    const llama_seq_id seq_id = 0;
-
     // reuse as much as possible from the old draft context
     // ideally, the draft context should be as big as the target context and we will always reuse the entire prompt
     for (int i = 0; i < (int) prompt.size(); ++i) {
@@ -210,7 +208,7 @@ llama_tokens common_speculative_gen_draft(
 
     for (size_t i = i_start + reuse_n; i < prompt_tgt.size(); ++i) {
         //LOG_DBG("i = %d, i_start = %d, reuse_n = %d, i - i_start = %d, id = %6d\n", i, i_start, reuse_n, i - i_start, prompt_tgt[i]);
-        llama_batch_ext_add_text(batch.get(), prompt_tgt[i], i - i_start, &seq_id, 1, false);
+        batch.add_text(prompt_tgt[i], i - i_start, 0, false);
 
         prompt.push_back(prompt_tgt[i]);
     }
@@ -227,7 +225,7 @@ llama_tokens common_speculative_gen_draft(
     LOG_DBG("%s: n_past = %d\n", __func__, n_past);
 
     llama_batch_ext_clear(batch.get());
-    llama_batch_ext_add_text(batch.get(), id_last, n_past, &seq_id, 1, true);
+    batch.add_text(id_last, n_past, 0, true);
 
     prompt.push_back(id_last);
 
@@ -266,7 +264,7 @@ llama_tokens common_speculative_gen_draft(
             break;
         }
 
-        llama_batch_ext_add_text(batch.get(), id, n_past + i + 1, &seq_id, 1, true);
+        batch.add_text( id, n_past + i + 1, 0, true);
 
         // evaluate the drafted tokens on the draft model
         llama_decode_ext(ctx, batch.get());

examples/gritlm/gritlm.cpp

@@ -1,6 +1,7 @@
 #include "arg.h"
 #include "common.h"
 #include "llama.h"
+#include "llama-cpp.h"
 
 #include <string>
 #include <vector>
@@ -13,10 +14,10 @@ static std::vector<std::vector<float>> encode(llama_context * ctx, const std::ve
     const llama_model * model = llama_get_model(ctx);
     const llama_vocab * vocab = llama_model_get_vocab(model);
 
-    llama_batch_ext * batch = llama_batch_ext_init(llama_n_batch(ctx), 1);
+    llama_batch_ext_ptr batch(llama_batch_ext_init(llama_n_batch(ctx), 1));
 
     for (uint64_t i = 0; i < sentences.size(); i++) {
-        llama_batch_ext_clear(batch);
+        llama_batch_ext_clear(batch.get());
 
         const std::string input_string = instruction + sentences[i];
 
@@ -41,8 +42,7 @@ static std::vector<std::vector<float>> encode(llama_context * ctx, const std::ve
 
         // add input to batch (this increments n_tokens)
         for (int32_t j = 0; j < n_toks; j++) {
-            const llama_seq_id seq_id = 0;
-            llama_batch_ext_add_text(batch, inputs[j], j, &seq_id, 1 , j >= n_inst);
+            batch.add_text(inputs[j], j, 0, j >= n_inst);
         }
 
         // clear previous kv_cache values (irrelevant for embeddings)
@@ -51,7 +51,7 @@ static std::vector<std::vector<float>> encode(llama_context * ctx, const std::ve
         llama_set_causal_attn(ctx, false);
 
         // run model
-        llama_decode_ext(ctx, batch);
+        llama_decode_ext(ctx, batch.get());
 
         // get embedding dimensions
         uint64_t n_embd = llama_model_n_embd(model);
@@ -90,8 +90,6 @@ static std::vector<std::vector<float>> encode(llama_context * ctx, const std::ve
 #endif
     }
 
-    llama_batch_ext_free(batch);
-
     return result;
 }
 
@@ -107,26 +105,25 @@ static std::string generate(llama_context * ctx, llama_sampler * smpl, const std
     llama_set_embeddings(ctx, false);
     llama_set_causal_attn(ctx, true);
 
-    llama_batch_ext * bat = llama_batch_ext_init(llama_n_batch(ctx), 1);
+    llama_batch_ext_ptr batch(llama_batch_ext_init(llama_n_batch(ctx), 1));
 
     std::vector<llama_token> inputs = common_tokenize(vocab, prompt, false, true);
     int32_t i_current_token = 0;
 
     while (true) {
-        llama_batch_ext_clear(bat);
+        llama_batch_ext_clear(batch.get());
         {
             const int32_t n_inputs = inputs.size();
 
             for (int32_t i = 0; i < n_inputs; i++) {
-                const llama_seq_id seq_id = 0;
-                llama_batch_ext_add_text(bat, inputs[i], i_current_token++, &seq_id, 1, i == n_inputs - 1);
+                batch.add_text(inputs[i], i_current_token++, 0, i == n_inputs - 1);
             }
         }
         inputs.clear();
 
-        llama_decode_ext(ctx, bat);
+        llama_decode_ext(ctx, batch.get());
 
-        llama_token token = llama_sampler_sample(smpl, ctx, llama_batch_ext_get_n_tokens(bat) - 1);
+        llama_token token = llama_sampler_sample(smpl, ctx, llama_batch_ext_get_n_tokens(batch.get()) - 1);
 
         if (token == eos_token) {
             break;
@@ -147,8 +144,6 @@ static std::string generate(llama_context * ctx, llama_sampler * smpl, const std
         std::printf("\n");
     }
 
-    llama_batch_ext_free(bat);
-
     return result;
 }
 

examples/llava/gemma3-cli.cpp

@@ -92,8 +92,7 @@ static int eval_text(gemma3_context & ctx, std::string input, bool logits_last =
     llama_tokens tokens = common_tokenize(ctx.lctx, input, false, true);
     llama_batch_ext_clear(ctx.batch.get());
     for (llama_token & t : tokens) {
-        llama_seq_id seq_id = 0;
-        llama_batch_ext_add_text(ctx.batch.get(), t, ctx.n_past++, &seq_id, 1, false);
+        ctx.batch.add_text(t, ctx.n_past++, 0, false);
     }
     if (logits_last) {
         llama_batch_ext_set_output_last(ctx.batch.get());
@@ -180,8 +179,7 @@ static int generate_response(gemma3_context & ctx, common_sampler * smpl, int n_
 
         // eval the token
         llama_batch_ext_clear(ctx.batch.get());
-        llama_seq_id seq_id = 0;
-        llama_batch_ext_add_text(ctx.batch.get(), token_id, ctx.n_past++, &seq_id, 1, true);
+        ctx.batch.add_text(token_id, ctx.n_past++, 0, true);
         if (llama_decode_ext(ctx.lctx, ctx.batch.get())) {
             LOG_ERR("failed to decode token\n");
             return 1;

examples/llava/qwen2vl-cli.cpp

@@ -101,8 +101,7 @@ static bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_toke
         llama_batch_ext_ptr batch(llama_batch_ext_init(n_eval, 1));
         for (int j = 0; j < n_eval; j++) {
             llama_token token = tokens[i + j];
-            llama_seq_id seq_id = 0;
-            llama_batch_ext_add_text(batch.get(), token, pos[j], &seq_id, 1, false);
+            batch.add_text(token, pos[j], 0, false);
         }
         llama_batch_ext_set_output_last(batch.get());
 

examples/lookup/lookup.cpp

@@ -5,6 +5,7 @@
 #include "sampling.h"
 #include "log.h"
 #include "llama.h"
+#include "llama-cpp.h"
 
 #include <cstdint>
 #include <cstdio>
@@ -110,7 +111,7 @@ int main(int argc, char ** argv){
 
     std::vector<llama_token> draft;
 
-    llama_batch_ext * batch_tgt = llama_batch_ext_init(params.n_ctx, 1);
+    llama_batch_ext_ptr batch_tgt(llama_batch_ext_init(params.n_ctx, 1));
 
     // debug
     struct llama_kv_cache_view kvc_view = llama_kv_cache_view_init(ctx, 1);
@@ -196,9 +197,8 @@ int main(int argc, char ** argv){
         // clean the cache of draft tokens that weren't accepted
         llama_kv_self_seq_rm(ctx, 0, n_past, -1);
 
-        const llama_seq_id seq_id = 0;
-        llama_batch_ext_clear(batch_tgt);
-        llama_batch_ext_add_text(batch_tgt, draft[0], n_past, &seq_id, 1, true);
+        llama_batch_ext_clear(batch_tgt.get());
+        batch_tgt.add_text(draft[0], n_past, 0, true);
 
         // Draft already contains a single token sampled from the model:
         GGML_ASSERT(draft.size() == 1);
@@ -208,13 +208,13 @@ int main(int argc, char ** argv){
         common_ngram_cache_draft(inp, draft, n_draft, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, ngram_cache_context, ngram_cache_dynamic, ngram_cache_static);
 
         for (size_t i = 1; i < draft.size(); ++i) {
-            llama_batch_ext_add_text(batch_tgt, draft[i], n_past + i, &seq_id, 1, true);
+            batch_tgt.add_text(draft[i], n_past + i, 0, true);
         }
 
         t_draft_us += ggml_time_us() - t_start_draft_us;
         n_drafted += draft.size() - 1;
 
-        llama_decode_ext(ctx, batch_tgt);
+        llama_decode_ext(ctx, batch_tgt.get());
         ++n_past;
 
         draft.erase(draft.begin());
@@ -246,8 +246,6 @@ int main(int argc, char ** argv){
 
     common_sampler_free(smpl);
 
-    llama_batch_ext_free(batch_tgt);
-
     llama_backend_free();
 
     LOG("\n\n");

examples/parallel/parallel.cpp

@@ -6,6 +6,7 @@
 #include "sampling.h"
 #include "log.h"
 #include "llama.h"
+#include "llama-cpp.h"
 
 #include <cmath>
 #include <cstdio>
@@ -174,7 +175,7 @@ int main(int argc, char ** argv) {
 
     // the max batch size is as large as the context to handle cases where we get very long input prompt from multiple
     // users. regardless of the size, the main loop will chunk the batch into a maximum of params.n_batch tokens at a time
-    llama_batch_ext * batch = llama_batch_ext_init(n_ctx, 1);
+    llama_batch_ext_ptr batch(llama_batch_ext_init(n_ctx, 1));
 
     int32_t n_total_prompt = 0;
     int32_t n_total_gen    = 0;
@@ -192,11 +193,10 @@ int main(int argc, char ** argv) {
         LOG_INF("%s: Evaluating the system prompt ...\n", __func__);
 
         for (int32_t i = 0; i < n_tokens_system; ++i) {
-            llama_seq_id seq_id = 0;
-            llama_batch_ext_add_text(batch, tokens_system[i], i, &seq_id, 1, false);
+            batch.add_text(tokens_system[i], i, 0, false);
         }
 
-        if (llama_decode_ext(ctx, batch) != 0) {
+        if (llama_decode_ext(ctx, batch.get()) != 0) {
             LOG_ERR("%s: llama_decode() failed\n", __func__);
             return 1;
         }
@@ -217,23 +217,23 @@ int main(int argc, char ** argv) {
             common_kv_cache_dump_view_seqs(kvc_view, 40);
         }
 
-        llama_batch_ext_clear(batch);
+        llama_batch_ext_clear(batch.get());
 
         // decode any currently ongoing sequences
         for (auto & client : clients) {
             if (client.seq_id == -1) {
                 continue;
             }
 
-            client.i_batch = llama_batch_ext_get_n_tokens(batch);
+            client.i_batch = llama_batch_ext_get_n_tokens(batch.get());
 
             llama_seq_id seq_id = client.id + 1;
-            llama_batch_ext_add_text(batch, client.sampled, n_tokens_system + client.n_prompt + client.n_decoded, &seq_id, 1, true);
+            batch.add_text(client.sampled, n_tokens_system + client.n_prompt + client.n_decoded, seq_id, true);
 
             client.n_decoded += 1;
         }
 
-        if (llama_batch_ext_get_n_tokens(batch) == 0) {
+        if (llama_batch_ext_get_n_tokens(batch.get()) == 0) {
             // all sequences have ended - clear the entire KV cache
             for (int i = 1; i <= n_clients; ++i) {
                 llama_kv_self_seq_rm(ctx, i, -1, -1);
@@ -245,7 +245,7 @@ int main(int argc, char ** argv) {
         }
 
         // insert new sequences for decoding
-        if (cont_batching || llama_batch_ext_get_n_tokens(batch) == 0) {
+        if (cont_batching || llama_batch_ext_get_n_tokens(batch.get()) == 0) {
             for (auto & client : clients) {
                 if (client.seq_id == -1 && g_seq_id < n_seq) {
                     client.seq_id = g_seq_id;
@@ -265,17 +265,17 @@ int main(int argc, char ** argv) {
 
                     for (size_t i = 0; i < tokens_prompt.size(); ++i) {
                         llama_seq_id seq_id = client.id + 1;
-                        llama_batch_ext_add_text(batch, tokens_prompt[i], i + n_tokens_system, &seq_id, 1, false);
+                        batch.add_text(tokens_prompt[i], i + n_tokens_system, seq_id, false);
                     }
 
                     // extract the logits only for the last token
-                    if (llama_batch_ext_get_n_tokens(batch) > 0) {
-                        llama_batch_ext_set_output_last(batch);
+                    if (llama_batch_ext_get_n_tokens(batch.get()) > 0) {
+                        llama_batch_ext_set_output_last(batch.get());
                     }
 
                     client.n_prompt  = tokens_prompt.size();
                     client.n_decoded = 0;
-                    client.i_batch   = llama_batch_ext_get_n_tokens(batch) - 1;
+                    client.i_batch   = llama_batch_ext_get_n_tokens(batch.get()) - 1;
 
                     LOG_INF("\033[31mClient %3d, seq %4d, started decoding ...\033[0m\n", client.id, client.seq_id);
 
@@ -289,14 +289,14 @@ int main(int argc, char ** argv) {
             }
         }
 
-        if (llama_batch_ext_get_n_tokens(batch) == 0) {
+        if (llama_batch_ext_get_n_tokens(batch.get()) == 0) {
             break;
         }
 
         // process in chunks of params.n_batch
         int32_t n_batch = params.n_batch;
 
-        int32_t n_tokens_in_batch = llama_batch_ext_get_n_tokens(batch);
+        int32_t n_tokens_in_batch = llama_batch_ext_get_n_tokens(batch.get());
         for (int32_t i = 0; i < (int32_t) n_tokens_in_batch; i += n_batch) {
             // experiment: process in powers of 2
             //if (i + n_batch > (int32_t) batch.n_tokens && n_batch > 32) {
@@ -307,7 +307,7 @@ int main(int argc, char ** argv) {
 
             const int32_t n_tokens = std::min(n_batch, (int32_t) (n_tokens_in_batch - i));
 
-            llama_batch_ext * batch_view = llama_batch_ext_get_view(batch, i, n_tokens);
+            llama_batch_ext * batch_view = llama_batch_ext_get_view(batch.get(), i, n_tokens);
             const int ret = llama_decode_ext(ctx, batch_view);
             llama_batch_ext_free(batch_view);
             if (ret != 0) {
@@ -413,8 +413,6 @@ int main(int argc, char ** argv) {
     // TODO: print sampling/grammar timings for all clients
     llama_perf_context_print(ctx);
 
-    llama_batch_ext_free(batch);
-
     llama_backend_free();
 
     LOG("\n\n");

examples/passkey/passkey.cpp

@@ -144,8 +144,7 @@ int main(int argc, char ** argv) {
         llama_batch_ext_clear(batch.get());
 
         for (int j = 0; j < n_batch && i + j < n_tokens_all; j++) {
-            llama_seq_id seq_id = 0;
-            llama_batch_ext_add_text(batch.get(), tokens_list[i + j], n_past++, &seq_id, 1, false);
+            batch.add_text(tokens_list[i + j], n_past++, 0, false);
         }
 
         if (i + n_batch >= n_tokens_all) {
@@ -179,8 +178,7 @@ int main(int argc, char ** argv) {
         llama_batch_ext_clear(batch.get());
 
         for (int j = 0; j < n_batch && i + j < n_tokens_all; j++) {
-            llama_seq_id seq_id = 0;
-            llama_batch_ext_add_text(batch.get(), tokens_list[i + j], n_past++, &seq_id, 1, false);
+            batch.add_text(tokens_list[i + j], n_past++, 0, false);
         }
 
         if (i + n_batch >= n_tokens_all) {

examples/save-load-state/save-load-state.cpp

@@ -80,8 +80,7 @@ int main(int argc, char ** argv) {
         result0 += next_token_str;
 
         llama_batch_ext_clear(batch.get());
-        llama_seq_id seq_id = 0;
-        llama_batch_ext_add_text(batch.get(), next_token, 0, &seq_id, 1, true);
+        batch.add_text(next_token, 0, 0, true);
 
         if (llama_decode_ext(ctx, batch.get())) {
             fprintf(stderr, "\n%s : failed to evaluate\n", __func__);
@@ -133,8 +132,7 @@ int main(int argc, char ** argv) {
         result1 += next_token_str;
 
         llama_batch_ext_clear(batch.get());
-        llama_seq_id seq_id = 0;
-        llama_batch_ext_add_text(batch.get(), next_token, 0, &seq_id, 1, true);
+        batch.add_text(next_token, 0, 0, true);
 
         if (llama_decode_ext(ctx2, batch.get())) {
             fprintf(stderr, "\n%s : failed to evaluate\n", __func__);
@@ -215,8 +213,7 @@ int main(int argc, char ** argv) {
         result2 += next_token_str;
 
         llama_batch_ext_clear(batch.get());
-        llama_seq_id seq_id = 1; // seq 1 instead of 0
-        llama_batch_ext_add_text(batch.get(), next_token, 0, &seq_id, 1, true);
+        batch.add_text(next_token, 0, 1, true);
 
         if (llama_decode_ext(ctx3, batch.get())) {
             fprintf(stderr, "\n%s : failed to evaluate\n", __func__);