larger warmup batch

LostRuins · LostRuins · commit b3143384b4f4 · 2025-04-05T10:57:04.000+08:00
diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp
@@ -2297,8 +2297,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
             }
         }
 
-        //determine mem per token
-        std::vector<int> tmp = {1, 2, 3, 4};
+        //warmup at least 33 tokens to trigger batch
+        std::vector<int> tmp;
+        for (int i = 1; i <= 33; ++i) {
+            tmp.push_back(i);
+        }
         llama_kv_self_clear(llama_ctx_v4);
         auto er = llama_decode(llama_ctx_v4, llama_batch_get_one(tmp.data(), tmp.size()));
         if(er!=0)

Original file line number	Diff line number	Diff line change
`@@ -2297,8 +2297,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in`
`2297`	`2297`	`}`
`2298`	`2298`	`}`
`2299`	`2299`
`2300`		`- //determine mem per token`
`2301`		`- std::vector<int> tmp = {1, 2, 3, 4};`
	`2300`	`+ //warmup at least 33 tokens to trigger batch`
	`2301`	`+ std::vector<int> tmp;`
	`2302`	`+ for (int i = 1; i <= 33; ++i) {`
	`2303`	`+ tmp.push_back(i);`
	`2304`	`+ }`
`2302`	`2305`	`llama_kv_self_clear(llama_ctx_v4);`
`2303`	`2306`	`auto er = llama_decode(llama_ctx_v4, llama_batch_get_one(tmp.data(), tmp.size()));`
`2304`	`2307`	`if(er!=0)`