Skip to content

Commit 7b3da05

Browse files
ubergarmubergarm
committed
Use ikawrakow's adaptation of sweep-bench
This patch is not my own work but taken from this gzip: ikawrakow/ik_llama.cpp#354 Thanks ikawrakow and saood06 for this!
1 parent 16588ae commit 7b3da05

File tree

2 files changed

+101
-37
lines changed

2 files changed

+101
-37
lines changed

‎examples/CMakeLists.txt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ else()
3232
add_subdirectory(speculative)
3333
add_subdirectory(speculative-simple)
3434
add_subdirectory(sweep-bench)
35-
add_subdirectory(tokenize)
36-
add_subdirectory(tts)
3735
add_subdirectory(gen-docs)
3836
add_subdirectory(training)
3937
add_subdirectory(diffusion)

‎examples/sweep-bench/sweep-bench.cpp‎

Lines changed: 101 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1+
#include "common.h"
12
#include "arg.h"
2-
#include "log.h"
33
#include "ggml.h"
44
#include "llama.h"
55
#include "common.h"
6-
#include "../src/llama-vocab.h"
6+
//#include "llama-vocab.h"
7+
#include "log.h"
78

89
#ifdef _WIN32
910
#define WIN32_LEAN_AND_MEAN
@@ -20,49 +21,96 @@
2021
#include <vector>
2122

2223
static void print_usage(int, char ** argv) {
23-
LOG_INF("\nexample usage:\n");
24-
LOG_INF("\n %s -m model.gguf -c 8192 -b 2048 -ub 512\n", argv[0]);
25-
LOG_INF("\n");
24+
LOG("\nexample usage:\n");
25+
LOG("\n %s -m model.gguf -c 8192 -b 2048 -ub 512\n", argv[0]);
26+
LOG("\n");
2627
}
2728

2829
int main(int argc, char ** argv) {
29-
common_params params;
3030

31-
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
31+
std::vector<char*> args;
32+
args.reserve(argc);
33+
args.push_back(argv[0]);
34+
35+
bool sweep_bench_output_jsonl = false;
36+
37+
for (int i = 1; i < argc; ++i) {
38+
std::string arg{argv[1]};
39+
if (arg == "--output-format") {
40+
bool invalid_arg = false;
41+
if (i < argc-1) {
42+
arg = argv[++i];
43+
if (arg == "jsonl") sweep_bench_output_jsonl = true;
44+
else if (arg == "md") sweep_bench_output_jsonl = false;
45+
else invalid_arg = true;
46+
} else {
47+
invalid_arg = true;
48+
}
49+
if (invalid_arg) {
50+
LOG("Invalid arg"); return 1;
51+
}
52+
} else {
53+
args.push_back(argv[i]);
54+
}
55+
}
56+
57+
common_params params;
58+
if (!common_params_parse(args.size(), args.data(), params, LLAMA_EXAMPLE_BENCH, print_usage)) {
3259
return 1;
3360
}
3461

3562
common_init();
3663

64+
//gpt_params params;
65+
66+
//if (!gpt_params_parse(argc, argv, params)) {
67+
// print_usage(argc, argv);
68+
// return 1;
69+
//}
70+
3771
// init LLM
72+
3873
llama_backend_init();
3974
llama_numa_init(params.numa);
4075

4176
// initialize the model
42-
common_init_result llama_init = common_init_from_params(params);
4377

44-
llama_model * model = llama_init.model.get();
45-
llama_context * ctx = llama_init.context.get();
78+
//llama_model_params model_params = llama_model_params_from_gpt_params(params);
79+
llama_model_params model_params = common_model_params_to_llama(params);
80+
81+
//llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params);
82+
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);
4683

47-
if (model == nullptr || ctx == nullptr) {
48-
LOG_ERR("%s : failed to init\n", __func__);
84+
if (model == NULL) {
85+
fprintf(stderr , "%s: error: unable to load model\n" , __func__);
4986
return 1;
5087
}
5188

52-
// print system information
53-
{
54-
LOG_INF("\n");
55-
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
56-
LOG_INF("\n");
89+
//llama_context_params ctx_params = llama_context_params_from_gpt_params(params);
90+
llama_context_params ctx_params = common_context_params_to_llama(params);
91+
92+
//llama_context * ctx = llama_new_context_with_model(model, ctx_params);
93+
llama_context * ctx = llama_init_from_model(model, ctx_params);
94+
95+
if (ctx == NULL) {
96+
fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);
97+
return 1;
5798
}
5899

59100
const unsigned int n_kv_max = llama_n_ctx(ctx);
60101

61-
const llama_vocab * vocab = llama_model_get_vocab(model);
62-
llama_token bos = vocab->token_bos();
63-
const unsigned int n_vocab = llama_vocab_n_tokens(vocab);
64102

65-
// decode in batches of n_batch tokens
103+
auto vocab = llama_model_get_vocab(model);
104+
auto n_vocab = llama_vocab_n_tokens(vocab);
105+
auto bos = llama_vocab_bos(vocab);
106+
107+
//const llama_vocab * vocab = llama_get_vocab(ctx);
108+
//llama_token bos = llama_token_bos_impl(*vocab);
109+
//llama_token eos = llama_token_eos_impl(*vocab);
110+
111+
//const unsigned int n_vocab = llama_n_vocab(model);
112+
113+
// decode in batches of ctx_params.n_batch tokens
66114
auto decode_helper = [](llama_context * ctx, llama_batch & batch, int32_t n_batch) {
67115
for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {
68116
const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));
@@ -92,43 +140,45 @@ int main(int argc, char ** argv) {
92140
const unsigned int pp = params.n_ubatch;
93141
const unsigned int tg = params.n_ubatch / 4;
94142

95-
const unsigned int n_threads = params.cpuparams.n_threads;
96-
const unsigned int n_threads_batch = params.cpuparams_batch.n_threads;
97-
const int32_t n_batch = llama_n_batch(ctx);
98-
99-
LOG_INF("\n");
100-
LOG_INF("%s: n_kv_max = %d, n_batch = %d, n_ubatch = %d, flash_attn = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.n_gpu_layers, n_threads, n_threads_batch);
101-
LOG_INF("\n");
102-
LOG_INF("|%6s | %6s | %6s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s");
103-
LOG_INF("|%6s-|-%6s-|-%6s-|-%8s-|-%8s-|-%8s-|-%8s-|\n", "------", "------", "------", "--------", "--------", "--------", "--------");
143+
if (!sweep_bench_output_jsonl) {
144+
LOG_INF("\n");
145+
LOG_INF("%s: n_kv_max = %d, n_batch = %d, n_ubatch = %d, flash_attn = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch);
146+
LOG_INF("\n");
147+
LOG_INF("|%6s | %6s | %6s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s");
148+
LOG_INF("|%6s-|-%6s-|-%6s-|-%8s-|-%8s-|-%8s-|-%8s-|\n", "------", "------", "------", "--------", "--------", "--------", "--------");
149+
}
104150

105151
llama_batch batch = llama_batch_init(n_kv_max, 0, 1);
106152

107153
// warm up
108154
{
109155
common_batch_add(batch, bos, 0, { 0 }, false);
156+
//llama_batch_add(batch, bos, 0, { 0 }, false);
110157

111-
if (!decode_helper(ctx, batch, n_batch)) {
158+
if (!decode_helper(ctx, batch, ctx_params.n_batch)) {
112159
LOG_INF("%s: llama_decode() failed\n", __func__);
113160
return 1;
114161
}
115162
}
116163

117164
common_batch_clear(batch);
165+
//llama_batch_clear(batch);
118166
llama_kv_self_clear(ctx);
119167

120168
for (unsigned int n_kv = 0; n_kv < n_kv_max; n_kv += params.n_ubatch) {
121169
// clean up KV cache before generation
122-
llama_kv_self_seq_rm(ctx, 0,n_kv, -1);
170+
llama_kv_self_seq_rm(ctx, 0, n_kv, -1);
123171

124172
// first measure token generation performance at this context size
125173
const auto t_tg_start = ggml_time_us();
126174

127175
for (unsigned int i = 0; i < tg; ++i) {
128176
common_batch_clear(batch);
129177
common_batch_add(batch, std::rand() % n_vocab, n_kv + i, { 0 }, true);
178+
//llama_batch_clear(batch);
179+
//llama_batch_add(batch, std::rand() % n_vocab, n_kv + i, { 0 }, true);
130180

131-
if (!decode_helper(ctx, batch, n_batch)) {
181+
if (!decode_helper(ctx, batch, ctx_params.n_batch)) {
132182
LOG_INF("%s: llama_decode() failed\n", __func__);
133183
return 1;
134184
}
@@ -141,16 +191,18 @@ int main(int argc, char ** argv) {
141191

142192
// prepare batch of pp size for prompt processing performance measurement
143193
common_batch_clear(batch);
194+
//llama_batch_clear(batch);
144195

145196
for (unsigned int i = 0; i < pp; ++i) {
146197
common_batch_add(batch, std::rand() % n_vocab, n_kv + i, { 0 }, false);
198+
//llama_batch_add(batch, std::rand() % n_vocab, n_kv + i, { 0 }, false);
147199
}
148200
batch.logits[batch.n_tokens - 1] = true;
149201

150202
// measure prompt processing performance
151203
const auto t_pp_start = ggml_time_us();
152204

153-
if (!decode_helper(ctx, batch, n_batch)) {
205+
if (!decode_helper(ctx, batch, ctx_params.n_batch)) {
154206
LOG_INF("%s: llama_decode() failed\n", __func__);
155207
return 1;
156208
}
@@ -164,9 +216,23 @@ int main(int argc, char ** argv) {
164216
const float speed_pp = pp / t_pp;
165217
const float speed_tg = tg / t_tg;
166218

167-
LOG_INF("|%6d | %6d | %6d | %8.3f | %8.2f | %8.3f | %8.2f |\n", pp, tg, n_kv, t_pp, speed_pp, t_tg, speed_tg);
219+
if(sweep_bench_output_jsonl) {
220+
LOG_INF(
221+
"{\"n_kv_max\": %d, \"n_batch\": %d, \"n_ubatch\": %d, \"flash_attn\": %d, \"n_gpu_layers\": %d, \"n_threads\": %u, \"n_threads_batch\": %u, "
222+
"\"pp\": %d, \"tg\": %d, \"n_kv\": %d, \"t_pp\": %f, \"speed_pp\": %f, \"t_tg\": %f, \"speed_tg\": %f }\n",
223+
n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch,
224+
pp, tg, n_kv, t_pp, speed_pp, t_tg, speed_tg
225+
);
226+
} else {
227+
LOG_INF("|%6d | %6d | %6d | %8.3f | %8.2f | %8.3f | %8.2f |\n", pp, tg, n_kv, t_pp, speed_pp, t_tg, speed_tg);
228+
}
168229
}
169230

231+
llama_batch_free(batch);
232+
233+
llama_free(ctx);
234+
llama_model_free(model);
235+
170236
llama_backend_free();
171237

172238
return 0;

0 commit comments

Comments
 (0)