Skip to content

Commit 86a442a

Browse files
authored
Merge pull request #3 from ubergarm/ug/port-sweep-bench
Ug/port sweep bench
2 parents 825fe5f + 92be986 commit 86a442a

File tree

5 files changed

+453
-0
lines changed

5 files changed

+453
-0
lines changed

examples/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ else()
3131
add_subdirectory(simple-chat)
3232
add_subdirectory(speculative)
3333
add_subdirectory(speculative-simple)
34+
add_subdirectory(sweep-bench)
3435
add_subdirectory(gen-docs)
3536
add_subdirectory(training)
3637
add_subdirectory(diffusion)
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(TARGET llama-sweep-bench)
2+
add_executable(${TARGET} sweep-bench.cpp)
3+
install(TARGETS ${TARGET} RUNTIME)
4+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
5+
target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/sweep-bench/README.md

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# ik_llama.cpp/example/sweep-bench
2+
3+
Benchmark the prompt processing and token generation performance of `ik_llama.cpp`
4+
by doing a sweep over a whole context size and gathering performance metrics
5+
in each ubatch-sized window. Only a single token sequence is used.
6+
7+
The benchmark steps are:
8+
9+
for each ubatch-sized window in context:
10+
11+
1. generate ubatch/4 tokens (not the whole window to save some time)
12+
2. measure generation performance
13+
3. remove generated tokens from KV cache
14+
4. prepare a ubatch-sized batch of random tokens
15+
4. process prepated batch
16+
5. measure prompt processing performance
17+
18+
The purpose of the benchmark is to visualize how the performance changes with
19+
the context size without averaging the metrics values over the whole context.
20+
21+
## Usage
22+
23+
./llama-sweep-bench -c 8704 -ub 512 -m models/Meta-Llama-3.2-3B-Instruct-Q8_0.gguf
24+
25+
## Sample results
26+
27+
- `PP` - prompt tokens per ubatch
28+
- `TG` - generated tokens per ubatch
29+
- `N_KV` - current KV cache size
30+
- `T_PP` - prompt processing time (i.e. time to first token)
31+
- `S_PP` - prompt processing speed (`(B*PP)/T_PP` or `PP/T_PP`)
32+
- `T_TG` - time to generate all batches
33+
- `S_TG` - text generation speed (`(B*TG)/T_TG`)
34+
35+
| PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s |
36+
|-------|--------|--------|----------|----------|----------|----------|
37+
| 512 | 128 | 0 | 1.100 | 465.51 | 2.311 | 55.38 |
38+
| 512 | 128 | 512 | 1.183 | 432.97 | 1.895 | 67.55 |
39+
| 512 | 128 | 1024 | 1.305 | 392.38 | 2.071 | 61.81 |
40+
| 512 | 128 | 1536 | 1.279 | 400.42 | 2.164 | 59.14 |
41+
| 512 | 128 | 2048 | 1.571 | 325.96 | 2.280 | 56.14 |
42+
| 512 | 128 | 2560 | 1.431 | 357.87 | 2.418 | 52.94 |
43+
| 512 | 128 | 3072 | 1.515 | 337.93 | 2.566 | 49.88 |
44+
| 512 | 128 | 3584 | 1.588 | 322.34 | 2.722 | 47.03 |
45+
| 512 | 128 | 4096 | 1.675 | 305.70 | 2.864 | 44.69 |
46+
| 512 | 128 | 4608 | 1.769 | 289.50 | 2.999 | 42.68 |
47+
| 512 | 128 | 5120 | 1.845 | 277.48 | 3.102 | 41.26 |
48+
| 512 | 128 | 5632 | 1.893 | 270.46 | 3.219 | 39.76 |
49+
| 512 | 128 | 6144 | 1.953 | 262.20 | 3.348 | 38.23 |
50+
| 512 | 128 | 6656 | 2.018 | 253.71 | 3.474 | 36.84 |
51+
| 512 | 128 | 7168 | 2.078 | 246.34 | 3.589 | 35.66 |
52+
| 512 | 128 | 7680 | 2.140 | 239.22 | 3.717 | 34.43 |
53+
| 512 | 128 | 8192 | 2.196 | 233.15 | 3.854 | 33.21 |
54+
55+
### JSONL output
56+
57+
Pass `--output-format jsonl` to output JSONL instead of Markdown, á la
58+
59+
```json lines
60+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 0, "t_pp": 1.093814, "speed_pp": 468.086884, "t_tg": 1.780312, "speed_tg": 71.897514 }
61+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 512, "t_pp": 1.169302, "speed_pp": 437.868073, "t_tg": 1.897474, "speed_tg": 67.458099 }
62+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 1024, "t_pp": 1.183700, "speed_pp": 432.542053, "t_tg": 2.059179, "speed_tg": 62.160694 }
63+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 1536, "t_pp": 1.428625, "speed_pp": 358.386566, "t_tg": 2.160639, "speed_tg": 59.241734 }
64+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 2048, "t_pp": 1.360647, "speed_pp": 376.291595, "t_tg": 2.274003, "speed_tg": 56.288403 }
65+
```
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import pandas as pd
2+
import matplotlib.pyplot as plt
3+
import numpy as np
4+
import argparse
5+
6+
parser = argparse.ArgumentParser()
7+
parser.add_argument('file', nargs='+')
8+
args = parser.parse_args()
9+
10+
df = None
11+
12+
#for jsonl_file in args.file:
13+
# # Read JSONL file into DataFrame
14+
# df_part = pd.read_json(jsonl_file, lines=True)
15+
# df_part['label'] = jsonl_file
16+
# if df is None:
17+
# df = df_part
18+
# else:
19+
# df = pd.concat([df, df_part])
20+
#
21+
22+
23+
24+
for md_file in args.file:
25+
# Read markdown table file into DataFrame
26+
df_part = pd.read_csv(md_file, sep=r'\s*\|\s*', engine='python',
27+
header=0, skiprows=[1])
28+
29+
# Clean up columns (remove empty columns from markdown formatting)
30+
df_part = df_part.iloc[:, 1:-1]
31+
df_part.columns = [col.strip() for col in df_part.columns]
32+
33+
# Rename columns to match expected names
34+
df_part = df_part.rename(columns={
35+
'N_KV': 'n_kv',
36+
'S_PP t/s': 'speed_pp',
37+
'S_TG t/s': 'speed_tg'
38+
})
39+
40+
# Convert to numeric types
41+
df_part['n_kv'] = pd.to_numeric(df_part['n_kv'])
42+
df_part['speed_pp'] = pd.to_numeric(df_part['speed_pp'])
43+
df_part['speed_tg'] = pd.to_numeric(df_part['speed_tg'])
44+
45+
# Add label and append to main DataFrame
46+
df_part['label'] = md_file
47+
df = pd.concat([df, df_part]) if df is not None else df_part
48+
49+
# Group by label and n_kv, calculate mean and std for both speed metrics
50+
df_grouped = df.groupby(['label', 'n_kv']).agg({
51+
'speed_pp': ['mean', 'std'],
52+
'speed_tg': ['mean', 'std']
53+
}).reset_index()
54+
55+
# Flatten multi-index columns
56+
df_grouped.columns = ['label', 'n_kv', 'speed_pp_mean', 'speed_pp_std',
57+
'speed_tg_mean', 'speed_tg_std']
58+
59+
# Replace NaN with 0 (std for a single sample is NaN)
60+
df_grouped['speed_pp_std'] = df_grouped['speed_pp_std'].fillna(0)
61+
df_grouped['speed_tg_std'] = df_grouped['speed_tg_std'].fillna(0)
62+
63+
# Prepare ticks values for X axis (prune for readability)
64+
x_ticks = df['n_kv'].unique()
65+
while len(x_ticks) > 16:
66+
x_ticks = x_ticks[::2]
67+
68+
# Get unique labels and color map
69+
labels = df_grouped['label'].unique()
70+
colors = plt.cm.rainbow(np.linspace(0, 1, len(labels)))
71+
72+
# Create prompt processing plot
73+
plt.figure(figsize=(10, 6))
74+
ax1 = plt.gca()
75+
plt.grid()
76+
ax1.set_xticks(x_ticks)
77+
78+
# Plot each label's data
79+
for label, color in zip(labels, colors):
80+
label_data = df_grouped[df_grouped['label'] == label].sort_values('n_kv')
81+
pp = ax1.errorbar(label_data['n_kv'], label_data['speed_pp_mean'],
82+
yerr=label_data['speed_pp_std'], color=color,
83+
marker='o', linestyle='-', label=label)
84+
85+
# Add labels and title
86+
ax1.set_xlabel('Context Length (tokens)')
87+
ax1.set_ylabel('Prompt Processing Rate (t/s)')
88+
plt.title('Prompt Processing Performance Comparison')
89+
ax1.legend(loc='upper right')
90+
91+
# Adjust layout and save
92+
plt.tight_layout()
93+
plt.savefig('performance_comparison_pp.png', bbox_inches='tight')
94+
plt.close()
95+
96+
# Create token generation plot
97+
plt.figure(figsize=(10, 6))
98+
ax1 = plt.gca()
99+
plt.grid()
100+
ax1.set_xticks(x_ticks)
101+
102+
# Plot each model's data
103+
for label, color in zip(labels, colors):
104+
label_data = df_grouped[df_grouped['label'] == label].sort_values('n_kv')
105+
tg = ax1.errorbar(label_data['n_kv'], label_data['speed_tg_mean'],
106+
yerr=label_data['speed_tg_std'], color=color,
107+
marker='s', linestyle='-', label=label)
108+
109+
# Add labels and title
110+
ax1.set_xlabel('Context Length (n_kv)')
111+
ax1.set_ylabel('Token Generation Rate (t/s)')
112+
plt.title('Token Generation Performance Comparison')
113+
ax1.legend(loc='upper right')
114+
115+
# Adjust layout and save
116+
plt.tight_layout()
117+
plt.savefig('performance_comparison_tg.png', bbox_inches='tight')
118+
plt.close()

0 commit comments

Comments
 (0)