Skip to content

Commit 1550333

Browse files
author
chibu
committed
add config examples json
1 parent 81f5199 commit 1550333

13 files changed

+158
-1
lines changed

‎src/automation/configs.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
#DEFAULT_OUTPUT_URI = "gs://neuralmagic-clearml"
44
DEFAULT_OUTPUT_URI = "http://10.128.20.60:8081"
55
DEFAULT_RESEARCH_BRANCH = "main"
6-
DEFAULT_GUIDELLM_SCENARIO = "chat"
6+
#DEFAULT_GUIDELLM_SCENARIO = "chat"
7+
DEFAULT_GUIDELLM_SCENARIO = "benchmarking_summarization"

‎src/automation/standards/benchmarking/benchmarking_128k.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 128000,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 128000,
8+
"output_tokens": 2048,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 2048
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_16k.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 16000,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 16000,
8+
"output_tokens": 2048,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 2048
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_32k.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 32000,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 32000,
8+
"output_tokens": 2048,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 2048
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_64k.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 64000,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 64000,
8+
"output_tokens": 2048,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 2048
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_chat.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 512,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 512,
8+
"output_tokens": 256,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 256
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_code_completion.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 256,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 256,
8+
"output_tokens": 1024,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 1024
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_code_fixing.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 1024,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 1024,
8+
"output_tokens": 1024,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 1024
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_docstring_generation.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 768,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 768,
8+
"output_tokens": 128,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 128
12+
}
13+
}

‎src/automation/standards/benchmarking/benchmarking_instruction.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"rate_type": "sweep",
3+
"data": {
4+
"prompt_tokens": 256,
5+
"prompt_tokens_stdev": 128,
6+
"prompt_tokens_min": 1,
7+
"prompt_tokens_max": 256,
8+
"output_tokens": 128,
9+
"output_tokens_stdev": 64,
10+
"output_tokens_min": 1,
11+
"output_tokens_max": 128
12+
}
13+
}

0 commit comments

Comments
 (0)