Skip to content

Commit 552eac2

Browse files
dchourasiadchourasia
committed
Merge remote-tracking branch 'upstream/release'
2 parents 9c2f1e8 + 81ee364 commit 552eac2

File tree

764 files changed

+45789
-15996
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

764 files changed

+45789
-15996
lines changed

‎.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
2+
model_name: "neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8"
3+
tasks:
4+
- name: "gsm8k"
5+
metrics:
6+
- name: "exact_match,strict-match"
7+
value: 0.356
8+
- name: "exact_match,flexible-extract"
9+
value: 0.358
10+
limit: 1000
11+
num_fewshot: 5

‎.buildkite/lm-eval-harness/configs/models-small.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Meta-Llama-3-8B-Instruct.yaml
22
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
3-
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
3+
Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
44
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
55
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
66
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml

‎.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,6 @@ while getopts "m:b:l:f:" OPT; do
4141
done
4242

4343
lm_eval --model hf \
44-
--model_args pretrained=$MODEL,parallelize=True \
45-
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
46-
--batch_size $BATCH_SIZE
44+
--model_args "pretrained=$MODEL,parallelize=True" \
45+
--tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
46+
--batch_size "$BATCH_SIZE"

‎.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
4646
done
4747

4848
lm_eval --model vllm \
49-
--model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50-
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51-
--batch_size $BATCH_SIZE
49+
--model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend=ray,trust_remote_code=true,max_model_len=4096" \
50+
--tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
51+
--batch_size "$BATCH_SIZE"

‎.buildkite/lm-eval-harness/run-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ while getopts "c:t:" OPT; do
3030
done
3131

3232
# Parse list of configs.
33-
IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
33+
IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
3434

3535
for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
3636
do

‎.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656

5757
def read_markdown(file):
5858
if os.path.exists(file):
59-
with open(file, "r") as f:
59+
with open(file) as f:
6060
return f.read() + "\n"
6161
else:
6262
return f"{file} not found.\n"
@@ -75,14 +75,14 @@ def results_to_json(latency, throughput, serving):
7575
# collect results
7676
for test_file in results_folder.glob("*.json"):
7777

78-
with open(test_file, "r") as f:
78+
with open(test_file) as f:
7979
raw_result = json.loads(f.read())
8080

8181
if "serving" in str(test_file):
8282
# this result is generated via `benchmark_serving.py`
8383

8484
# attach the benchmarking command to raw_result
85-
with open(test_file.with_suffix(".commands"), "r") as f:
85+
with open(test_file.with_suffix(".commands")) as f:
8686
command = json.loads(f.read())
8787
raw_result.update(command)
8888

@@ -97,7 +97,7 @@ def results_to_json(latency, throughput, serving):
9797
# this result is generated via `benchmark_latency.py`
9898

9999
# attach the benchmarking command to raw_result
100-
with open(test_file.with_suffix(".commands"), "r") as f:
100+
with open(test_file.with_suffix(".commands")) as f:
101101
command = json.loads(f.read())
102102
raw_result.update(command)
103103

@@ -119,7 +119,7 @@ def results_to_json(latency, throughput, serving):
119119
# this result is generated via `benchmark_throughput.py`
120120

121121
# attach the benchmarking command to raw_result
122-
with open(test_file.with_suffix(".commands"), "r") as f:
122+
with open(test_file.with_suffix(".commands")) as f:
123123
command = json.loads(f.read())
124124
raw_result.update(command)
125125

‎.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,15 +72,15 @@ def main(args):
7272

7373
# collect results
7474
for test_file in results_folder.glob("*_nightly_results.json"):
75-
with open(test_file, "r") as f:
75+
with open(test_file) as f:
7676
results = results + json.loads(f.read())
7777

7878
# generate markdown table
7979
df = pd.DataFrame.from_dict(results)
8080

8181
md_table = tabulate(df, headers='keys', tablefmt='pipe', showindex=False)
8282

83-
with open(args.description, "r") as f:
83+
with open(args.description) as f:
8484
description = f.read()
8585

8686
description = description.format(

‎.buildkite/nightly-benchmarks/scripts/launch-server.sh‎

Lines changed: 25 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -50,58 +50,54 @@ launch_trt_server() {
5050
git clone https://github.com/triton-inference-server/tensorrtllm_backend.git
5151
git lfs install
5252
cd tensorrtllm_backend
53-
git checkout $trt_llm_version
54-
tensorrtllm_backend_dir=$(pwd)
53+
git checkout "$trt_llm_version"
5554
git submodule update --init --recursive
5655

5756
# build trtllm engine
5857
cd /tensorrtllm_backend
59-
cd ./tensorrt_llm/examples/${model_type}
58+
cd "./tensorrt_llm/examples/${model_type}"
6059
python3 convert_checkpoint.py \
61-
--model_dir ${model_path} \
62-
--dtype ${model_dtype} \
63-
--tp_size ${model_tp_size} \
64-
--output_dir ${trt_model_path}
60+
--model_dir "${model_path}" \
61+
--dtype "${model_dtype}" \
62+
--tp_size "${model_tp_size}" \
63+
--output_dir "${trt_model_path}"
6564
trtllm-build \
66-
--checkpoint_dir ${trt_model_path} \
65+
--checkpoint_dir "${trt_model_path}" \
6766
--use_fused_mlp \
6867
--reduce_fusion disable \
6968
--workers 8 \
70-
--gpt_attention_plugin ${model_dtype} \
71-
--gemm_plugin ${model_dtype} \
72-
--tp_size ${model_tp_size} \
73-
--max_batch_size ${max_batch_size} \
74-
--max_input_len ${max_input_len} \
75-
--max_seq_len ${max_seq_len} \
76-
--max_num_tokens ${max_num_tokens} \
77-
--output_dir ${trt_engine_path}
69+
--gpt_attention_plugin "${model_dtype}" \
70+
--gemm_plugin "${model_dtype}" \
71+
--tp_size "${model_tp_size}" \
72+
--max_batch_size "${max_batch_size}" \
73+
--max_input_len "${max_input_len}" \
74+
--max_seq_len "${max_seq_len}" \
75+
--max_num_tokens "${max_num_tokens}" \
76+
--output_dir "${trt_engine_path}"
7877

7978
# handle triton protobuf files and launch triton server
8079
cd /tensorrtllm_backend
8180
mkdir triton_model_repo
8281
cp -r all_models/inflight_batcher_llm/* triton_model_repo/
8382
cd triton_model_repo
8483
rm -rf ./tensorrt_llm/1/*
85-
cp -r ${trt_engine_path}/* ./tensorrt_llm/1
84+
cp -r "${trt_engine_path}"/* ./tensorrt_llm/1
8685
python3 ../tools/fill_template.py -i tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,engine_dir:/tensorrtllm_backend/triton_model_repo/tensorrt_llm/1,decoupled_mode:true,batching_strategy:inflight_fused_batching,batch_scheduler_policy:guaranteed_no_evict,exclude_input_in_output:true,triton_max_batch_size:2048,max_queue_delay_microseconds:0,max_beam_width:1,max_queue_size:2048,enable_kv_cache_reuse:false
87-
python3 ../tools/fill_template.py -i preprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5
88-
python3 ../tools/fill_template.py -i postprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false
89-
python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:$max_batch_size
90-
python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:"False",bls_instance_count:1
86+
python3 ../tools/fill_template.py -i preprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5"
87+
python3 ../tools/fill_template.py -i postprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false"
88+
python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:"$max_batch_size"
89+
python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt "triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:False,bls_instance_count:1"
9190
cd /tensorrtllm_backend
9291
python3 scripts/launch_triton_server.py \
93-
--world_size=${model_tp_size} \
92+
--world_size="${model_tp_size}" \
9493
--model_repo=/tensorrtllm_backend/triton_model_repo &
9594

9695
}
9796

9897
launch_tgi_server() {
9998
model=$(echo "$common_params" | jq -r '.model')
10099
tp=$(echo "$common_params" | jq -r '.tp')
101-
dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
102-
dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
103100
port=$(echo "$common_params" | jq -r '.port')
104-
num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
105101
server_args=$(json2args "$server_params")
106102

107103
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -129,10 +125,7 @@ launch_tgi_server() {
129125
launch_lmdeploy_server() {
130126
model=$(echo "$common_params" | jq -r '.model')
131127
tp=$(echo "$common_params" | jq -r '.tp')
132-
dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
133-
dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
134128
port=$(echo "$common_params" | jq -r '.port')
135-
num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
136129
server_args=$(json2args "$server_params")
137130

138131
server_command="lmdeploy serve api_server $model \
@@ -149,10 +142,7 @@ launch_sglang_server() {
149142

150143
model=$(echo "$common_params" | jq -r '.model')
151144
tp=$(echo "$common_params" | jq -r '.tp')
152-
dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
153-
dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
154145
port=$(echo "$common_params" | jq -r '.port')
155-
num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
156146
server_args=$(json2args "$server_params")
157147

158148
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -185,10 +175,7 @@ launch_vllm_server() {
185175

186176
model=$(echo "$common_params" | jq -r '.model')
187177
tp=$(echo "$common_params" | jq -r '.tp')
188-
dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
189-
dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
190178
port=$(echo "$common_params" | jq -r '.port')
191-
num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
192179
server_args=$(json2args "$server_params")
193180

194181
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -217,19 +204,19 @@ launch_vllm_server() {
217204

218205
main() {
219206

220-
if [[ $CURRENT_LLM_SERVING_ENGINE == "trt" ]]; then
207+
if [[ "$CURRENT_LLM_SERVING_ENGINE" == "trt" ]]; then
221208
launch_trt_server
222209
fi
223210

224-
if [[ $CURRENT_LLM_SERVING_ENGINE == "tgi" ]]; then
211+
if [[ "$CURRENT_LLM_SERVING_ENGINE" == "tgi" ]]; then
225212
launch_tgi_server
226213
fi
227214

228-
if [[ $CURRENT_LLM_SERVING_ENGINE == "lmdeploy" ]]; then
215+
if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then
229216
launch_lmdeploy_server
230217
fi
231218

232-
if [[ $CURRENT_LLM_SERVING_ENGINE == "sglang" ]]; then
219+
if [[ "$CURRENT_LLM_SERVING_ENGINE" == "sglang" ]]; then
233220
launch_sglang_server
234221
fi
235222

‎.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ main() {
1616
fi
1717

1818
# initial annotation
19-
description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
19+
#description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
2020

2121
# download results
22-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
22+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
2323
mkdir -p results/
2424
/workspace/buildkite-agent artifact download 'results/*nightly_results.json' results/
2525
ls
@@ -30,15 +30,15 @@ main() {
3030
/workspace/buildkite-agent artifact upload "results.zip"
3131

3232
# upload benchmarking scripts
33-
cd $VLLM_SOURCE_CODE_LOC/
33+
cd "$VLLM_SOURCE_CODE_LOC/"
3434
zip -r nightly-benchmarks.zip .buildkite/ benchmarks/
3535
/workspace/buildkite-agent artifact upload "nightly-benchmarks.zip"
3636

37-
cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
37+
cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
3838
# upload benchmarking pipeline
3939
/workspace/buildkite-agent artifact upload "nightly-pipeline.yaml"
4040

41-
cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
41+
cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
4242
/workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly-annotation.md
4343

4444

@@ -75,4 +75,4 @@ main() {
7575
# /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly_results.md
7676
}
7777

78-
main "$@"
78+
main "$@"

‎.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ check_gpus() {
1212
echo "Need at least 1 GPU to run benchmarking."
1313
exit 1
1414
fi
15-
declare -g gpu_type=$(echo $(nvidia-smi --query-gpu=name --format=csv,noheader) | awk '{print $2}')
15+
declare -g gpu_type="$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')"
1616
echo "GPU type is $gpu_type"
1717
}
1818

@@ -102,7 +102,7 @@ kill_gpu_processes() {
102102
pkill -f text-generation
103103
pkill -f lmdeploy
104104

105-
while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
105+
while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
106106
sleep 1
107107
done
108108
}
@@ -119,8 +119,8 @@ wait_for_server() {
119119
ensure_installed() {
120120
# Ensure that the given command is installed by apt-get
121121
local cmd=$1
122-
if ! which $cmd >/dev/null; then
123-
apt-get update && apt-get install -y $cmd
122+
if ! which "$cmd" >/dev/null; then
123+
apt-get update && apt-get install -y "$cmd"
124124
fi
125125
}
126126

@@ -173,13 +173,11 @@ run_serving_tests() {
173173
echo "Reuse previous server for test case $test_name"
174174
else
175175
kill_gpu_processes
176-
bash $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh \
176+
bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
177177
"$server_params" "$common_params"
178178
fi
179179

180-
wait_for_server
181-
182-
if [ $? -eq 0 ]; then
180+
if wait_for_server; then
183181
echo ""
184182
echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
185183
else
@@ -190,13 +188,13 @@ run_serving_tests() {
190188

191189
# prepare tokenizer
192190
# this is required for lmdeploy.
193-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
191+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
194192
rm -rf /tokenizer_cache
195193
mkdir /tokenizer_cache
196194
python3 ../.buildkite/nightly-benchmarks/scripts/download-tokenizer.py \
197195
--model "$model" \
198196
--cachedir /tokenizer_cache
199-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
197+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
200198

201199

202200
# change model name for lmdeploy (it will not follow standard hf name)
@@ -307,11 +305,11 @@ run_serving_tests() {
307305
prepare_dataset() {
308306

309307
# download sharegpt dataset
310-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
308+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
311309
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
312310

313311
# duplicate sonnet by 4x, to allow benchmarking with input length 2048
314-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
312+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
315313
echo "" > sonnet_4x.txt
316314
for _ in {1..4}
317315
do
@@ -339,17 +337,17 @@ main() {
339337

340338
prepare_dataset
341339

342-
cd $VLLM_SOURCE_CODE_LOC/benchmarks
340+
cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
343341
declare -g RESULTS_FOLDER=results/
344342
mkdir -p $RESULTS_FOLDER
345-
BENCHMARK_ROOT=$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
343+
BENCHMARK_ROOT="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
346344

347345
# run the test
348-
run_serving_tests $BENCHMARK_ROOT/tests/nightly-tests.json
346+
run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
349347

350348
# upload benchmark results to buildkite
351349
python3 -m pip install tabulate pandas
352-
python3 $BENCHMARK_ROOT/scripts/summary-nightly-results.py
350+
python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
353351
upload_to_buildkite
354352

355353
}

0 commit comments

Comments
 (0)