chore: expose bench configs in env var

Signed-off-by: Huamin Chen <[email protected]>

Author: rootfs

bench/run_bench.sh

@@ -1,13 +1,21 @@
 #!/bin/bash
 
-set -x 
+# Example usage:
+# Quick run:
+# SAMPLES_PER_CATEGORY=5 CONCURRENT_REQUESTS=4 VLLM_MODELS="openai/gpt-oss-20b" ROUTER_MODELS="auto" ./run_bench.sh
+# Long run:
+# SAMPLES_PER_CATEGORY=100 CONCURRENT_REQUESTS=4 VLLM_MODELS="openai/gpt-oss-20b" ROUTER_MODELS="auto" ./run_bench.sh
 
-export ROUTER_API_KEY="1234567890"
-export VLLM_API_KEY="1234567890"
-export ROUTER_ENDPOINT="http://localhost:8801/v1"
-export VLLM_ENDPOINT="http://localhost:8000/v1"
-export ROUTER_MODELS="auto"
-export VLLM_MODELS="openai/gpt-oss-20b"
+set -x -e
+
+export ROUTER_API_KEY="${ROUTER_API_KEY:-1234567890}"
+export VLLM_API_KEY="${VLLM_API_KEY:-1234567890}"
+export ROUTER_ENDPOINT="${ROUTER_ENDPOINT:-http://localhost:8801/v1}"
+export VLLM_ENDPOINT="${VLLM_ENDPOINT:-http://localhost:8000/v1}"
+export ROUTER_MODELS="${ROUTER_MODELS:-auto}"
+export VLLM_MODELS="${VLLM_MODELS:-openai/gpt-oss-20b}"
+export SAMPLES_PER_CATEGORY="${SAMPLES_PER_CATEGORY:-5}"
+export CONCURRENT_REQUESTS="${CONCURRENT_REQUESTS:-4}"
 
 # Run the benchmark
 python router_reason_bench.py \
@@ -19,16 +27,33 @@ python router_reason_bench.py \
   --vllm-endpoint "$VLLM_ENDPOINT" \
   --vllm-api-key "$VLLM_API_KEY" \
   --vllm-models "$VLLM_MODELS" \
-  --samples-per-category 5 \
+  --samples-per-category "$SAMPLES_PER_CATEGORY" \
   --vllm-exec-modes NR XC \
-  --concurrent-requests 4 \
+  --concurrent-requests "$CONCURRENT_REQUESTS" \
   --output-dir results/reasonbench
 
 # Generate plots
-VLLM_MODEL_FIRST="${VLLM_MODELS%% *}"
-ROUTER_MODEL_FIRST="${ROUTER_MODELS%% *}"
-VLLM_MODELS_SAFE="${VLLM_MODEL_FIRST//\//_}"
-ROUTER_MODELS_SAFE="${ROUTER_MODEL_FIRST//\//_}"
+echo "Processing model paths..."
+echo "VLLM_MODELS: $VLLM_MODELS"
+echo "ROUTER_MODELS: $ROUTER_MODELS"
+
+# Get first model name and make it path-safe
+VLLM_MODEL_FIRST=$(echo "$VLLM_MODELS" | cut -d' ' -f1)
+ROUTER_MODEL_FIRST=$(echo "$ROUTER_MODELS" | cut -d' ' -f1)
+echo "First models: VLLM=$VLLM_MODEL_FIRST, Router=$ROUTER_MODEL_FIRST"
+
+# Replace / with _ for path safety
+VLLM_MODELS_SAFE=$(echo "$VLLM_MODEL_FIRST" | tr '/' '_')
+ROUTER_MODELS_SAFE=$(echo "$ROUTER_MODEL_FIRST" | tr '/' '_')
+echo "Safe paths: VLLM=$VLLM_MODELS_SAFE, Router=$ROUTER_MODELS_SAFE"
+
+# Construct the full paths
+VLLM_SUMMARY="results/reasonbench/vllm::${VLLM_MODELS_SAFE}/summary.json"
+ROUTER_SUMMARY="results/reasonbench/router::${ROUTER_MODELS_SAFE}/summary.json"
+echo "Looking for summaries at:"
+echo "VLLM: $VLLM_SUMMARY"
+echo "Router: $ROUTER_SUMMARY"
+
 python bench_plot.py \
-  --summary "results/reasonbench/vllm::${VLLM_MODELS_SAFE}/summary.json" \
-  --router-summary "results/reasonbench/router::${ROUTER_MODELS_SAFE}/summary.json"
+  --summary "$VLLM_SUMMARY" \
+  --router-summary "$ROUTER_SUMMARY"