add online runs for disagg

Author: sixiang-google

cases/hourly_disagg.csv

@@ -1,2 +1,6 @@
-Device,Model,MaxNumSeqs,MaxNumBatchedTokens,TensorParallelSize,MaxModelLen,Dataset,InputLen,OutputLen,ExpectedETEL,NumPrompts
-v6e-8,meta-llama/Llama-3.1-8B-Instruct,128,1024,2,2048,sonnet,1800,128,,1000
+Device,Model,MaxNumSeqs,MaxNumBatchedTokens,TensorParallelSize,MaxModelLen,Dataset,InputLen,OutputLen,ExpectedETEL,NumPrompts,MODELTAG,PREFIX_LEN,RequestRate
+v6e-8,meta-llama/Llama-3.1-8B-Instruct,512,8192,4,2048,mlperf,1024,1024,,10000,,,inf
+v6e-8,meta-llama/Llama-3.1-8B-Instruct,512,8192,4,2048,mlperf,1024,1024,,10000,,,50
+v6e-8,meta-llama/Llama-3.1-8B-Instruct,512,8192,4,2048,mlperf,1024,1024,,10000,,,60
+v6e-8,Qwen/Qwen3-32B,128,8192,4,2048,mlperf,1024,1024,,5000,,,inf
+v6e-8,Qwen/Qwen3-32B,128,8192,4,2048,mlperf,1024,1024,,5000,,,20

database/vllm_bm_20251117.ddl

@@ -0,0 +1,2 @@
+ALTER TABLE RunRecord ADD COLUMN RequestRate FLOAT64 DEFAULT(0);
+CREATE INDEX IDX_RunRecord_RequestRate ON RunRecord (RequestRate);

scripts/agent/run_bm.sh

@@ -215,6 +215,13 @@ run_benchmark(){
   echo "$throughput $p99_e2el"
 }
 
+# If REQUEST_RATE is provided, run the benchmark once with it and exit.
+if [[ -n "${REQUEST_RATE:-}" ]]; then
+  echo "REQUEST_RATE is set to '$REQUEST_RATE'. Running benchmark once."
+  run_benchmark "$REQUEST_RATE"
+  exit 0
+fi
+
 read throughput p99_e2el < <(run_benchmark "inf" | tail -n 1)
 
 echo "throughput:$throughput"

scripts/scheduler/hourly_run.sh

@@ -131,6 +131,6 @@ fi
 #   ./scripts/scheduler/create_job.sh ./cases/autotune_ali.csv "" $TAG AUTOTUNE
 # fi
 
-echo LOCAL_PATCH=1 ./scripts/scheduler/create_job.sh ./cases/hourly_disagg.csv "" $TAG HOURLY_DISAGG TPU_INFERENCE "PREFILL_SLICES=2;DECODE_SLICES=2;TPU_BACKEND_TYPE=jax"
-LOCAL_PATCH=1 ./scripts/scheduler/create_job.sh ./cases/hourly_disagg.csv "" $TAG HOURLY_DISAGG TPU_INFERENCE "PREFILL_SLICES=2;DECODE_SLICES=2;TPU_BACKEND_TYPE=jax"
+echo LOCAL_PATCH=1 ./scripts/scheduler/create_job.sh ./cases/hourly_disagg.csv "" $TAG HOURLY_DISAGG TPU_INFERENCE "PREFILL_SLICES=4;DECODE_SLICES=4;TPU_BACKEND_TYPE=jax"
+LOCAL_PATCH=1 ./scripts/scheduler/create_job.sh ./cases/hourly_disagg.csv "" $TAG HOURLY_DISAGG TPU_INFERENCE "PREFILL_SLICES=4;DECODE_SLICES=4;TPU_BACKEND_TYPE=jax"
 

scripts/scheduler/schedule_run.sh

@@ -58,7 +58,8 @@ tail -n +2 "$CSV_FILE" | while read -r line || [ -n "${line}" ]; do
     EXPECTED_ETEL \
     NUM_PROMPTS \
     MODELTAG \
-    PREFIX_LEN <<< "$line"
+    PREFIX_LEN \
+    REQUEST_RATE <<< "$line"
 
   RECORD_ID=$(uuidgen | tr 'A-Z' 'a-z')
 
@@ -79,7 +80,7 @@ tail -n +2 "$CSV_FILE" | while read -r line || [ -n "${line}" ]; do
     --sql="INSERT INTO RunRecord (
       RecordId, Status, CreatedTime, Device, Model, RunType, CodeHash,
       MaxNumSeqs, MaxNumBatchedTokens, TensorParallelSize, MaxModelLen,
-      Dataset, InputLen, OutputLen, LastUpdate, CreatedBy,JobReference, ExpectedETEL, NumPrompts, ModelTag, PrefixLen, ExtraEnvs
+      Dataset, InputLen, OutputLen, LastUpdate, CreatedBy,JobReference, ExpectedETEL, NumPrompts, ModelTag, PrefixLen, ExtraEnvs, RequestRate
     ) VALUES (
       '$RECORD_ID', 'CREATED', PENDING_COMMIT_TIMESTAMP(), '$DEVICE', '$MODEL', '$RUN_TYPE', '$CODEHASH',
       $MAX_NUM_SEQS,
@@ -96,7 +97,8 @@ tail -n +2 "$CSV_FILE" | while read -r line || [ -n "${line}" ]; do
       ${NUM_PROMPTS:-1000},
       '${MODELTAG:-PROD}',
       ${PREFIX_LEN:-0},
-      '$EXTRA_ENVS'
+      '$EXTRA_ENVS',
+      '${REQUEST_RATE:-inf}'
     );"
 
   # If insert failed, just continue without publishing