[Run] Capture the logs for all pods (from llm-d stack) at the end (#638)

Whenever run is invoked, try to capture the logs from gaie, decode,
prefill and inference-gateway pods at the end of a run.

The llm-d-benchmark executable now has indepedent try loops for the
harness and analyzer.

Finally, a few improvements on `preprocess/set_llmdbench_environment.py`

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

Author: maugustosilva

build/llm-d-benchmark.sh

@@ -1,7 +1,10 @@
 #!/usr/bin/env bash
-export LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC=1
+export LLMDBENCH_RUN_EXPERIMENT_HARNESS_LOADGEN_EC=1
+export LLMDBENCH_RUN_EXPERIMENT_HARNESS_REPORT_EC=1
+
 export LLMDBENCH_RUN_EXPERIMENT_HARNESS_NAME_AUTO=1
 export LLMDBENCH_RUN_EXPERIMENT_HARNESS_WORKLOAD_AUTO=1
+export LLMDBENCH_RUN_EXPERIMENT_HARNESS_MAX_TRIES=${LLMDBENCH_RUN_EXPERIMENT_HARNESS_MAX_TRIES:-3}
 
 function show_usage {
     echo -e "Usage: $0 -l/--harness [harness used to generate load (default=$LLMDBENCH_HARNESS_NAME, possible values $(ls $LLMDBENCH_RUN_WORKSPACE_DIR/profiles/ | sed -n ':a;N;$!ba;s/\n/,/g;p')] \n \
@@ -93,10 +96,10 @@ fi
 
 env | grep ^LLMDBENCH | grep -v BASE64 | sort
 
-# Repeat run until success
+
 echo "Running harness: /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}"
 counter=1
-while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC -ne 0 && "${counter}" -le 3 ]]; do
+while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_LOADGEN_EC -ne 0 && "${counter}" -le $LLMDBENCH_RUN_EXPERIMENT_HARNESS_MAX_TRIES ]]; do
   /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}
   ec=$?
   if [[ $ec -ne 0 ]]; then
@@ -105,7 +108,7 @@ while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC -ne 0 && "${counter}" -le 3 ]]; do
     counter="$(( ${counter} + 1 ))"
     set -x
   else
-    export LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC=0
+    export LLMDBENCH_RUN_EXPERIMENT_HARNESS_LOADGEN_EC=0
   fi
 done
 echo "Harness completed: /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}"
@@ -115,18 +118,23 @@ if [[ -f ~/fixbashrc ]]; then
 fi
 
 echo "Running analysis: /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER}"
-# Try to run analysis twice then give up
+counter=1
+while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_REPORT_EC -ne 0 && "${counter}" -le $LLMDBENCH_RUN_EXPERIMENT_HARNESS_MAX_TRIES ]]; do
 /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER}
 ec=$?
 if [[ $ec -ne 0 ]]; then
-  echo "execution of /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER} failed, wating 120 seconds and trying again"
-  sleep 120
-  set -x
-  /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER}
-fi
+    echo "execution of /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER} failed, wating 30 seconds and trying again"
+    sleep 30
+    counter="$(( ${counter} + 1 ))"
+    set -x
+  else
+    export LLMDBENCH_RUN_EXPERIMENT_HARNESS_REPORT_EC=0
+  fi
+done
+
 
 if [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_NAME_AUTO -eq 0 ]]; then
   echo "Done. Data is available at \"$LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR\""
 fi
 # Return with error code of first iteration of experiment analyzer
-exit $ec
+exit $((LLMDBENCH_RUN_EXPERIMENT_HARNESS_LOADGEN_EC + LLMDBENCH_RUN_EXPERIMENT_HARNESS_REPORT_EC))

setup/env.sh

@@ -119,7 +119,7 @@ export LLMDBENCH_VLLM_COMMON_ACCELERATOR_NR=${LLMDBENCH_VLLM_COMMON_ACCELERATOR_
 export LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM=${LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM:-1}
 export LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM=${LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM:-1}
 export LLMDBENCH_VLLM_COMMON_DATA_LOCAL_PARALLELISM=${LLMDBENCH_VLLM_COMMON_DATA_LOCAL_PARALLELISM:-1}
-# export LLMDBENCH_VLLM_COMMON_NUM_WORKERS_PARALLELISM=${LLMDBENCH_VLLM_COMMON_NUM_WORKERS_PARALLELISM:-1}
+export LLMDBENCH_VLLM_COMMON_NUM_WORKERS_PARALLELISM=${LLMDBENCH_VLLM_COMMON_NUM_WORKERS_PARALLELISM:-1}
 export LLMDBENCH_VLLM_COMMON_ACCELERATOR_MEM_UTIL=${LLMDBENCH_VLLM_COMMON_ACCELERATOR_MEM_UTIL:-0.95}
 export LLMDBENCH_VLLM_COMMON_CPU_NR=${LLMDBENCH_VLLM_COMMON_CPU_NR:-4}
 export LLMDBENCH_VLLM_COMMON_CPU_MEM=${LLMDBENCH_VLLM_COMMON_CPU_MEM:-40Gi}

setup/functions.sh

@@ -509,20 +509,11 @@ function deploy_harness_config {
         announce "✅ Collected analysis for pods with label \"app=${LLMDBENCH_HARNESS_POD_LABEL}\" at: \"${LLMDBENCH_CONTROL_WORK_DIR}/analysis/\""
 
         announce "🗑️ Deleting pods with label \"app=${LLMDBENCH_HARNESS_POD_LABEL}\" for model \"$model\" ..."
-        llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete pod -l app=${LLMDBENCH_HARNESS_POD_LABEL}" \
-            ${LLMDBENCH_CONTROL_DRY_RUN} \
-            ${LLMDBENCH_CONTROL_VERBOSE}
-        llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete pod -l app=llm-d-benchmark-harness" \
+        llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete pod -l function=load_generator" \
             ${LLMDBENCH_CONTROL_DRY_RUN} \
             ${LLMDBENCH_CONTROL_VERBOSE}
         announce "✅ Pods with label \"app=${LLMDBENCH_HARNESS_POD_LABEL}\" for model \"$model\" deleted"
 
-      announce "ℹ️ Capturing the current status of all pods in namespace \"$LLMDBENCH_VLLM_COMMON_NAMESPACE\" to ${pod_results_dir}/pod_status.txt..."
-      llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE get pods -o wide > ${pod_results_dir}/pod_status.txt" \
-      ${LLMDBENCH_CONTROL_DRY_RUN} \
-      ${LLMDBENCH_CONTROL_VERBOSE}
-      announce "✅ Pod status captured."
-
     elif [[ $LLMDBENCH_HARNESS_WAIT_TIMEOUT -eq 0 ]]; then
       announce "ℹ️ Harness was started with LLMDBENCH_HARNESS_WAIT_TIMEOUT=0. Will NOT wait for pod \"${LLMDBENCH_HARNESS_POD_LABEL}\" for model \"$model\" to be in \"Completed\" state. The pod can be accessed through \"${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} exec -it pod/<POD_NAME> -- bash\""
       announce "ℹ️ To list pod names \"${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} get pods -l app=${LLMDBENCH_HARNESS_POD_LABEL}\""
@@ -536,6 +527,39 @@ function deploy_harness_config {
 }
 export -f deploy_harness_config
 
+function capture_pod_logs {
+    local model=$1
+    local local_results_dir=$2
+
+    local modelid_label=$(model_attribute $model modelid_label)
+
+    for i in $(seq 1 "$LLMDBENCH_HARNESS_LOAD_PARALLELISM"); do
+      pod_results_dir="${local_results_dir}_${i}"
+      pod_analysis_dir="${local_analysis_dir}_${i}"
+
+      announce "ℹ️ Capturing the current status of all pods in namespace \"$LLMDBENCH_VLLM_COMMON_NAMESPACE\" to ${pod_results_dir}/pod_status.txt ..."
+      llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE get pods -o wide > ${pod_results_dir}/pod_status.txt" \
+      ${LLMDBENCH_CONTROL_DRY_RUN} \
+      ${LLMDBENCH_CONTROL_VERBOSE}
+      announce "✅ Pod status captured."
+
+      announce "ℹ️ Capturing logs for all pods in namespace \"$LLMDBENCH_VLLM_COMMON_NAMESPACE\" to ${pod_results_dir}/logs/ ..."
+      mkdir -p ${pod_results_dir}/logs/
+      llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE logs --tail=-1 --prefix=true -l llm-d.ai/model=\"$modelid_label\" > ${pod_results_dir}/logs/modelserving_pods.log"  \
+      ${LLMDBENCH_CONTROL_DRY_RUN} \
+      ${LLMDBENCH_CONTROL_VERBOSE}
+
+      llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE logs --tail=-1 --prefix=true -l inferencepool=\"${modelid_label}-gaie-epp\" > ${pod_results_dir}/logs/epp_pods.log"  \
+      ${LLMDBENCH_CONTROL_DRY_RUN} \
+      ${LLMDBENCH_CONTROL_VERBOSE}
+
+      llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE logs --tail=-1 --prefix=true -l \"app.kubernetes.io/component=inference-gateway\" > ${pod_results_dir}/logs/igw_pods.log"  \
+      ${LLMDBENCH_CONTROL_DRY_RUN} \
+      ${LLMDBENCH_CONTROL_VERBOSE}
+    done
+}
+export -f capture_pod_logs
+
 function create_harness_pod {
 
   local _podname=$1
@@ -560,6 +584,7 @@ metadata:
   namespace: ${LLMDBENCH_HARNESS_NAMESPACE}
   labels:
     app: ${LLMDBENCH_HARNESS_POD_LABEL}
+    function: load_generator
 spec:
   containers:
   - name: harness
@@ -825,14 +850,13 @@ export -f generate_profile_parameter_treatments
 
 function cleanup_pre_execution {
   announce "🗑️ Deleting pods with label \"${LLMDBENCH_HARNESS_POD_LABEL}\"..."
-  llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete pod -l app=${LLMDBENCH_HARNESS_POD_LABEL} --ignore-not-found" ${LLMDBENCH_CONTROL_DRY_RUN} ${LLMDBENCH_CONTROL_VERBOSE}
+  llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete pod -l app=${LLMDBENCH_HARNESS_POD_LABEL},function=load_generator --ignore-not-found" ${LLMDBENCH_CONTROL_DRY_RUN} ${LLMDBENCH_CONTROL_VERBOSE}
   # Sanitize the stack name to make it a valid K8s/OpenShift resource name
   local LLMDBENCH_HARNESS_SANITIZED_STACK_NAME=$(echo "${LLMDBENCH_HARNESS_STACK_NAME}" | $LLMDBENCH_CONTROL_SCMD 's|[/:]|-|g')
   llmdbench_execute_cmd "${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} delete job lmbenchmark-evaluate-${LLMDBENCH_HARNESS_SANITIZED_STACK_NAME} --ignore-not-found" ${LLMDBENCH_CONTROL_DRY_RUN} ${LLMDBENCH_CONTROL_VERBOSE}
   announce "ℹ️ Done deleting pods with label \"${LLMDBENCH_HARNESS_POD_LABEL}\" (it will be now recreated)"
 
 }
-
 export -f cleanup_pre_execution
 
 function validate_model_name {