[CI] Improve error handling and validation in Jenkins pipeline, tuna-script and tuningRunner (#1913)

Improve CI robustness and error visibility by enhancing error handling in the Jenkins pipeline, validating output files from tuna-script and guarding against unbound variables. (Together with the changes from ROCm/MITuna#999)

---------

Signed-off-by: Djordje Antic <[email protected]>

Author: dorde-antic

mlir/utils/jenkins/Jenkinsfile

@@ -966,30 +966,45 @@ pipeline {
                                             git branch: "pf-tuna-rocmlir-3", poll: false, url: 'https://github.com/ROCm/MITuna.git'
                                         }
                                         dir('build') {
+                                            def tuningLog = "tune_rocmlir_${CHIP}.log"
+                                            sh """echo "=== Tuning rocMLIR for ${CHIP} ===" | tee ${tuningLog}"""
                                             // Tune gemms with default datatypes, fail if the DB is not created
                                             // (Includes int8xint8->int8 for performance comparisons against CK.)
                                             sh """../mlir/utils/tuna/tuna-script.sh -o gemm \
                                                     -c ../mlir/utils/performance/configs/tier1-gemm-configs \
-                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv
-                                                [ -f mlir_tuning_${CHIP}.tsv ]"""
+                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv 2>&1 | tee -a ${tuningLog}
+                                                [ -f mlir_tuning_${CHIP}.tsv ]""" 
                                             // Tune resnet50 and unet configs
                                             sh """../mlir/utils/tuna/tuna-script.sh -o convolution \
                                                     -c ../mlir/utils/performance/configs/tier1-conv-configs \
-                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv"""
+                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv 2>&1 | tee -a ${tuningLog}"""
                                             // Tune attention configs
                                             sh """../mlir/utils/tuna/tuna-script.sh -o attention \
                                                     -c ../mlir/utils/performance/configs/tier1-attention-configs \
-                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv"""
+                                                    -t ${WORKSPACE}/MITuna -f mlir_tuning_${CHIP}.tsv 2>&1 | tee -a ${tuningLog}"""
                                             // Tune gemms with default datatypes, fail if the DB is not created (quick tuning)
                                             // (Includes int8xint8->int8 for performance comparisons against CK.)
                                             sh """../mlir/utils/tuna/tuna-script.sh -o gemm \
                                                     -c ../mlir/utils/performance/configs/tier1-gemm-configs -s quick \
-                                                    -t ${WORKSPACE}/MITuna -f mlir_quick_tuning_${CHIP}.tsv
+                                                    -t ${WORKSPACE}/MITuna -f mlir_quick_tuning_${CHIP}.tsv 2>&1 | tee -a ${tuningLog}
                                                 [ -f mlir_quick_tuning_${CHIP}.tsv ]"""
                                             // Tune resnet50 and unet configs (quick tuning)
                                             sh """../mlir/utils/tuna/tuna-script.sh -o convolution \
                                                     -c ../mlir/utils/performance/configs/tier1-conv-configs -s quick \
-                                                    -t ${WORKSPACE}/MITuna -f mlir_quick_tuning_${CHIP}.tsv"""
+                                                    -t ${WORKSPACE}/MITuna -f mlir_quick_tuning_${CHIP}.tsv 2>&1 | tee -a ${tuningLog}"""
+                                            sh """echo "=== Tuning rocMLIR for ${CHIP} completed ===" | tee -a ${tuningLog}"""
+                                            // Check for errors in the tuning log
+                                            script {
+                                                def tuneLog = readFile(tuningLog).split('\n')
+                                                def errors = tuneLog.findAll { it =~ /(?i)\berror\b/ }
+
+                                                if (errors) {
+                                                    error("Tuning failed: Detected errors in tuning log")
+                                                    currentBuild.result = 'FAILURE'
+                                                } else {
+                                                    echo "No errors found in tuning log"
+                                                }
+                                            }
                                         }
                                     }
                                 }

mlir/utils/performance/tuningRunner.py

@@ -401,7 +401,8 @@ def main(args=None):
 
     if winners is None:
         # Tuning aborted, bail
-        return
+        print("Tuning aborted")
+        return 1
 
     if parsed_args.debug:
         print(allData, file=sys.stderr)

mlir/utils/tuna/tuna-script.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Ensures that the script exits on error and returns the exit status of the last command that failed.
+set -euo pipefail
+
 # For installing mysql 8.0 for testing, or for running with an isolated database.
 function mysql_setup_generic
 {
@@ -42,6 +45,28 @@ function clear_tables
     mysql --user root --database tuna -e "delete from rocmlir_${tablekind}_config;"
 }
 
+validate_tuning_file() {
+    local f="$1"
+    if [ ! -s "$f" ]; then
+        echo "ERROR: $f is missing or empty!"
+        exit 1
+    fi
+    header=$(head -n1 "$f")
+    # Check for two headers next to each other anywhere in the file
+    if awk -v h="$header" 'prev==h && $0==h {print; exit 1} {prev=$0}' "$f"; then
+        true
+    else
+        echo "ERROR: $f has duplicate header lines next to each other!"
+        exit 1
+    fi
+    # Existing checks
+    data_line=$(awk 'NR>1 && $0 !~ /^\s*$/ && $0 != header {print; exit}' header="$header" "$f")
+    if [ -z "$data_line" ]; then
+        echo "ERROR: $f has no data after header!"
+        exit 1
+    fi
+}
+
 function tuna_run
 {
     kind=$1
@@ -60,10 +85,9 @@ function tuna_run
     fi
     (cd "${ROCMLIR_DIR}"/build/ || exit 1 ; ${TUNA_DIR}/tuna/go_fish.py rocmlir --execute --session_id "$session" $factor)
     ${TUNA_DIR}/tuna/rocmlir/export_configs.py --session_id "$session" --append -f "$OUT_FILE"
+    validate_tuning_file "$OUT_FILE"
 }
 
-
-
 usage() { echo "$0 usage:" && grep " .)\ #" "$0"; exit 0; }
 [ $# -eq 0 ] && usage
 
@@ -118,7 +142,7 @@ export TUNA_DB_USER_NAME=root
 export TUNA_DB_USER_PASSWORD=
 export TUNA_DB_HOSTNAME=127.0.0.1
 export TUNA_DB_NAME=tuna
-export PYTHONPATH=$TUNA_DIR:$PYTHONPATH
+export PYTHONPATH=$TUNA_DIR:${PYTHONPATH:-}
 
 # If no mysqld running, assume it and Tuna need to be set up.
 # Otherwise, assume the usual setup.
@@ -130,7 +154,7 @@ else
     TUNA_DB_HOSTNAME=localhost
 fi
 
-if [ "$VIRTUAL_ENV" = "" ]; then
+if [ "${VIRTUAL_ENV:-}" = "" ]; then
     source /tuna-venv/bin/activate
 fi