[0.6.0-UT] Adding abort support to run_multi_gpu (#529)

Author: gulsumgudukbay

build/rocm/run_multi_gpu.sh

@@ -28,7 +28,7 @@ detect_amd_gpus() {
         echo "Error: lspci command not found. Aborting."
         exit 1
     fi
-    # Count AMD GPUs.
+    # Count AMD/ATI GPU controllers.
     local count
     count=$(rocm-smi | grep -E '^Device' -A 1000 | awk '$1 ~ /^[0-9]+$/ {count++} END {print count}')
     echo "$count"
@@ -73,17 +73,36 @@ run_tests() {
         echo "Running multi-GPU test: $test_file"
 
         # Define file paths for abort detection (files created by conftest.py)
+        last_running_file="${LOG_DIR}/${test_name}_last_running.json"
         json_log_file="${LOG_DIR}/multi_gpu_${test_name}_log.json"
         html_log_file="${LOG_DIR}/multi_gpu_${test_name}_log.html"
 
-        # Run the test
+        # Run the test (conftest.py will create the last_running_file automatically)
         python3 -m pytest \
             --html="$html_log_file" \
             --json-report \
             --json-report-file="$json_log_file" \
             --reruns 3 \
             "$test_file"
-
+        
+        # Check for aborted test and handle it
+        if [[ -f "$last_running_file" ]]; then
+            echo "Abort detected for test: $test_name"
+            # Get the absolute path of the script directory
+            script_dir="$(cd "$(dirname "$0")" && pwd)"
+            # Convert relative paths to absolute paths
+            abs_json_log_file="$(realpath "$json_log_file")"
+            abs_html_log_file="$(realpath "$html_log_file")"
+            abs_last_running_file="$(realpath "$last_running_file")"
+            
+            cd "$script_dir"
+            python3 -c "
+from run_single_gpu import handle_abort
+import sys
+success = handle_abort('$abs_json_log_file', '$abs_html_log_file', '$abs_last_running_file', 'multi_gpu_$test_name')
+sys.exit(0 if success else 1)
+"
+        fi
     done
 
     # Merge individual HTML reports into one.