[CI] Enhance the UT results check (#1876)

RUIJIEZHONG66166 · libohao1201 · web-flow · commit 1f7a57f50745 · 2025-07-31T09:20:31.000Z
- Print the new failed cases after filtered by known issue list.
- Enhance the timeout check mechanism, which helps capture the case with
hang issue
- Add UT total number summary in action summary

---------

Co-authored-by: libohao1201 &lt;libo.hao@intel.com&gt;
diff --git a/.github/scripts/check-ut.py b/.github/scripts/check-ut.py
@@ -233,6 +233,17 @@ def print_summary():
     print("### Results Summary")
     print_header = True
 
+    totals = {
+        'Category': '**Total**',
+        'UT': '',
+        'Test cases': 0,
+        'Passed': 0,
+        'Skipped': 0,
+        'Failures': 0,
+        'Errors': 0,
+        'Source': ''
+    }
+
     for summary in summaries:
         print_md_row({
             'Category': summary['Category'],
@@ -246,6 +257,14 @@ def print_summary():
         }, print_header)
         print_header = False
 
+        totals['Test cases'] += summary['Test cases']
+        totals['Passed'] += summary['Passed']
+        totals['Skipped'] += summary['Skipped']
+        totals['Failures'] += summary['Failures']
+        totals['Errors'] += summary['Errors']
+
+    print_md_row(totals)
+
 def main():
     for input_file in args.input_files:
         if input_file.endswith('.log'):
diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh
@@ -9,6 +9,10 @@ compare_and_filter_logs() {
     local file_known_issue="$2"
     local output_file="${3:-${file_UT%.*}_filtered.log}"
     local filtered_content="${file_UT%.*}_removed.log"
+    local temp_file="temp_parts.log"
+    local temp_file_labeled="temp_parts_labeled.log"
+    local temp_output="${3:-${file_UT%.*}_filtered_temp.log}"
+    local temp_final="${file_UT%.*}_final_temp.log"
 
     if [[ $# -lt 2 ]]; then
         echo "[ERROR] Need 2 files to compare"
@@ -21,7 +25,6 @@ compare_and_filter_logs() {
         echo "$file_UT contains $file_known_issue"
     else
         echo "$file_UT does not contain $file_known_issue"
-        return 1
     fi
 
     # Filter the same content from file_UT as file_known_issue
@@ -31,34 +34,73 @@ compare_and_filter_logs() {
     # Keep the filtered UT cases
     grep -noFf "$file_known_issue" "$file_UT" > "$filtered_content"
     echo "Filtered cases file: $filtered_content"
+    true > "$temp_file"
+    true > "$temp_file_labeled"
+    true > "$temp_output"
+    true > "$temp_final"
+    grep -E '\.py$|,' "$output_file" > "$temp_output"
+    while IFS= read -r line; do
+        IFS=',' read -ra parts <<< "$line"
+        for part in "${parts[@]}"; do
+            part_trimmed=$(echo "$part" | xargs)
+            if [[ -n "$part_trimmed" ]] && ! grep -qF "$part_trimmed" "$file_known_issue"; then
+                echo "$part_trimmed" >> "$temp_file"
+                echo -e "\n\033[1;33m[Check the failed cases in summary line]\033[0m"
+                echo -e "\033[1;33mCase not found in ${file_known_issue}: '${part_trimmed}' (from line: '${line}')\033[0m"
+            else
+                echo -e "\n\033[1;33m[Check the failed cases in summary line]\033[0m"
+                echo -e "\n\033[1;32m${part_trimmed} found in ${file_known_issue} (from line: '${line}')\033[0m"
+            fi
+        done
+    done < "$temp_output"
+
+    awk '{print $0 " [in summary line]"}' "$temp_file" > "$temp_file_labeled"
+    grep -vE '\.py$|,' "$output_file" > "$temp_final"
+    cat "$temp_file_labeled" >> "$temp_final"
+    mv "$temp_final" "$output_file"
+
+    echo -e "\n\033[1;31m[New failed cases Summary]\033[0m"
+    if [[ -z "$(tr -d ' \t\n\r\f' < "$output_file" 2>/dev/null)" ]]; then
+        echo -e "\033[1;32mNo new failed cases found\033[0m"
+    else
+        echo -e "\n\033[1;31mNew failed cases, not in known issues\033[0m"
+        cat "$output_file"
+    fi
+
     if [[ -s "$filtered_content" ]]; then
-        echo -e "\n\033[1;31m[Filtered Cases]\033[0m"
+        echo -e "\n\033[1;31m[These failed cases are in skip list, will filter]\033[0m"
         awk -F':' '{
             line_number = $1
             $1 = ""
             gsub(/^ /, "", $0)
             printf "\033[33m%3d\033[0m: %s\n", line_number, $0
         }' "$filtered_content"
     else
-        echo -e "\n\033[1;32mNo Filtered Cases\033[0m"
+        echo -e "\n\033[1;32mNo Skipped Cases\033[0m"
     fi
+
+    rm -f ${temp_output} ${temp_file} ${temp_final}
 }
 
 if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' ]]; then
     grep -E "FAILED" "${ut_suite}"_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_failed.log
     grep -E "have failures" "${ut_suite}"_test.log | awk '{print $1}' >> ./"${ut_suite}"_failed.log
+    grep -E "Timeout" "${ut_suite}"_test.log | grep "test" >> ./"${ut_suite}"_failed.log
     grep "PASSED" "${ut_suite}"_test.log | awk '{print $1}' > ./"${ut_suite}"_passed.log
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite}"
+    echo -e "========================================================================="
+    cat "./${ut_suite}_failed.log"
+    echo -e "========================================================================="
+    echo -e "Checking Failed cases in ${ut_suite}"
+    echo -e "========================================================================="
     compare_and_filter_logs "${ut_suite}"_failed.log Known_issue.log
     if [[ -f "${ut_suite}_failed_filtered.log" ]]; then
       num_failed=$(wc -l < "./${ut_suite}_failed_filtered.log")
     else
       num_failed=$(wc -l < "./${ut_suite}_failed.log")
     fi
     num_passed=$(wc -l < "./${ut_suite}_passed.log")
-    echo -e "========================================================================="
-    echo -e "Show Failed cases in ${ut_suite}"
-    echo -e "========================================================================="
-    cat "./${ut_suite}_failed.log"
     if [[ $num_failed -gt 0 ]] || [[ $num_passed -le 0 ]]; then
       echo -e "[ERROR] UT ${ut_suite} test Fail"
       exit 1
@@ -69,28 +111,36 @@ fi
 if [[ "${ut_suite}" == 'op_ut' ]]; then
     grep -E "FAILED" op_ut_with_skip_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_with_skip_test_failed.log
     grep -E "have failures" op_ut_with_skip_test.log | awk '{print $1}' >> ./"${ut_suite}"_with_skip_test_failed.log
+    grep -E "Timeout" op_ut_with_skip_test.log | grep "test" >> ./"${ut_suite}"_with_skip_test_failed.log
     grep -E "FAILED" op_ut_with_only_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_with_only_test_failed.log
     grep -E "have failures" op_ut_with_only_test.log | awk '{print $1}' >> ./"${ut_suite}"_with_only_test_failed.log
+    grep -E "Timeout" op_ut_with_only_test.log | grep "test" >> ./"${ut_suite}"_with_only_test_failed.log
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite} with skip"
+    echo -e "========================================================================="
+    cat "./${ut_suite}_with_skip_test_failed.log"
+    echo -e "========================================================================="
+    echo -e "Checking Failed cases in ${ut_suite} with skip"
+    echo -e "========================================================================="
     compare_and_filter_logs "${ut_suite}"_with_skip_test_failed.log Known_issue.log
     if [[ -f "${ut_suite}_with_skip_test_failed_filtered.log" ]]; then
       num_failed_with_skip=$(wc -l < "./${ut_suite}_with_skip_test_failed_filtered.log")
     else
       num_failed_with_skip=$(wc -l < "./${ut_suite}_with_skip_test_failed.log")
     fi
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite} with only"
+    echo -e "========================================================================="
+    cat "./${ut_suite}_with_only_test_failed.log"
+    echo -e "========================================================================="
+    echo -e "Checking Failed cases in ${ut_suite} with only"
+    echo -e "========================================================================="
     compare_and_filter_logs "${ut_suite}"_with_only_test_failed.log Known_issue.log
     if [[ -f "${ut_suite}_with_only_test_failed_filtered.log" ]]; then
       num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed_filtered.log")
     else
       num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed.log")
     fi
-    echo -e "========================================================================="
-    echo -e "Show Failed cases in ${ut_suite} with skip"
-    echo -e "========================================================================="
-    cat "./${ut_suite}_with_skip_test_failed.log"
-    echo -e "========================================================================="
-    echo -e "Show Failed cases in ${ut_suite} with only"
-    echo -e "========================================================================="
-    cat "./${ut_suite}_with_only_test_failed.log"
     ((num_failed=num_failed_with_skip+num_failed_with_only))
     grep "PASSED" op_ut_with_skip_test.log | awk '{print $1}' > ./"${ut_suite}"_with_skip_test_passed.log
     grep "PASSED" op_ut_with_only_test.log | awk '{print $1}' > ./"${ut_suite}"_with_only_test_passed.log
@@ -136,16 +186,19 @@ fi
 if [[ "${ut_suite}" == 'xpu_distributed' ]]; then
     grep -E "^FAILED" xpu_distributed_test.log | awk '{print $2}' > ./"${ut_suite}"_xpu_distributed_test_failed.log
     grep -E "have failures" xpu_distributed_test.log | awk '{print $1}' >> ./"${ut_suite}"_xpu_distributed_test_failed.log
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite} xpu distributed"
+    echo -e "========================================================================="
+    cat "./${ut_suite}_xpu_distributed_test_failed.log"
+    echo -e "========================================================================="
+    echo -e "Checking Failed cases in ${ut_suite} xpu distributed"
+    echo -e "========================================================================="
     compare_and_filter_logs "${ut_suite}"_xpu_distributed_test_failed.log Known_issue.log
     if [[ -f "${ut_suite}_xpu_distributed_test_failed_filtered.log" ]]; then
       num_failed_xpu_distributed=$(wc -l < "./${ut_suite}_xpu_distributed_test_failed_filtered.log")
     else
       num_failed_xpu_distributed=$(wc -l < "./${ut_suite}_xpu_distributed_test_failed.log")
     fi
-    echo -e "========================================================================="
-    echo -e "Show Failed cases in ${ut_suite} xpu distributed"
-    echo -e "========================================================================="
-    cat "./${ut_suite}_xpu_distributed_test_failed.log"
     ((num_failed=num_failed_xpu_distributed))
     if [[ $num_failed -gt 0 ]]; then
       echo -e "[ERROR] UT ${ut_suite} test Fail"
diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml
@@ -63,18 +63,16 @@ jobs:
           - name: 'op_regression'
             condition: ${{ contains(inputs.ut, 'op_regression') }}
             directory: 'test/regressions'
-            command: 'pytest --timeout 600 -v --junit-xml=../../ut_log/op_regression.xml'
+            command: 'pytest --timeout 600 --timeout_method=thread -v --junit-xml=../../ut_log/op_regression.xml'
             log_prefix: 'op_regression'
-            timeout: 8000
             additional_steps: |
               clinfo --list
               pip install pytest pytest-timeout
           - name: 'op_regression_dev1'
             condition: ${{ contains(inputs.ut, 'op_regression_dev1') }}
             directory: 'test/regressions'
-            command: 'pytest --timeout 600 -v test_operation_on_device_1.py --junit-xml=$GITHUB_WORKSPACE/ut_log/op_regression_dev1.xml'
+            command: 'pytest --timeout 600 --timeout_method=thread -v test_operation_on_device_1.py --junit-xml=$GITHUB_WORKSPACE/ut_log/op_regression_dev1.xml'
             log_prefix: 'op_regression_dev1'
-            timeout: 8000
             additional_steps: |
               clinfo --list
               unset ZE_AFFINITY_MASK
@@ -83,9 +81,8 @@ jobs:
           - name: 'op_transformers'
             condition: ${{ contains(inputs.ut, 'op_transformers') }}
             directory: '../pytorch'
-            command: 'pytest --timeout 600 -v test/test_transformers.py -k xpu --junit-xml=$GITHUB_WORKSPACE/ut_log/op_transformers.xml'
+            command: 'pytest --timeout 600 --timeout_method=thread -v test/test_transformers.py -k xpu --junit-xml=$GITHUB_WORKSPACE/ut_log/op_transformers.xml'
             log_prefix: 'op_transformers'
-            timeout: 3600
             additional_steps: |
               pip install pytest pytest-timeout
               export PYTORCH_TEST_WITH_SLOW=1
@@ -94,7 +91,6 @@ jobs:
             directory: '../pytorch/third_party/torch-xpu-ops/test/xpu/extended/'
             command: 'python run_test_with_skip.py'
             log_prefix: 'op_extended'
-            timeout: 10000
             additional_steps: |
               pip install pytest pytest-timeout
               export PYTORCH_TEST_WITH_SLOW=1
@@ -107,7 +103,7 @@ jobs:
             command_script: |
               export PYTORCH_ENABLE_XPU_FALLBACK=1
               export PYTORCH_TEST_WITH_SLOW=1
-              timeout 10000 python run_test_with_skip.py \
+              python run_test_with_skip.py \
                 2>$GITHUB_WORKSPACE/ut_log/op_ut/op_ut_with_skip_test_error.log | \
                 tee $GITHUB_WORKSPACE/ut_log/op_ut/op_ut_with_skip_test.log
               cp *.xml $GITHUB_WORKSPACE/ut_log
@@ -147,7 +143,6 @@ jobs:
               eval $test_cmd 2>$GITHUB_WORKSPACE/ut_log/torch_xpu/torch_xpu_test_error.log | \
                 tee $GITHUB_WORKSPACE/ut_log/torch_xpu/torch_xpu_test.log
             log_prefix: 'torch_xpu'
-            timeout: 10000
             additional_steps: |
               pip install pytest pytest-timeout
           - name: 'xpu_profiling'
@@ -188,11 +183,13 @@ jobs:
       - name: Checkout torch-xpu-ops
         uses: actions/checkout@v4
       - name: Create unique workspace
+        shell: bash -xe {0}
         run: |
           # Create unique conda env for each UT test
           random=$(head /dev/urandom | tr -dc A-Za-z0-9_ | head -c ${1:-5} | xargs)
           echo "CONDA_ENV_NAME=xpu_op_${ZE_AFFINITY_MASK}_${{ matrix.test.name }}_${random}" >> $GITHUB_ENV
       - name: Create Conda Env
+        shell: bash -xe {0}
         run: |
           pwd
           which conda
@@ -206,6 +203,7 @@ jobs:
         with:
           name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }}
       - name: Prepare Stock Pytorch
+        shell: bash -xe {0}
         run: |
           cd ../
           rm -rf ./pytorch || sudo rm -rf ./pytorch
@@ -229,6 +227,7 @@ jobs:
           git show -s && git status && git diff
           pip install -r .ci/docker/requirements-ci.txt
       - name: Prepare Torch-xpu-ops
+        shell: bash -xe {0}
         run: |
           cd ../pytorch
           rm -rf third_party/torch-xpu-ops
@@ -247,10 +246,12 @@ jobs:
           name: Triton-Wheel-${{ github.event.pull_request.number || github.sha }}
           path: ${{ github.workspace }}
       - name: Install Triton
+        shell: bash -xe {0}
         run: |
           source activate $CONDA_ENV_NAME
           pip install --force-reinstall ${{ github.workspace }}/pytorch_triton_xpu-*.whl
       - name: Torch Config
+        shell: bash -xe {0}
         run: |
           source activate $CONDA_ENV_NAME
           python -c "import torch; print(torch.__config__.show())"
@@ -264,6 +265,7 @@ jobs:
           rm -rf ~/.triton/cache || sudo rm -rf ~/.triton/cache
           echo "UT_NAME=${{ matrix.test.name }}" >> "${GITHUB_ENV}"
       - name: Run XPU UT Test
+        shell: bash -xe {0}
         if: ${{ matrix.test.condition }}
         run: |
           set -e
@@ -282,12 +284,13 @@ jobs:
             ${{ matrix.test.command_script }}
           SCRIPT
           else
-            timeout ${{ matrix.test.timeout }} ${{ matrix.test.command }} \
+            ${{ matrix.test.command }} \
               2>${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_prefix }}_test_error.log | \
               tee ${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_prefix }}_test.log
             ${{ matrix.test.xml_post_processing || '' }}
           fi
       - name: UT Test Results Summary
+        shell: bash -xe {0}
         if: ${{ matrix.test.condition }}
         run: |
           source activate $CONDA_ENV_NAME
diff --git a/test/xpu/extended/run_test_with_skip.py b/test/xpu/extended/run_test_with_skip.py
@@ -17,7 +17,7 @@
 skip_options += '"'
 
 os.environ["PYTORCH_TEST_WITH_SLOW"] = "1"
-test_command = "pytest --timeout 600 -v --junit-xml=./op_extended.xml test_ops_xpu.py"
+test_command = "pytest --timeout 600 -v --timeout_method=thread --junit-xml=./op_extended.xml test_ops_xpu.py"
 test_command += skip_options
 res = os.system(test_command)
 sys.exit(res)
