[CI] add nightly failure notify support (#124)

* night failure notify

* bug in shell check

* update inductor_perf_summary.py

Author: yudongsi

.github/scripts/inductor_perf_summary.py

@@ -41,9 +41,9 @@ def caculate_passrate(df, compiler):
 
 def get_perf_csv(precision, mode):
     target_path = 'inductor_log/huggingface/' + precision + '/inductor_' + args.suite + '_' + precision + '_' + mode + '_xpu_performance.csv'
-    target_ori_data = pd.read_csv(target_path, header=0, encoding='utf-8', names=['dev', 'name', 'batch_size', 'speedup', 'abs_latency', 'compilation_latency', 'compression_ratio'])
+    target_ori_data = pd.read_csv(target_path)
     target_data = target_ori_data.copy()
-    target_data.sort_values(by=['name'], key=lambda col: col.str.lower(), inplace=True)
+    target_data.sort_values(by=['name'])
     return target_data
 
 

.github/workflows/triton_xpu_backend_e2e_ci.yml

@@ -142,6 +142,7 @@ jobs:
 
       - name: Test Results Overview
         run: |
+          set +e
           cd ${HOME}/triton-preci/pytorch/inductor_log/huggingface
           cd amp_bf16
           echo -e "============ Acc Check for HF amp_bf16 ============" | tee -a ./e2e_summary.log

.github/workflows/triton_xpu_backend_e2e_nightly.yml

@@ -140,6 +140,7 @@ jobs:
 
       - name: ACC Test Results Overview
         run: |
+          set +e
           cd ${HOME}/triton-nightly/pytorch/inductor_log/huggingface
           cd amp_bf16
           echo -e "============ Acc Check for HF amp_bf16 ============" | tee -a ./e2e_summary.log
@@ -234,6 +235,7 @@ jobs:
 
       - name: Perf Test Results Generate and Overview
         run: |
+          set +e
           cd ${HOME}/triton-nightly/pytorch
           python inductor_perf_summary.py -s huggingface -p amp_bf16 amp_fp16
 
@@ -242,3 +244,20 @@ jobs:
         with:
           name: Triton-Inductor-E2E-Nightly-Data
           path: /home/gta/triton-nightly/pytorch/inductor_log/
+
+  Tests-Failure-And-Report:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    needs: [Tests-Env-Prepare, Accuracy-Test, Performance-Test]
+    if: always()
+    steps:
+      - name: Report github issue when failure detected
+        if: ${{ needs.Tests-Env-Prepare.result == 'failure' || needs.Accuracy-Test.result == 'failure' || needs.Performance-Test.result == 'failure' }}
+        run: |
+          build_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          notification_list="${{ secrets.ISSUE_NOTIFICATION_LIST }}"
+          repo="${{ github.repository }}"
+          gh --repo $repo issue comment 123  \
+              --body "E2E Nightly Failed $(date +'%Y-%m-%d'),See: $build_url
+                      cc $notificiation_list"

.github/workflows/triton_xpu_backend_nightly.yml

@@ -198,3 +198,20 @@ jobs:
         with:
           name: Triton-nightly-data
           path: /home/sdp/triton-nightly/triton_src/python/test/unit/language/*.log
+
+  Tests-Failure-And-Report:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    needs: Nightly-Tests
+    if: always()
+    steps:
+      - name: Report github issue when failure detected
+        if: ${{ needs.Nightly-Tests.result == 'failure' }}
+        run: |
+          build_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          notification_list="${{ secrets.ISSUE_NOTIFICATION_LIST }}"
+          repo="${{ github.repository }}"
+          gh --repo $repo issue comment 123  \
+              --body "UT Nightly Failed $(date +'%Y-%m-%d'),See: $build_url
+                      cc $notificiation_list"