check-inductor-report.py

Author: kwasd

.github/workflows/e2e-reusable.yml

@@ -228,9 +228,18 @@ jobs:
           if [[ "${{ inputs.only_one_model }}" ]]; then
             bash -e $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh ${{ inputs.suite }} ${{ inputs.dtype }} ${{ inputs.mode }} ${{ inputs.test_mode }} xpu 0 static 1 0 ${{ inputs.only_one_model }}
           elif [[ "${{ inputs.models }}" == "subset" ]]; then
+            models_subset_file = "$GITHUB_WORKSPACE/.github/models/${{ inputs.test_mode }}/${{ inputs.suite }}.txt"
             while read model; do
               bash -e $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh ${{ inputs.suite }} ${{ inputs.dtype }} ${{ inputs.mode }} ${{ inputs.test_mode }} xpu 0 static 1 0 $model
-            done < $GITHUB_WORKSPACE/.github/models/${{ inputs.test_mode }}/${{ inputs.suite }}.txt
+            done < $models_subset_file
+            if [[ "${{ inputs.test_all_subset_models }}" == true ]]
+              python $GITHUB_WORKSPACE/scripts/check-inductor-logs.py --models-file="$models_subset_file" \
+                --suite=${{ inputs.suite }} \
+                --dtype=${{ inputs.dtype }} \
+                --mode=${{ inputs.mode }} \
+                --test_mode=${{ inputs.test_mode }} \
+                --device=xpu
+            fi
           else
             bash -e $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh ${{ inputs.suite }} ${{ inputs.dtype }} ${{ inputs.mode }} ${{ inputs.test_mode }} xpu 0 static 1 0
           fi

scripts/check-inductor-report.py

@@ -0,0 +1,64 @@
+import sys
+import argparse
+from pathlib import Path
+import csv
+
+def check_report(suite, dtype, mode, test_mode, device, models_file):
+    inductor_log_dir = Path('inductor_log') / suite / dtype
+    inductor_report_filename = f"inductor_{suite}_{dtype}_{mode}_{device}_{test_mode}.csv"
+    inductor_report_path = Path(inductor_log_dir / inductor_report_filename)
+
+    subset = []
+    report = []
+    exitcode = 0
+
+    with open(models_file) as f:
+        subset = f.read().strip().split('\n')
+
+    with open(inductor_report_path) as f:
+        reader = csv.reader(f)
+        report_with_header = []
+        for l in reader:
+            report_with_header.append(l)
+        for r in report_with_header[1:]:
+            if r[0] == device:
+                report.append(r)
+
+    test_list = [r[1] for r in report]
+
+    if test_mode == "performance":
+        for m in subset:
+            if m not in test_list:
+                exitcode = 1
+                print(f"Test is not found in report: {m}")
+
+    if test_mode == "accuracy":
+        test_statuses = [r[3] for r in report]
+        for m in subset:
+            try:
+                idx = test_list.index(m)
+            except ValueError:
+                exitcode = 1
+                print(f"Test is NOT FOUND: {m}")
+                continue
+            if test_statuses[idx] != "pass":
+                exitcode = 1
+                print(f"Test is NOT PASSED: {m}")
+    return exitcode
+
+def main():
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument('--suite', required=True)
+    argparser.add_argument('--dtype', required=True)
+    argparser.add_argument('--mode', required=True, choices=("training", "inference"))
+    argparser.add_argument('--test_mode', required=True, choices=("performance", "accuracy"))
+    argparser.add_argument('--device', help="i.e. xpu", required=True)
+    argparser.add_argument('--models-file', help="Subset of models list", required=True)
+    args = argparser.parse_args()
+    exitcode = check_report(args.suite, args.dtype, args.mode, args.test_mode, args.device, args.models_file)
+    print(f"Report check result: {'SUCCESS' if exitcode == 0 else 'FAIL'}")
+    exit(exitcode)
+
+
+if __name__ == "__main__":
+    main()