more info in model_results.json (huggingface#39783)

more info

Co-authored-by: ydshieh <[email protected]>

Author: ydshieh

utils/notification_service.py

@@ -75,6 +75,8 @@ def handle_test_results(test_results):
 
     failed = 0
     success = 0
+    errors = 0
+    skipped = 0
 
     # When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
     # When it is too long, those signs are not present.
@@ -83,10 +85,14 @@ def handle_test_results(test_results):
     for i, expression in enumerate(expressions):
         if "failed" in expression:
             failed += int(expressions[i - 1])
+        if "errors" in expression:
+            errors += int(expressions[i - 1])
         if "passed" in expression:
             success += int(expressions[i - 1])
+        if "skipped" in expression:
+            skipped += int(expressions[i - 1])
 
-    return failed, success, time_spent
+    return failed, errors, success, skipped, time_spent
 
 
 def handle_stacktraces(test_results):
@@ -1188,7 +1194,9 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
     matrix_job_results = {
         matrix_name: {
             "failed": {m: {"unclassified": 0, "single": 0, "multi": 0} for m in test_categories},
+            "errors": 0,
             "success": 0,
+            "skipped": 0,
             "time_spent": "",
             "failures": {},
             "job_link": {},
@@ -1213,8 +1221,10 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
                 # Link to the GitHub Action job
                 job = artifact_name_to_job_map[path]
                 matrix_job_results[matrix_name]["job_link"][artifact_gpu] = job["html_url"]
-                failed, success, time_spent = handle_test_results(artifact["stats"])
+                failed, errors, success, skipped, time_spent = handle_test_results(artifact["stats"])
                 matrix_job_results[matrix_name]["success"] += success
+                matrix_job_results[matrix_name]["errors"] += errors
+                matrix_job_results[matrix_name]["skipped"] += skipped
                 matrix_job_results[matrix_name]["time_spent"] += time_spent[1:-1] + ", "
 
                 stacktraces = handle_stacktraces(artifact["failures_line"])
@@ -1317,7 +1327,9 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
     additional_results = {
         key: {
             "failed": {"unclassified": 0, "single": 0, "multi": 0},
+            "errors": 0,
             "success": 0,
+            "skipped": 0,
             "time_spent": "",
             "error": False,
             "failures": {},
@@ -1343,9 +1355,11 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
             artifact = retrieve_artifact(path, artifact_gpu)
             stacktraces = handle_stacktraces(artifact["failures_line"])
 
-            failed, success, time_spent = handle_test_results(artifact["stats"])
+            failed, errors, success, skipped, time_spent = handle_test_results(artifact["stats"])
             additional_results[key]["failed"][artifact_gpu or "unclassified"] += failed
             additional_results[key]["success"] += success
+            additional_results[key]["errors"] += errors
+            additional_results[key]["skipped"] += skipped
             additional_results[key]["time_spent"] += time_spent[1:-1] + ", "
 
             if len(artifact["errors"]):