Improved error reporting for failed tolerance checks (#988)

Co-authored-by: Dimitrios Adam <[email protected]>
Co-authored-by: Spencer Bryngelson <[email protected]>
Co-authored-by: Spencer Bryngelson <[email protected]>

Author: 3 people

toolchain/mfc/packer/tol.py

@@ -18,6 +18,34 @@ def is_close(error: Error, tolerance: Tolerance) -> bool:
     return False
 
 
+def _format_error_diagnostics(max_abs_info, max_rel_info) -> str:
+    """Format diagnostic information for maximum errors among failing variables."""
+    diagnostic_msg = ""
+
+    if max_abs_info:
+        filepath, val_idx, g_val, c_val, abs_err, rel_err = max_abs_info
+        rel_error_str = f"{rel_err:.2E}" if not math.isnan(rel_err) else "NaN"
+        diagnostic_msg += f"\n\nDiagnostics - Maximum absolute error among FAILING variables:\n" \
+                         f" - File: {filepath}\n" \
+                         f" - Variable n°{val_idx+1}\n" \
+                         f" - Candidate: {c_val}\n" \
+                         f" - Golden: {g_val}\n" \
+                         f" - Absolute Error: {abs_err:.2E}\n" \
+                         f" - Relative Error: {rel_error_str}"
+
+    if max_rel_info:
+        filepath, val_idx, g_val, c_val, rel_err, abs_err = max_rel_info
+        diagnostic_msg += f"\n\nDiagnostics - Maximum relative error among FAILING variables:\n" \
+                         f" - File: {filepath}\n" \
+                         f" - Variable n°{val_idx+1}\n" \
+                         f" - Candidate: {c_val}\n" \
+                         f" - Golden: {g_val}\n" \
+                         f" - Relative Error: {rel_err:.2E}\n" \
+                         f" - Absolute Error: {abs_err:.2E}"
+
+    return diagnostic_msg
+
+
 # pylint: disable=too-many-return-statements
 def compare(candidate: Pack, golden: Pack, tol: Tolerance) -> typing.Tuple[Error, str]:
     # Keep track of the average error
@@ -45,23 +73,69 @@ def compare(candidate: Pack, golden: Pack, tol: Tolerance) -> typing.Tuple[Error
             error = compute_error(cVal, gVal)
             avg_err.push(error)
 
-            def raise_err(msg: str):
+            def raise_err_with_failing_diagnostics(msg: str):
+                # Find maximum errors among FAILING variables only
+                max_abs_info, max_rel_info = find_maximum_errors_among_failing(candidate, golden, tol)
+                diagnostic_msg = _format_error_diagnostics(max_abs_info, max_rel_info)
+
                 return None, f"""\
 Variable n°{valIndex+1} (1-indexed) in {gFilepath} {msg}:
   - Candidate:   {cVal}
   - Golden:      {gVal}
   - Error:       {error}
-  - Tolerance:   {tol}
+  - Tolerance:   {tol}{diagnostic_msg}
 """
 
             if math.isnan(gVal):
-                return raise_err("is NaN in the golden file")
-
+                return raise_err_with_failing_diagnostics("is NaN in the golden file")
             if math.isnan(cVal):
-                return raise_err("is NaN in the pack file")
-
+                return raise_err_with_failing_diagnostics("is NaN in the pack file")
             if not is_close(error, tol):
-                return raise_err("is not within tolerance")
+                return raise_err_with_failing_diagnostics("is not within tolerance")
 
     # Return the average relative error
     return avg_err.get(), None
+
+
+def find_maximum_errors_among_failing(candidate: Pack, golden: Pack, tol: Tolerance) -> typing.Tuple[typing.Optional[typing.Tuple[str, int, float, float, float, float]], typing.Optional[typing.Tuple[str, int, float, float, float, float]]]:
+    """
+    Scan all files to find the maximum absolute and relative errors among FAILING variables only.
+    A variable fails if is_close(error, tol) returns False.
+
+    Returns tuple of:
+    - max_abs_info: (filepath, var_index, golden_val, candidate_val, absolute_error, relative_error)
+    - max_rel_info: (filepath, var_index, golden_val, candidate_val, relative_error, absolute_error)
+    """
+    max_abs_error = -1.0
+    max_abs_info = None
+
+    max_rel_error = -1.0
+    max_rel_info = None
+
+    for gFilepath, gEntry in golden.entries.items():
+        cEntry = candidate.find(gFilepath)
+        if cEntry is None:
+            continue
+
+        for valIndex, (gVal, cVal) in enumerate(zip(gEntry.doubles, cEntry.doubles)):
+            # Skip NaN values in golden or candidate
+            if math.isnan(gVal) or math.isnan(cVal):
+                continue
+
+            error = compute_error(cVal, gVal)
+
+            # Only consider variables that FAIL tolerance
+            if is_close(error, tol):
+                continue  # Skip variables that pass tolerance
+
+            # Track maximum absolute error among failing variables
+            if error.absolute > max_abs_error:
+                max_abs_error = error.absolute
+                max_abs_info = (gFilepath, valIndex, gVal, cVal, error.absolute, error.relative)
+
+            # Track maximum relative error among failing variables (only if it's not NaN)
+            if not math.isnan(error.relative) and error.relative > max_rel_error:
+                max_rel_error = error.relative
+                max_rel_info = (gFilepath, valIndex, gVal, cVal, error.relative, error.absolute)
+
+    return max_abs_info, max_rel_info