Include partial grades in pass rate calculation

Updated process_results.py to count both 'Complete' and 'Partial' grades towards the pass rate and improved summary output. Enhanced quality_gate.py with clearer documentation and output messages to reflect the new pass rate calculation method.

Author: karangattu

tests/inspect-ai/utils/scripts/process_results.py

@@ -51,7 +51,9 @@ def process_inspect_results(result_file_path: Union[str, Path]) -> None:
         if s.get("scores", {}).get("model_graded_qa", {}).get("value") == "I"
     )
 
-    pass_rate = (passed_tests / total_tests) * 100 if total_tests > 0 else 0
+    # Calculate pass rate including both Complete and Partial grades
+    passing_tests = passed_tests + partial_tests
+    pass_rate = (passing_tests / total_tests) * 100 if total_tests > 0 else 0
 
     # Generate summary dictionary
     summary = {
@@ -63,7 +65,7 @@ def process_inspect_results(result_file_path: Union[str, Path]) -> None:
         "quality_gate_passed": pass_rate >= 80,  # 80% threshold
         "details": (
             f"Complete: {passed_tests}, Partial: {partial_tests}, "
-            f"Incomplete: {failed_tests}"
+            f"Incomplete: {failed_tests}, Passing: {passing_tests}/{total_tests}"
         ),
     }
 
@@ -74,10 +76,10 @@ def process_inspect_results(result_file_path: Union[str, Path]) -> None:
 
     print(f"\nSummary saved to: {summary_file_path}")
     print(
-        f"Processed {total_tests} tests: {passed_tests} passed, "
-        f"{partial_tests} partial, {failed_tests} failed"
+        f"Processed {total_tests} tests: {passed_tests} complete, "
+        f"{partial_tests} partial, {failed_tests} incomplete"
     )
-    print(f"Pass rate: {pass_rate:.1f}%")
+    print(f"Pass rate (Complete + Partial): {pass_rate:.1f}%")
 
 
 if __name__ == "__main__":

tests/inspect-ai/utils/scripts/quality_gate.py

@@ -5,7 +5,18 @@
 
 
 def check_quality_gate(results_dir: Union[str, Path], threshold: float = 80) -> None:
-    """Check if evaluation results meet quality gate"""
+    """
+    Check if evaluation results meet quality gate.
+
+    The quality gate is based on the pass_rate from the summary.json file.
+    Pass rate includes both 'Complete' (C) and 'Partial' (P) grades.
+    Tests with 'Incomplete' (I) grade do not count towards the pass rate.
+
+    Args:
+        results_dir: Directory containing the summary.json file
+        threshold: Minimum pass rate percentage required (default: 80%)
+    """
+
     summary_path = Path(results_dir) / "summary.json"
 
     if not summary_path.exists():
@@ -18,10 +29,14 @@ def check_quality_gate(results_dir: Union[str, Path], threshold: float = 80) ->
     pass_rate = summary.get("pass_rate", 0)
 
     if pass_rate >= threshold:
-        print(f"✅ Quality gate PASSED: {pass_rate:.1f}% >= {threshold}%")
+        print(
+            f"✅ Quality gate PASSED: {pass_rate:.1f}% >= {threshold}% (Complete + Partial grades)"
+        )
         sys.exit(0)
     else:
-        print(f"❌ Quality gate FAILED: {pass_rate:.1f}% < {threshold}%")
+        print(
+            f"❌ Quality gate FAILED: {pass_rate:.1f}% < {threshold}% (Complete + Partial grades)"
+        )
         sys.exit(1)