bump package versions to support gemini 3.1 and claude 4.6 (also rerun scoring and eval)

astabench/evals/sqa/retry_utils.py

@@ -4,7 +4,7 @@
 import logging
 import sqlite3
 from datetime import datetime, timedelta
-from typing import Any, Dict, Optional, Union, List, Tuple
+from typing import Any, Callable, Dict, Optional, Union, List, Tuple
 
 from inspect_ai.model import Model, GenerateConfig
 
@@ -19,6 +19,7 @@ async def generate_with_retry(
     max_retries: int = 20,
     base_delay: float = 2.0,
     desired_schema=None,
+    parsed_validator: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Any:
     """
     Generate response with retry logic and optional JSON parsing.
@@ -57,6 +58,8 @@ async def generate_with_retry(
                 if desired_schema:
                     parsed = desired_schema(**parsed)
                     parsed = parsed.model_dump(mode="json")
+                if parsed_validator:
+                    parsed_validator(parsed)
 
                 return result, parsed, attempt
             else:

astabench/evals/sqa/rubric.py

@@ -368,6 +368,47 @@ async def _assess_properties_independently(self, response, properties):
                 score_components[x.name] = assessment
         return score_components, prompt_logs
 
+    @staticmethod
+    def _validate_joint_assessment_payload(
+        parsed: Dict[str, Any], expected_criteria_count: int
+    ) -> None:
+        scores = parsed.get("scores")
+        if not isinstance(scores, list):
+            raise ValueError("Joint rubric scorer output is missing a 'scores' list")
+
+        seen_indices = []
+        for score in scores:
+            if not isinstance(score, dict):
+                raise ValueError(
+                    "Joint rubric scorer output contains a non-object score"
+                )
+            criteria_idx = score.get("criteria_idx")
+            if not isinstance(criteria_idx, int):
+                raise ValueError(
+                    "Joint rubric scorer output contains a non-integer criteria_idx"
+                )
+            if criteria_idx < 1 or criteria_idx > expected_criteria_count:
+                raise ValueError(
+                    f"Joint rubric scorer output contains out-of-range criteria_idx={criteria_idx}"
+                )
+            seen_indices.append(criteria_idx)
+
+        expected_indices = set(range(1, expected_criteria_count + 1))
+        actual_indices = set(seen_indices)
+        if (
+            actual_indices != expected_indices
+            or len(seen_indices) != expected_criteria_count
+        ):
+            missing = sorted(expected_indices - actual_indices)
+            duplicate = sorted(
+                {idx for idx in seen_indices if seen_indices.count(idx) > 1}
+            )
+            raise ValueError(
+                "Joint rubric scorer output did not cover every criterion exactly once. "
+                f"expected={expected_criteria_count} actual={len(seen_indices)} "
+                f"missing={missing} duplicate={duplicate}"
+            )
+
     async def _assess_properties_jointly(self, response, properties):
         info = {
             "step_name": "score_property",
@@ -461,6 +502,10 @@ async def _assess_properties_jointly(self, response, properties):
                 temperature=self.temperature,
                 top_p=self.top_p,
             ),
+            desired_schema=ResponseCriteriaScores,
+            parsed_validator=lambda parsed: self._validate_joint_assessment_payload(
+                parsed, len(has_criterion)
+            ),
         )
         info["system_prompt"] = system_prompt
         info["user_prompt"] = user_prompt
@@ -508,7 +553,15 @@ async def score_output_simplified(
             )
         score_components.update(assessments)
 
-        assert set(score_components.keys()) == set(score_weights.keys())
+        if set(score_components.keys()) != set(score_weights.keys()):
+            missing = sorted(set(score_weights.keys()) - set(score_components.keys()))
+            unexpected = sorted(
+                set(score_components.keys()) - set(score_weights.keys())
+            )
+            raise ValueError(
+                "Simplified rubric scoring produced mismatched components. "
+                f"missing={missing} unexpected={unexpected}"
+            )
         ann_score = sum(
             score_weights[key] * score_components[key] for key in score_weights
         )

dev_dvc_logs/debug_logs/.gitignore

@@ -16,3 +16,12 @@
 /task_sqa_solver_openscholar_rubric_eval.csv
 /task_sqa_solver_openscholar_citation_eval.csv
 /task_sqa_solver_openscholar_answer_precision_eval.csv
+/task_sqa_solver_sqa_claude-4.6_rubric_eval.csv
+/task_sqa_solver_sqa_claude-4.6_citation_eval.csv
+/task_sqa_solver_sqa_claude-4.6_answer_precision_eval.csv
+/task_sqa_solver_sqa_o3_high_rubric_eval.csv
+/task_sqa_solver_sqa_o3_high_citation_eval.csv
+/task_sqa_solver_sqa_o3_high_answer_precision_eval.csv
+/task_sqa_solver_sqa_gemini-3.1-pro-preview_rubric_eval.csv
+/task_sqa_solver_sqa_gemini-3.1-pro-preview_citation_eval.csv
+/task_sqa_solver_sqa_gemini-3.1-pro-preview_answer_precision_eval.csv