Resolve Task Metrics Immediately (UKGovernmentBEIS#2439)

* Resolve Task Metrics Immediately

Rather than pass task metrics thorugh to results computation, resolve the task metrics immediately onto the scorers and simply use those when computing results.

* Add tests

* Update CHANGELOG.md

* correct changelog

---------

Co-authored-by: jjallaire <jj.allaire@gmail.com>

Author: dragonstyle

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Sandbox Service: Specify instance externally so a single script can service multiple instances.
 - Agent Bridge: Capture message history in agent state for all bridge generations.
 - Agent Bridge: Embed sandbox service client in sandbox bridge proxy (for ease of bundling).
+- Scoring: Resolve task or eval level metrics onto scorers immediately rather than waiting until scoring.
 - Inspect View: Add support for cmd + arrow up/down to navigate the samples list.
 - Inspect View: Improve scroll keyboard handling in sample transcript view.
 - Inspect View: Improve scroll keyboard handling in sample messages view.

src/inspect_ai/_eval/score.py

@@ -15,6 +15,7 @@
 from inspect_ai._display import display as display_manager
 from inspect_ai._eval.context import init_task_context
 from inspect_ai._eval.loader import scorer_from_spec
+from inspect_ai._eval.task.task import resolve_scorer_metrics
 from inspect_ai._util._async import configured_async_backend, run_coroutine, tg_collect
 from inspect_ai._util.platform import platform_init, running_in_notebook
 from inspect_ai._util.registry import registry_create, registry_unqualified_name
@@ -247,6 +248,9 @@ async def _score_sample(idx_sample: int) -> None:
         # that will be taken care of in eval_results)
         log_metrics = metrics_from_log_header(log)
 
+        # resolve the scorer metrics onto the scorers
+        scorers = resolve_scorer_metrics(scorers, log_metrics) or []
+
         # override epochs_reducer if specified
         epochs_reducer = create_reducers(epochs_reducer)
         if epochs_reducer:
@@ -260,7 +264,6 @@ async def _score_sample(idx_sample: int) -> None:
             list(filter(None, scores)),
             epochs_reducer,
             scorers,
-            log_metrics,
         )
 
     return log

src/inspect_ai/_eval/task/results.py

@@ -69,7 +69,6 @@ def eval_results(
     scores: list[dict[str, SampleScore]],
     reducers: ScoreReducer | list[ScoreReducer] | None,
     scorers: list[Scorer] | None,
-    metrics: list[Metric] | dict[str, list[Metric]] | None,
 ) -> Tuple[EvalResults, list[EvalSampleReductions] | None]:
     # initialise results
     results = EvalResults(total_samples=samples, completed_samples=len(scores))
@@ -105,11 +104,9 @@ def eval_results(
             if len(reducers) == 0:
                 # Compute metrics without reduction since no reducers were
                 # explicitly specified
-                targets = metrics if metrics is not None else scorer_info.metrics
-
                 eval_scores = compute_eval_scores(
                     resolved_scores,
-                    targets,
+                    scorer_info.metrics,
                     scorer_name,
                     scorer_info,
                     None,
@@ -135,11 +132,9 @@ def eval_results(
                     sample_reductions.append(reduced_samples)
 
                     # Compute metrics for this scorer
-                    targets = metrics if metrics is not None else scorer_info.metrics
-
                     eval_scores = compute_eval_scores(
                         reduced_scores,
-                        targets,
+                        scorer_info.metrics,
                         scorer_name,
                         scorer_info,
                         reducer_display_nm,

src/inspect_ai/_eval/task/run.py

@@ -77,7 +77,7 @@
 )
 from inspect_ai.model._model import init_sample_model_usage, sample_model_usage
 from inspect_ai.scorer import Scorer, Target
-from inspect_ai.scorer._metric import Metric, SampleScore
+from inspect_ai.scorer._metric import SampleScore
 from inspect_ai.scorer._reducer.types import ScoreReducer
 from inspect_ai.scorer._score import init_scoring_context
 from inspect_ai.scorer._scorer import unique_scorer_name
@@ -311,7 +311,6 @@ def sample_complete(sample_score: dict[str, SampleScore]) -> None:
                         progress_results,
                         scorers,
                         task.epochs_reducer,
-                        task.metrics,
                     )
 
                 # initial progress
@@ -323,7 +322,6 @@ def sample_complete(sample_score: dict[str, SampleScore]) -> None:
                     progress_results,
                     scorers,
                     task.epochs_reducer,
-                    task.metrics,
                 )
 
                 async def run_sample(
@@ -383,7 +381,6 @@ async def run_sample(
                     scores=completed_scores,
                     reducers=task.epochs_reducer,
                     scorers=scorers,
-                    metrics=task.metrics,
                 )
 
             # collect eval data
@@ -477,7 +474,6 @@ def update_metrics_display_fn(
         list[dict[str, SampleScore]],
         list[Scorer] | None,
         ScoreReducer | list[ScoreReducer] | None,
-        list[Metric] | dict[str, list[Metric]] | None,
     ],
     None,
 ]:
@@ -488,7 +484,6 @@ def compute(
         sample_scores: list[dict[str, SampleScore]],
         scorers: list[Scorer] | None,
         reducers: ScoreReducer | list[ScoreReducer] | None,
-        metrics: list[Metric] | dict[str, list[Metric]] | None,
     ) -> None:
         # Don't compute metrics if they are not being displayed
         if not display_metrics:
@@ -503,7 +498,6 @@ def compute(
                 scores=sample_scores,
                 reducers=reducers,
                 scorers=scorers,
-                metrics=metrics,
             )
 
             # Name, reducer, value

src/inspect_ai/_eval/task/task.py

@@ -8,9 +8,11 @@
 from inspect_ai._util.logger import warn_once
 from inspect_ai._util.notgiven import NOT_GIVEN, NotGiven
 from inspect_ai._util.registry import (
+    RegistryInfo,
     is_registry_object,
     registry_info,
     registry_unqualified_name,
+    set_registry_info,
 )
 from inspect_ai.agent._agent import Agent, is_agent
 from inspect_ai.agent._as_solver import as_solver
@@ -141,7 +143,7 @@ def __init__(
         self.setup = setup
         self.solver = resolve_solver(solver)
         self.cleanup = cleanup
-        self.scorer = resolve_scorer(scorer)
+        self.scorer = resolve_scorer_metrics(resolve_scorer(scorer), metrics)
         self.metrics = metrics
         self.model = resolve_model(model)
         self.config = config
@@ -417,3 +419,17 @@ def resolve_scorer(scorer: Scorer | list[Scorer] | None) -> list[Scorer] | None:
     return (
         scorer if isinstance(scorer, list) else [scorer] if scorer is not None else None
     )
+
+
+def resolve_scorer_metrics(
+    scorers: list[Scorer] | None, metrics: list[Metric] | dict[str, list[Metric]] | None
+) -> list[Scorer] | None:
+    if scorers is not None and metrics is not None:
+        for scorer in scorers:
+            scorer_info = registry_info(scorer)
+            new_metadata = {**scorer_info.metadata, "metrics": metrics}
+            new_info = RegistryInfo(
+                type=scorer_info.type, name=scorer_info.name, metadata=new_metadata
+            )
+            set_registry_info(scorer, new_info)
+    return scorers

tests/scorer/test_task_scorer.py

@@ -0,0 +1,83 @@
+from inspect_ai import eval
+from inspect_ai._eval.task.task import Task
+from inspect_ai._util.registry import registry_info
+from inspect_ai.dataset._sources.csv import csv_dataset
+from inspect_ai.scorer._answer import answer
+from inspect_ai.scorer._classification import f1
+from inspect_ai.scorer._metrics import accuracy, mean
+from inspect_ai.scorer._metrics.std import bootstrap_stderr
+
+
+def test_task_with_metrics():
+    task = Task(scorer=f1(), metrics=[mean(), bootstrap_stderr()])
+
+    # ensure that metrics themselves remain unchanged
+    assert registry_info(task.metrics[0]).name == "inspect_ai/mean"
+    assert registry_info(task.metrics[1]).name == "inspect_ai/bootstrap_stderr"
+    assert task.scorer is not None
+
+    # ensure that the task metrics are correctly applied to the scorer
+    info = registry_info(task.scorer[0])
+    assert registry_info(info.metadata["metrics"][0]).name == "inspect_ai/mean"
+
+    info = registry_info(task.scorer[0])
+    assert (
+        registry_info(info.metadata["metrics"][1]).name == "inspect_ai/bootstrap_stderr"
+    )
+
+    # modify the task and ensure that the new metrics remain unchanged
+    task.scorer.append(answer("word"))
+    assert len(task.scorer) == 2
+    info = registry_info(task.scorer[1])
+    assert registry_info(info.metadata["metrics"][0]).name == "inspect_ai/accuracy"
+    assert registry_info(info.metadata["metrics"][1]).name == "inspect_ai/stderr"
+
+
+def test_task_score_results():
+    task = Task(
+        dataset=csv_dataset("tests/dataset/test_dataset/samples-md.csv"),
+        scorer=f1(),
+        metrics=[accuracy()],
+    )
+
+    # confirm the mean result is computed
+    log = eval(task, model="mockllm/model", sandbox=False)
+    assert len(log[0].results.scores) == 1
+    assert len(log[0].results.scores[0].metrics) == 1
+    assert "accuracy" in log[0].results.scores[0].metrics
+    assert "mean" not in log[0].results.scores[0].metrics
+    assert "stderr" not in log[0].results.scores[0].metrics
+
+
+def test_score_results():
+    task = Task(
+        dataset=csv_dataset("tests/dataset/test_dataset/samples-md.csv"),
+        scorer=f1(),
+    )
+
+    # confirm the mean result is computed
+    log = eval(task, model="mockllm/model")
+    assert len(log[0].results.scores) == 1
+    assert len(log[0].results.scores[0].metrics) == 2
+    assert "mean" in log[0].results.scores[0].metrics
+    assert "stderr" in log[0].results.scores[0].metrics
+
+
+def test_added_scores():
+    task = Task(
+        dataset=csv_dataset("tests/dataset/test_dataset/samples-md.csv"),
+        scorer=f1(),
+        metrics=[accuracy()],
+    )
+    task.scorer.append(answer("line"))
+
+    log = eval(task, model="mockllm/model")
+    assert len(log[0].results.scores) == 2
+    assert len(log[0].results.scores[0].metrics) == 1
+    assert "accuracy" in log[0].results.scores[0].metrics
+    assert "mean" not in log[0].results.scores[0].metrics
+    assert "stderr" not in log[0].results.scores[0].metrics
+
+    assert len(log[0].results.scores[1].metrics) == 2
+    assert "accuracy" in log[0].results.scores[1].metrics
+    assert "stderr" in log[0].results.scores[1].metrics