eval tui

configs/evals/debug.toml

@@ -1,7 +1,11 @@
 [[env]]
 env_id = "gsm8k"
-num_examples = 1
+num_examples = 20
 rollouts_per_example = 1
+sampling_args = { max_tokens = 1024 }
+independent_scoring = true
+save_results = true
+save_every = 10
 
 [[env]]
 env_id = "alphabet-sort"

configs/evals/single-turn.toml

@@ -0,0 +1,20 @@
+[[env]]
+env_id = "math500"
+num_examples = -1
+rollouts_per_example = 1
+
+[[env]]
+env_id = "aime2024"
+num_examples = -1
+rollouts_per_example = 8
+
+[[env]]
+env_id = "gpqa"
+num_examples = -1
+rollouts_per_example = 1
+
+[[env]]
+env_id = "livecodebench"
+num_examples = -1
+rollouts_per_example = 1
+max_concurrent = 16 # to limit sandbox usage

docs/reference.md

@@ -542,7 +542,6 @@ class EvalConfig(BaseModel):
     max_concurrent_generation: int | None = None
     max_concurrent_scoring: int | None = None
     extra_env_kwargs: dict = {}
-    print_results: bool = False
     verbose: bool = False
     state_columns: list[str] | None = None
     save_results: bool = False

tests/test_eval_cli.py

@@ -114,14 +114,14 @@ def _run_cli(monkeypatch, overrides, capture_all_configs: bool = False):
         "temperature": 0.9,
         "sampling_args": None,
         "verbose": False,
-        "print_results": False,
         "no_interleave_scoring": False,
         "state_columns": [],
         "save_results": False,
         "save_every": -1,
         "save_to_hf_hub": False,
         "hf_hub_dataset_name": "",
         "extra_env_kwargs": {},
+        "tui": False,
     }
     base_args.update(overrides)
     args_namespace = SimpleNamespace(**base_args)
@@ -136,7 +136,7 @@ def _run_cli(monkeypatch, overrides, capture_all_configs: bool = False):
     monkeypatch.setattr(vf_eval, "setup_logging", lambda *_, **__: None)
     monkeypatch.setattr(vf_eval, "load_endpoints", lambda *_: {})
 
-    async def fake_run_evaluation(config):
+    async def fake_run_evaluation(config, **kwargs):
         captured["sampling_args"] = dict(config.sampling_args)
         captured["configs"].append(config)
         metadata = _make_metadata(config)

verifiers/envs/environment.py

@@ -36,12 +36,15 @@
     ChatMessage,
     GenerateMetadata,
     GenerateOutputs,
+    LogCallback,
     Messages,
     MessageType,
     ModelResponse,
+    ProgressCallback,
     RolloutInput,
     RolloutTiming,
     SamplingArgs,
+    StartCallback,
     State,
 )
 from verifiers.utils.async_utils import maybe_semaphore
@@ -830,17 +833,25 @@ async def generate(
         state_columns: list[str] | None = None,
         save_results: bool = False,
         save_every: int = -1,
-        use_tqdm: bool = True,
         independent_scoring: bool = False,
+        on_start: StartCallback | None = None,
+        on_progress: ProgressCallback | None = None,
+        on_log: LogCallback | None = None,
     ) -> GenerateOutputs:
         """
         Generate rollouts for a set of inputs.
         """
+        on_log = on_log or self.logger.debug
+
         if isinstance(inputs, Dataset):
             inputs_list = inputs.to_list()
         elif isinstance(inputs, list):
             inputs_list = inputs
 
+        # notify caller of actual total count (useful when num_examples=-1)
+        if on_start is not None:
+            on_start(len(inputs_list))
+
         # resolve concurrency knobs
         gen_limit = max_concurrent_generation
         score_limit = max_concurrent_scoring
@@ -876,8 +887,6 @@ async def generate(
                     )
                 )
                 tasks[task] = i
-            pbar_total = len(inputs_list)
-            pbar_desc = f"Processing {len(inputs_list)} rollouts"
         else:
             input_groups: dict[int, list[RolloutInput]] = {}
             for input_item in inputs_list:
@@ -899,62 +908,41 @@ async def generate(
                     )
                 )
                 tasks[task] = i
-            pbar_total = len(group_list)
-            pbar_desc = f"Processing {len(group_list)} groups ({len(inputs_list)} total rollouts)"
-
-        # set up progress bar
-        pbar = None
-        if use_tqdm:
-            from tqdm import tqdm
-
-            pbar = tqdm(total=pbar_total, desc=pbar_desc, postfix=dict(reward="?"))
 
         # process tasks as they complete
-        reward_sum, reward_count = 0, 0
-        groups_or_rollouts_completed = 0
+        completed_groups_or_rollouts = 0
+        total_groups_or_rollouts = len(tasks)
         all_states: list[State] = []
-        try:
-            for coro in asyncio.as_completed(tasks.keys()):
-                result = await coro
-                # normalize: independent_scoring returns State, group returns list[State]
-                states = [result] if independent_scoring else result
-                all_states.extend(states)
-                groups_or_rollouts_completed += 1
-
-                # track reward for rolling average
-                for s in states:
-                    r = s.get("reward")
-                    if r is not None:
-                        reward_sum += r
-                        reward_count += 1
-
-                if pbar is not None:
-                    pbar.update(1)
-                    if reward_count > 0:
-                        pbar.set_postfix(reward=f"{reward_sum / reward_count:.3f}")
-
-                # save intermediate results
-                if (
-                    save_results
-                    and save_every > 0
-                    and groups_or_rollouts_completed % save_every == 0
-                ):
-                    temp_results = self._prepare_rollout_results(
-                        all_states,
-                        model,
-                        client,
-                        state_columns,
-                        results_path,
-                        gen_sampling_args,
-                        start_time,
-                    )
-                    self.logger.debug(
-                        f"Saving intermediate results to {temp_results['metadata']['path_to_save']}"
-                    )
-                    save_rollout_results(temp_results)
-        finally:
-            if pbar is not None:
-                pbar.close()
+        for coro in asyncio.as_completed(tasks.keys()):
+            result = await coro
+            # normalize: independent_scoring returns State, group returns list[State]
+            new_states = [result] if independent_scoring else result
+            all_states.extend(new_states)
+            completed_groups_or_rollouts += 1
+
+            # call progress callback with all finished states and new states
+            if on_progress is not None:
+                on_progress(all_states, new_states)
+
+            # save intermediate results
+            if (
+                save_results
+                and save_every > 0
+                and completed_groups_or_rollouts % save_every == 0
+            ):
+                temp_results = self._prepare_rollout_results(
+                    all_states,
+                    model,
+                    client,
+                    state_columns,
+                    results_path,
+                    gen_sampling_args,
+                    start_time,
+                )
+                on_log(
+                    f"Saving intermediate results ({completed_groups_or_rollouts}/{total_groups_or_rollouts} {('rollouts' if independent_scoring else 'groups')}) to {temp_results['metadata']['path_to_save']}"
+                )
+                save_rollout_results(temp_results)
 
         # sort by example_id to ensure deterministic ordering regardless of completion order
         all_states.sort(key=lambda s: s.get("example_id", 0))
@@ -969,9 +957,10 @@ async def generate(
             start_time,
         )
 
-        # Save if requested
+        # save if requested
         if save_results:
             save_rollout_results(results)
+            on_log(f"Saved final results to {results['metadata']['path_to_save']}")
 
         return results
 
@@ -1041,6 +1030,9 @@ async def evaluate(
         save_results: bool = False,
         save_every: int = -1,
         independent_scoring: bool = False,
+        on_start: StartCallback | None = None,
+        on_progress: ProgressCallback | None = None,
+        on_log: LogCallback | None = None,
         **kwargs,
     ) -> GenerateOutputs:
         """
@@ -1060,6 +1052,9 @@ async def evaluate(
             save_results=save_results,
             save_every=save_every,
             independent_scoring=independent_scoring,
+            on_start=on_start,
+            on_progress=on_progress,
+            on_log=on_log,
             **kwargs,
         )
 

verifiers/scripts/eval.py

@@ -20,6 +20,7 @@
     load_endpoints,
     load_toml_config,
     run_multi_evaluation,
+    run_multi_evaluation_tui,
 )
 
 logger = logging.getLogger(__name__)
@@ -250,6 +251,13 @@ def main():
         default={},
         help='Extra environment as JSON object (e.g., \'{"key": "value", "num": 42}\'). Passed to environment constructor.',
     )
+    parser.add_argument(
+        "--tui",
+        "-u",
+        default=False,
+        action="store_true",
+        help="Use TUI mode for live evaluation display",
+    )
     args = parser.parse_args()
 
     setup_logging("DEBUG" if args.verbose else os.getenv("VF_LOG_LEVEL", "INFO"))
@@ -415,7 +423,10 @@ def resolve_eval_config(raw_env_config: dict) -> EvalConfig:
         logger.debug(f"Evaluation config: {eval_config.model_dump_json(indent=2)}")
 
     multi_eval_config = MultiEvalConfig(env=eval_configs)
-    asyncio.run(run_multi_evaluation(multi_eval_config))
+    if args.tui:
+        asyncio.run(run_multi_evaluation_tui(multi_eval_config))
+    else:
+        asyncio.run(run_multi_evaluation(multi_eval_config))
 
 
 if __name__ == "__main__":

verifiers/types.py

@@ -139,6 +139,11 @@ def get(self, key: str, default: Any = None) -> Any:
 # oai tools
 JsonPrimitive = Literal["string", "number", "integer", "boolean", "array", "object"]
 
+# callbacks
+StartCallback = Callable[[int], None]  # total rollouts
+ProgressCallback = Callable[[list[State], list[State]], None]  # all_states, new_states
+LogCallback = Callable[[str], None]  # log messages
+
 
 class GenerateMetadata(TypedDict):
     """Pydantic model for generation metadata."""
@@ -237,6 +242,7 @@ class EvalConfig(BaseModel):
     extra_env_kwargs: dict = {}
     # logging
     verbose: bool = False
+    use_tqdm: bool = True
     # saving
     state_columns: list[str] | None = None
     save_results: bool = False