fix: fixed evaluation restore logic

Author: evilsocket

nerve/cli/agents.py

@@ -1,6 +1,7 @@
 import pathlib
 import typing as t
 
+import natsort
 import requests
 import typer
 from termcolor import colored
@@ -74,7 +75,11 @@ def _show_installed_agents(path: pathlib.Path) -> None:
         print()
         print(f"📁 Installed in {path.absolute()}:\n")
 
+        items = []
         for item in path.iterdir():
+            items.append(item)
+
+        for item in natsort.natsorted(items):
             if Workflow.is_workflow(item):
                 workflow = Workflow.from_path(item)
                 print(

nerve/cli/eval.py

@@ -77,7 +77,7 @@ def eval(
             args.timeout = config.limits.timeout
 
     eval_name = colored(args.input_path.name, "green", attrs=["bold"])
-    logger.info(f"📊 {args.generator} vs {eval_name} | cases: {len(cases)} | runs: {runs}")
+    logger.info(f"📊 {args.generator} / {eval_name} / cases: {len(cases)} / runs: {runs}")
 
     if output.exists():
         logger.info(f"📊 loading evaluation results from {output}")
@@ -106,7 +106,7 @@ def eval(
                 run_output = asyncio.run(_run_case(args, case))
                 evaluation.add_run(case.name, run_output)
 
-            _show_run(run_output, run + 1, runs, eval_name, case.name)
+            _show_run(args, run_output, runs, run, case.name, do_run)
 
             if evaluation.needs_flush():
                 # save at each run so we can restore later
@@ -122,16 +122,17 @@ def eval(
     _show_results(evaluation)
 
 
-def _show_run(output: Output, run: int, runs: int, eval_name: str, case_name: str) -> None:
+def _show_run(args: Arguments, output: Output, runs: int, run: int, case_name: str, live: bool) -> None:
     usage = output.usage
+    one_of = f"[{run + 1}/{runs}]" if live else f"({run + 1}/{runs})"
+    subject = f"{one_of} {args.generator} / {args.input_path.name} / {case_name}"
+    stats = (
+        f"{output.steps} steps, {output.time:.1f} s, {usage.get('total_tokens', 0)} tokens, {usage.get('cost', 0.0)} $"
+    )
     if output.task_success:
-        logger.success(
-            f"   [{run + 1}/{runs}] {eval_name} / {case_name} : {output.steps} steps | {output.time:.1f} s | {usage.get('total_tokens', 0)} tokens | {usage.get('cost', 0.0)} $"
-        )
+        logger.success(f"   {subject} : {stats}")
     else:
-        logger.error(
-            f"     [{run + 1}/{runs}] {eval_name} / {case_name} : {output.steps} steps | {output.time:.1f} s | {usage.get('total_tokens', 0)} tokens | {usage.get('cost', 0.0)} $"
-        )
+        logger.error(f"     {subject} : {stats}")
 
 
 def _show_results(eval: Evaluation) -> None:

nerve/runtime/eval.py

@@ -78,7 +78,7 @@ def remove_run(self, case_name: str, run_idx: int) -> None:
         self._flush = True
 
     def num_runs(self, case_name: str) -> int:
-        return len(self.runs[case_name])
+        return len(self.runs[case_name]) if case_name in self.runs else 0
 
     def num_run_steps(self, case_name: str, run_idx: int) -> int:
         return self.runs[case_name][run_idx].steps

nerve/runtime/runner.py

@@ -208,13 +208,17 @@ def __init__(
         )
         self._stdout_fn: t.Callable[[str], t.Awaitable[None]] = _default_stdout_fn
         self._stderr_fn: t.Callable[[str], t.Awaitable[None]] = _default_stderr_fn
+        self._process: asyncio.subprocess.Process | None = None
 
         if clean_at_exit:
             atexit.register(self._clean_up)
 
     def _clean_up(self) -> None:
         if self.events_file.exists():
             logger.debug(f"removing events file {self.events_file}")
+            if self._process is not None:
+                self._process.kill()
+                self._process = None
             self.events_file.unlink()
 
     def set_stdout_fn(self, fn: t.Callable[[str], t.Awaitable[None]]) -> None:
@@ -247,20 +251,20 @@ async def read_stream(stream: asyncio.StreamReader | None, name: str) -> None:
 
                 outerr[name].append(line.decode().rstrip())
 
-        process = await asyncio.create_subprocess_exec(
+        self._process = await asyncio.create_subprocess_exec(
             *self.command_line,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             env=os.environ.copy(),
         )
-        stdout_task = asyncio.create_task(read_stream(process.stdout, "stdout"))
-        stderr_task = asyncio.create_task(read_stream(process.stderr, "stderr"))
+        stdout_task = asyncio.create_task(read_stream(self._process.stdout, "stdout"))
+        stderr_task = asyncio.create_task(read_stream(self._process.stderr, "stderr"))
 
         # wait for the process and stdout/stderr readers to complete
-        await process.wait()
+        await self._process.wait()
         await asyncio.gather(stdout_task, stderr_task)
 
-        logger.debug(f"process exited with code {process.returncode}, reading events ...")
+        logger.debug(f"process exited with code {self._process.returncode}, reading events ...")
 
         # read the events file
         events = []
@@ -285,10 +289,13 @@ async def read_stream(stream: asyncio.StreamReader | None, name: str) -> None:
 
         logger.debug(f"output value: {parsed.output_object}")
 
+        exit_code = self._process.returncode or 0
+        self._process = None
+
         return Output(
             generated_at=generated_at,
             command_line=self.command_line,
-            exit_code=process.returncode or 0,
+            exit_code=exit_code,
             stdout=outerr["stdout"],
             stderr=outerr["stderr"],
             events=events,