Add resume metadata, tests, and bump version

Author: miciav

docs/cli.md

@@ -29,6 +29,8 @@ Order used by commands that need a config:
   Dev-only: provision containers and run via Ansible (requires `.lb_dev_cli` or `LB_ENABLE_TEST_CLI=1`).
 - `lb run ... --multipass [--nodes N]`
   Dev-only: provision Multipass VMs and run via Ansible (requires `.lb_dev_cli` or `LB_ENABLE_TEST_CLI=1`).
+- `lb resume [RUN_ID] [-c FILE] [--root PATH] [--remote/--no-remote] [--docker|--multipass]`
+  Resume a previous run; without RUN_ID, pick interactively from `benchmark_results/`.
 - `lb runs list [--root PATH] [-c FILE]` / `lb runs show RUN_ID [--root PATH] [-c FILE]`
   Inspect stored runs under `benchmark_results/`.
 - `lb analyze [RUN_ID] [--kind aggregate] [--root PATH] [--workload NAME] [--host NAME]`

lb_app/client.py

@@ -45,9 +45,30 @@ def list_runs(self, config: BenchmarkConfig) -> Iterable[RunJournal]:
     def get_run_plan(self, config: BenchmarkConfig, tests: Sequence[str], execution_mode: str = "remote"):
         return self._run_service.get_run_plan(config, list(tests), execution_mode=execution_mode)
 
-    def _provision(self, config: BenchmarkConfig, execution_mode: str, node_count: int, docker_engine: str | None = None):
+    def _provision(
+        self,
+        config: BenchmarkConfig,
+        execution_mode: str,
+        node_count: int,
+        *,
+        docker_engine: str | None = None,
+        resume: str | None = None,
+    ):
         """Provision nodes according to execution mode; returns updated config and provisioner result."""
         mode = ProvisioningMode(execution_mode)
+        node_names = None
+        if resume and mode in (ProvisioningMode.DOCKER, ProvisioningMode.MULTIPASS):
+            node_names = self._resume_node_names(config, resume)
+            if not node_names:
+                raise ProvisioningError(
+                    "Unable to determine previous container/VM names for resume; "
+                    "ensure the run journal or host directories are available."
+                )
+            if node_count != len(node_names):
+                raise ProvisioningError(
+                    "Resume node count does not match original run; "
+                    "use --nodes to match the previous run."
+                )
         if mode is ProvisioningMode.REMOTE:
             request = ProvisioningRequest(
                 mode=ProvisioningMode.REMOTE,
@@ -58,13 +79,15 @@ def _provision(self, config: BenchmarkConfig, execution_mode: str, node_count: i
             request = ProvisioningRequest(
                 mode=ProvisioningMode.DOCKER,
                 count=node_count,
+                node_names=node_names,
                 docker_engine=docker_engine or "docker",
             )
         else:
             temp_dir = config.output_dir.parent / "temp_keys"
             request = ProvisioningRequest(
                 mode=ProvisioningMode.MULTIPASS,
                 count=node_count,
+                node_names=node_names,
                 state_dir=temp_dir,
             )
         result = self._provisioner.provision(request)
@@ -83,6 +106,49 @@ def _provision(self, config: BenchmarkConfig, execution_mode: str, node_count: i
         config.remote_execution.enabled = True
         return config, result
 
+    @staticmethod
+    def _resume_node_names(config: BenchmarkConfig, resume: str) -> list[str] | None:
+        from lb_app.services.run_journal import (
+            find_latest_journal,
+            find_latest_results_run,
+        )
+
+        run_root = None
+        journal_path = None
+        if resume == "latest":
+            journal_path = find_latest_journal(config)
+            if journal_path is not None:
+                run_root = journal_path.parent
+            else:
+                latest = find_latest_results_run(config)
+                if latest:
+                    journal_path = latest[1]
+                    run_root = journal_path.parent
+        else:
+            run_root = config.output_dir / resume
+            journal_path = run_root / "run_journal.json"
+
+        if journal_path is not None and journal_path.exists():
+            try:
+                journal = RunJournal.load(journal_path)
+            except Exception:
+                journal = None
+            if journal is not None:
+                names = sorted(
+                    {task.host for task in journal.tasks.values() if task.host}
+                )
+                if names:
+                    return names
+
+        if run_root is not None and run_root.exists():
+            names = sorted(
+                entry.name
+                for entry in run_root.iterdir()
+                if entry.is_dir() and not entry.name.startswith("_")
+            )
+            return names or None
+        return None
+
     def start_run(self, request: RunRequest, hooks: UIHooks) -> RunResult | None:
         cfg = request.config
         target_tests = list(
@@ -105,6 +171,7 @@ def start_run(self, request: RunRequest, hooks: UIHooks) -> RunResult | None:
             setup=request.setup,
             stop_file=request.stop_file,
             execution_mode=request.execution_mode,
+            node_count=request.node_count,
             preloaded_config=cfg,
         )
 
@@ -115,7 +182,8 @@ def start_run(self, request: RunRequest, hooks: UIHooks) -> RunResult | None:
                 cfg,
                 request.execution_mode,
                 request.node_count,
-                request.docker_engine,
+                docker_engine=request.docker_engine,
+                resume=request.resume,
             )
         except ProvisioningError as exc:
             hooks.on_warning(f"Provisioning failed: {exc}", ttl=5)

lb_app/services/run_service.py

@@ -135,6 +135,7 @@ def build_context(
         resume: Optional[str] = None,
         stop_file: Optional[Path] = None,
         execution_mode: str = "remote",
+        node_count: int | None = None,
     ) -> RunContext:
         """Compute the run context and registry."""
         registry = self._registry_factory()
@@ -151,6 +152,7 @@ def build_context(
             resume_latest=resume == "latest",
             stop_file=stop_file,
             execution_mode=execution_mode,
+            node_count=node_count,
         )
 
     def create_session(
@@ -167,6 +169,7 @@ def create_session(
         setup: bool = True,
         stop_file: Optional[Path] = None,
         execution_mode: str = "remote",
+        node_count: int | None = None,
         preloaded_config: BenchmarkConfig | None = None,
     ) -> RunContext:
         """
@@ -190,6 +193,7 @@ def create_session(
             resume=resume,
             stop_file=stop_file,
             execution_mode=execution_mode,
+            node_count=node_count,
         )
         return context
 
@@ -751,6 +755,16 @@ def _prepare_journal_and_dashboard(
                 context, run_id
             )
 
+        if journal.metadata is not None:
+            journal.metadata.setdefault("execution_mode", context.execution_mode)
+            node_count = context.node_count
+            if node_count is None:
+                if context.execution_mode in ("docker", "multipass"):
+                    node_count = max(1, len(context.config.remote_hosts or []))
+                else:
+                    node_count = len(context.config.remote_hosts or []) or 1
+            journal.metadata.setdefault("node_count", node_count)
+
         # Persist the initial state so resume is possible even if execution aborts early
         journal_path.parent.mkdir(parents=True, exist_ok=True)
         journal.save(journal_path)

lb_app/services/run_types.py

@@ -36,6 +36,7 @@ class RunContext:
     resume_latest: bool = False
     stop_file: Path | None = None
     execution_mode: str = "remote"
+    node_count: int | None = None
 
 
 @dataclass

lb_provisioner/engine/service.py

@@ -40,7 +40,8 @@ def __init__(
 
     def provision(self, request: ProvisioningRequest) -> ProvisioningResult:
         """Provision resources according to the request."""
-        self._enforce_limits(request.count)
+        effective_count = len(request.node_names) if request.node_names else request.count
+        self._enforce_limits(effective_count)
         self._assert_ui_caller()
 
         if request.mode is ProvisioningMode.REMOTE:

lb_provisioner/models/types.py

@@ -27,6 +27,7 @@ class ProvisioningRequest:
     mode: ProvisioningMode
     count: int = 1
     remote_hosts: Optional[List[RemoteHostSpec]] = None
+    node_names: Optional[List[str]] = None
     docker_engine: str = "docker"
     docker_image: str = "ubuntu:24.04"
     multipass_image: str = "24.04"

lb_provisioner/providers/docker.py

@@ -32,12 +32,20 @@ def provision(self, request: ProvisioningRequest) -> List[ProvisionedNode]:
         if not shutil.which(engine):
             raise ProvisioningError(f"{engine} not found in PATH")
 
-        count = max(1, min(request.count, MAX_NODES))
+        if request.node_names:
+            names = list(request.node_names)
+            count = len(names)
+        else:
+            names = []
+            count = max(1, min(request.count, MAX_NODES))
         state_root = request.state_dir or Path("/tmp/lb_docker_keys")
         state_root.mkdir(parents=True, exist_ok=True)
         nodes: List[ProvisionedNode] = []
         for idx in range(count):
-            name = f"lb-docker-{uuid.uuid4().hex[:8]}-{idx}"
+            if names:
+                name = names[idx]
+            else:
+                name = f"lb-docker-{uuid.uuid4().hex[:8]}-{idx}"
             key_path = state_root / f"{name}_id_rsa"
             pub_path = state_root / f"{name}_id_rsa.pub"
             self._generate_ssh_keypair(key_path)

lb_provisioner/providers/multipass.py

@@ -38,13 +38,18 @@ def provision(self, request: ProvisioningRequest) -> List[ProvisionedNode]:
         if not shutil.which("multipass"):
             raise ProvisioningError("Multipass CLI not found in PATH")
 
-        count = max(1, min(request.count, MAX_NODES))
+        if request.node_names:
+            names = list(request.node_names)
+            count = len(names)
+        else:
+            names = []
+            count = max(1, min(request.count, MAX_NODES))
         nodes: List[ProvisionedNode] = []
         state_root = request.state_dir or self.base_state_dir
         state_root.mkdir(parents=True, exist_ok=True)
 
-        for _ in range(count):
-            vm_name = f"lb-worker-{uuid.uuid4().hex[:8]}"
+        for idx in range(count):
+            vm_name = names[idx] if names else f"lb-worker-{uuid.uuid4().hex[:8]}"
             key_path = state_root / f"{vm_name}_id_rsa"
             pub_path = state_root / f"{vm_name}_id_rsa.pub"
             self._generate_ephemeral_keys(key_path)