feat: add pre-built image auto-detection to commit0 and gaia benchmarks

Add local_image_exists() to image_utils and use it in commit0/gaia
docker workspace paths. If the expected agent-server image already
exists locally, use DockerWorkspace directly; otherwise fall back to
building on-the-fly with DockerDevWorkspace as before.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: simonrosenberg

benchmarks/commit0/run_infer.py

@@ -24,7 +24,7 @@
     construct_eval_output_dir,
     get_default_on_result_writer,
 )
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -36,7 +36,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace, DockerWorkspace
 
 
 logger = get_logger(__name__)
@@ -188,16 +188,26 @@ def prepare_workspace(
         logger.info(f"Using base docker image: {base_docker_image}")
 
         if self.metadata.workspace_type == "docker":
-            # Build agent-server image from base commit0 image
-            workspace = DockerDevWorkspace(
-                base_image=base_docker_image,
-                working_dir="/workspace",
-                target=build_target,
-                forward_env=forward_env or [],
-            )
-            logger.info(
-                f"Building workspace from {base_docker_image}. This may take a while..."
+            custom_tag = extract_custom_tag(base_docker_image)
+            suffix = f"-{build_target}" if build_target != "binary" else ""
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
+            if local_image_exists(agent_server_image):
+                logger.info(f"Using pre-built image {agent_server_image}")
+                workspace = DockerWorkspace(
+                    server_image=agent_server_image,
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
+            else:
+                logger.info(f"Building workspace from {base_docker_image}...")
+                workspace = DockerDevWorkspace(
+                    base_image=base_docker_image,
+                    working_dir="/workspace",
+                    target=build_target,
+                    forward_env=forward_env or [],
+                )
         elif self.metadata.workspace_type == "remote":
             runtime_api_key = os.getenv("RUNTIME_API_KEY")
             if not runtime_api_key:

benchmarks/gaia/run_infer.py

@@ -27,7 +27,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import EvalInstance, EvalMetadata, EvalOutput
 from benchmarks.utils.version import SDK_SHORT_SHA
@@ -47,7 +47,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace, DockerWorkspace
 
 
 logger = get_logger(__name__)
@@ -156,12 +156,23 @@ def prepare_workspace(
         logger.info(f"Preparing workspace for instance {instance.id}")
 
         if self.metadata.workspace_type == "docker":
-            # Use DockerDevWorkspace with base image (same as main branch)
-            workspace = DockerDevWorkspace(
-                base_image="nikolaik/python-nodejs:python3.12-nodejs22",
-                working_dir="/workspace",
-                forward_env=forward_env or [],
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-gaia-binary"
             )
+            if local_image_exists(agent_server_image):
+                logger.info(f"Using pre-built image {agent_server_image}")
+                workspace = DockerWorkspace(
+                    server_image=agent_server_image,
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
+            else:
+                logger.info("Building workspace from nikolaik/python-nodejs:python3.12-nodejs22...")
+                workspace = DockerDevWorkspace(
+                    base_image="nikolaik/python-nodejs:python3.12-nodejs22",
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
         elif self.metadata.workspace_type == "remote":
             # For workflow, use APIRemoteWorkspace with pre-built GAIA image
             # GAIA uses a universal agent server image (one image for all instances)

benchmarks/utils/image_utils.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import base64
+import subprocess
 import sys
 
 import requests
@@ -54,6 +55,15 @@ def _ghcr_token(repo: str, username: str | None, pat: str | None) -> str | None:
     return None
 
 
+def local_image_exists(image: str) -> bool:
+    """Check if a Docker image exists in the local Docker daemon."""
+    result = subprocess.run(
+        ["docker", "image", "inspect", image],
+        capture_output=True,
+    )
+    return result.returncode == 0
+
+
 def image_exists(
     image_ref: str,
     gh_username: str | None = None,