feat: auto-detect pre-built images across all benchmarks

Add local_image_exists() to image_utils — checks if a Docker image
exists in the local daemon. Replace all SKIP_BUILD env var checks
across every benchmark with auto-detection: if the expected
agent-server image exists locally, skip building; otherwise build
on-the-fly as before.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: simonrosenberg

benchmarks/commit0/run_infer.py

@@ -24,7 +24,7 @@
     construct_eval_output_dir,
     get_default_on_result_writer,
 )
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -36,7 +36,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace, DockerWorkspace
 
 
 logger = get_logger(__name__)
@@ -188,16 +188,26 @@ def prepare_workspace(
         logger.info(f"Using base docker image: {base_docker_image}")
 
         if self.metadata.workspace_type == "docker":
-            # Build agent-server image from base commit0 image
-            workspace = DockerDevWorkspace(
-                base_image=base_docker_image,
-                working_dir="/workspace",
-                target=build_target,
-                forward_env=forward_env or [],
-            )
-            logger.info(
-                f"Building workspace from {base_docker_image}. This may take a while..."
+            custom_tag = extract_custom_tag(base_docker_image)
+            suffix = f"-{build_target}" if build_target != "binary" else ""
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
+            if local_image_exists(agent_server_image):
+                logger.info(f"Using pre-built image {agent_server_image}")
+                workspace = DockerWorkspace(
+                    server_image=agent_server_image,
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
+            else:
+                logger.info(f"Building workspace from {base_docker_image}...")
+                workspace = DockerDevWorkspace(
+                    base_image=base_docker_image,
+                    working_dir="/workspace",
+                    target=build_target,
+                    forward_env=forward_env or [],
+                )
         elif self.metadata.workspace_type == "remote":
             runtime_api_key = os.getenv("RUNTIME_API_KEY")
             if not runtime_api_key:

benchmarks/gaia/run_infer.py

@@ -27,7 +27,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import EvalInstance, EvalMetadata, EvalOutput
 from benchmarks.utils.version import SDK_SHORT_SHA
@@ -47,7 +47,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace, DockerWorkspace
 
 
 logger = get_logger(__name__)
@@ -156,12 +156,23 @@ def prepare_workspace(
         logger.info(f"Preparing workspace for instance {instance.id}")
 
         if self.metadata.workspace_type == "docker":
-            # Use DockerDevWorkspace with base image (same as main branch)
-            workspace = DockerDevWorkspace(
-                base_image="nikolaik/python-nodejs:python3.12-nodejs22",
-                working_dir="/workspace",
-                forward_env=forward_env or [],
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-gaia-binary"
             )
+            if local_image_exists(agent_server_image):
+                logger.info(f"Using pre-built image {agent_server_image}")
+                workspace = DockerWorkspace(
+                    server_image=agent_server_image,
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
+            else:
+                logger.info("Building workspace from nikolaik/python-nodejs:python3.12-nodejs22...")
+                workspace = DockerDevWorkspace(
+                    base_image="nikolaik/python-nodejs:python3.12-nodejs22",
+                    working_dir="/workspace",
+                    forward_env=forward_env or [],
+                )
         elif self.metadata.workspace_type == "remote":
             # For workflow, use APIRemoteWorkspace with pre-built GAIA image
             # GAIA uses a universal agent server image (one image for all instances)

benchmarks/multiswebench/run_infer.py

@@ -26,7 +26,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -212,20 +212,10 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("MULTI_SWE_BENCH_SKIP_BUILD", "0").lower() in (
-                "1",
-                "true",
-                "yes",
-            )
-            logger.info(f"MULTI_SWE_BENCH_SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
+            if not local_image_exists(agent_server_image):
                 logger.info(
                     f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/multiswebench/build_images.py and set "
-                    "MULTI_SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
+                    f"for instance {instance.id}. This may take a while..."
                 )
                 output = build_image(
                     base_image=official_docker_image,

benchmarks/swebench/run_infer.py

@@ -26,7 +26,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -160,16 +160,10 @@ def prepare_workspace(
         agent_server_image = base_agent_image
 
         if self.metadata.workspace_type == "docker":
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
+            if not local_image_exists(agent_server_image):
                 logger.info(
                     f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swebench/build_images.py and set "
-                    "SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
+                    f"for instance {instance.id}. This may take a while..."
                 )
                 output = build_image(
                     base_image=official_docker_image,

benchmarks/swebenchmultimodal/run_infer.py

@@ -24,7 +24,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -165,16 +165,10 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
+            if not local_image_exists(agent_server_image):
                 logger.info(
                     f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swebenchmultimodal/build_images.py and set "
-                    "SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
+                    f"for instance {instance.id}. This may take a while..."
                 )
 
                 output = build_image(

benchmarks/swefficiency/run_infer.py

@@ -21,7 +21,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.models import (
     EvalInstance,
     EvalMetadata,
@@ -208,11 +208,7 @@ def prepare_workspace(
         logger.info(f"Agent server image: {agent_server_image}")
 
         if self.metadata.workspace_type == "docker":
-            # Build agent-server image from base swefficiency image
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "0").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-
-            if not SKIP_BUILD:
+            if not local_image_exists(agent_server_image):
                 logger.info(
                     f"Building workspace from {base_docker_image} "
                     f"for instance {instance.id}. "

benchmarks/swtbench/run_infer.py

@@ -18,7 +18,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import image_exists, local_image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -171,28 +171,19 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
-                logger.info(
-                    f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swtbench/build_images.py and set "
-                    "SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
-                )
-                # For SWT-bench, we use DockerDevWorkspace with base_image
-                workspace = DockerDevWorkspace(
-                    base_image=official_docker_image,
+            if local_image_exists(agent_server_image):
+                logger.info(f"Using pre-built image {agent_server_image}")
+                workspace = DockerWorkspace(
+                    server_image=agent_server_image,
                     working_dir="/workspace",
-                    target=build_target,
                     forward_env=forward_env or [],
                 )
             else:
-                workspace = DockerWorkspace(
-                    server_image=agent_server_image,
+                logger.info(f"Building workspace from {official_docker_image}...")
+                workspace = DockerDevWorkspace(
+                    base_image=official_docker_image,
                     working_dir="/workspace",
+                    target=build_target,
                     forward_env=forward_env or [],
                 )
         elif self.metadata.workspace_type == "remote":

benchmarks/utils/image_utils.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import base64
+import subprocess
 import sys
 
 import requests
@@ -54,6 +55,15 @@ def _ghcr_token(repo: str, username: str | None, pat: str | None) -> str | None:
     return None
 
 
+def local_image_exists(image: str) -> bool:
+    """Check if a Docker image exists in the local Docker daemon."""
+    result = subprocess.run(
+        ["docker", "image", "inspect", image],
+        capture_output=True,
+    )
+    return result.returncode == 0
+
+
 def image_exists(
     image_ref: str,
     gh_username: str | None = None,