feat: auto-detect pre-built Docker images across all benchmarks

simonrosenberg · claude · simonrosenberg · commit 31454bf6561d · 2026-02-27T07:12:56.000-03:00
Add local_image_exists() and create_docker_workspace() to image_utils,
and ensure_local_image() to build_utils. These centralize the pattern
of checking for a pre-built image before building on-the-fly.

Replace all SKIP_BUILD env var checks across every benchmark:
- commit0, gaia, swtbench: use create_docker_workspace()
- swebench, swebenchmultimodal, multiswebench, swefficiency: use
  ensure_local_image()

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/commit0/run_infer.py b/benchmarks/commit0/run_infer.py
@@ -24,7 +24,7 @@
     construct_eval_output_dir,
     get_default_on_result_writer,
 )
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import create_docker_workspace, image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -36,7 +36,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace
 
 
 logger = get_logger(__name__)
@@ -188,15 +188,16 @@ def prepare_workspace(
         logger.info(f"Using base docker image: {base_docker_image}")
 
         if self.metadata.workspace_type == "docker":
-            # Build agent-server image from base commit0 image
-            workspace = DockerDevWorkspace(
-                base_image=base_docker_image,
-                working_dir="/workspace",
-                target=build_target,
-                forward_env=forward_env or [],
+            custom_tag = extract_custom_tag(base_docker_image)
+            suffix = f"-{build_target}" if build_target != "binary" else ""
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            logger.info(
-                f"Building workspace from {base_docker_image}. This may take a while..."
+            workspace = create_docker_workspace(
+                agent_server_image=agent_server_image,
+                base_image=base_docker_image,
+                build_target=build_target,
+                forward_env=forward_env,
             )
         elif self.metadata.workspace_type == "remote":
             runtime_api_key = os.getenv("RUNTIME_API_KEY")
diff --git a/benchmarks/gaia/run_infer.py b/benchmarks/gaia/run_infer.py
@@ -27,7 +27,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import create_docker_workspace, image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import EvalInstance, EvalMetadata, EvalOutput
 from benchmarks.utils.version import SDK_SHORT_SHA
@@ -47,7 +47,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace
+from openhands.workspace import APIRemoteWorkspace
 
 
 logger = get_logger(__name__)
@@ -156,11 +156,14 @@ def prepare_workspace(
         logger.info(f"Preparing workspace for instance {instance.id}")
 
         if self.metadata.workspace_type == "docker":
-            # Use DockerDevWorkspace with base image (same as main branch)
-            workspace = DockerDevWorkspace(
+            agent_server_image = (
+                f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-gaia-binary"
+            )
+            workspace = create_docker_workspace(
+                agent_server_image=agent_server_image,
                 base_image="nikolaik/python-nodejs:python3.12-nodejs22",
-                working_dir="/workspace",
-                forward_env=forward_env or [],
+                build_target="binary",
+                forward_env=forward_env,
             )
         elif self.metadata.workspace_type == "remote":
             # For workflow, use APIRemoteWorkspace with pre-built GAIA image
diff --git a/benchmarks/multiswebench/run_infer.py b/benchmarks/multiswebench/run_infer.py
@@ -14,7 +14,7 @@
 from benchmarks.multiswebench.download_dataset import download_and_concat_dataset
 from benchmarks.multiswebench.scripts.data.data_change import format_data_for_inference
 from benchmarks.utils.args_parser import get_parser
-from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.build_utils import ensure_local_image
 from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
@@ -212,36 +212,14 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("MULTI_SWE_BENCH_SKIP_BUILD", "0").lower() in (
-                "1",
-                "true",
-                "yes",
+            built = ensure_local_image(
+                agent_server_image=agent_server_image,
+                base_image=official_docker_image,
+                custom_tag=custom_tag,
+                target=build_target,
             )
-            logger.info(f"MULTI_SWE_BENCH_SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
-                logger.info(
-                    f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/multiswebench/build_images.py and set "
-                    "MULTI_SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
-                )
-                output = build_image(
-                    base_image=official_docker_image,
-                    target_image=EVAL_AGENT_SERVER_IMAGE,
-                    custom_tag=custom_tag,
-                    target=build_target,
-                    push=False,
-                )
-                logger.info(f"Image build output: {output}")
-                assert output.error is None, f"Image build failed: {output.error}"
-                if agent_server_image not in output.tags:
-                    raise RuntimeError(
-                        f"Built image tags {output.tags} do not include expected tag "
-                        f"{agent_server_image}"
-                    )
-
+            if not built:
+                logger.info(f"Using pre-built image {agent_server_image}")
             workspace = DockerWorkspace(
                 server_image=agent_server_image,
                 working_dir="/workspace",
diff --git a/benchmarks/swebench/run_infer.py b/benchmarks/swebench/run_infer.py
@@ -14,7 +14,7 @@
 )
 from benchmarks.swebench.config import INFER_DEFAULTS
 from benchmarks.utils.args_parser import get_parser
-from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.build_utils import ensure_local_image
 from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
@@ -160,38 +160,24 @@ def prepare_workspace(
         agent_server_image = base_agent_image
 
         if self.metadata.workspace_type == "docker":
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
+            built = ensure_local_image(
+                agent_server_image=base_agent_image,
+                base_image=official_docker_image,
+                custom_tag=custom_tag,
+                target=build_target,
+            )
+            if built and wrap_needed:
+                wrapped_result = wrap_image(base_agent_image, push=False)
+            elif not built and wrap_needed:
                 logger.info(
-                    f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swebench/build_images.py and set "
-                    "SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
-                )
-                output = build_image(
-                    base_image=official_docker_image,
-                    target_image=EVAL_AGENT_SERVER_IMAGE,
-                    custom_tag=custom_tag,
-                    target=build_target,
-                    push=False,
+                    f"Using pre-built image {base_agent_image} "
+                    "(assumed already wrapped)"
                 )
-                logger.info(f"Image build output: {output}")
-                assert output.error is None, f"Image build failed: {output.error}"
-                if base_agent_image not in output.tags:
+                if wrapped_result.error:
                     raise RuntimeError(
-                        f"Built image tags {output.tags} do not include expected tag "
-                        f"{base_agent_image}"
+                        "Wrapped image build failed: "
+                        f"{wrapped_result.error}; log={wrapped_result.log_path}"
                     )
-                if wrap_needed:
-                    wrapped_result = wrap_image(base_agent_image, push=False)
-                    if wrapped_result.error:
-                        raise RuntimeError(
-                            "Wrapped image build failed: "
-                            f"{wrapped_result.error}; log={wrapped_result.log_path}"
-                        )
 
             workspace = DockerWorkspace(
                 server_image=agent_server_image,
diff --git a/benchmarks/swebenchmultimodal/run_infer.py b/benchmarks/swebenchmultimodal/run_infer.py
@@ -12,7 +12,7 @@
 )
 from benchmarks.swebenchmultimodal.config import INFER_DEFAULTS
 from benchmarks.utils.args_parser import get_parser
-from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.build_utils import ensure_local_image
 from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
@@ -165,33 +165,14 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
-                logger.info(
-                    f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swebenchmultimodal/build_images.py and set "
-                    "SWE_BENCH_SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
-                )
-
-                output = build_image(
-                    base_image=official_docker_image,
-                    target_image=EVAL_AGENT_SERVER_IMAGE,
-                    custom_tag=custom_tag,
-                    target=build_target,
-                    push=False,
-                )
-                logger.info(f"Image build output: {output}")
-                assert output.error is None, f"Image build failed: {output.error}"
-                if agent_server_image not in output.tags:
-                    raise RuntimeError(
-                        f"Built image tags {output.tags} do not include expected tag "
-                        f"{agent_server_image}"
-                    )
-
+            built = ensure_local_image(
+                agent_server_image=agent_server_image,
+                base_image=official_docker_image,
+                custom_tag=custom_tag,
+                target=build_target,
+            )
+            if not built:
+                logger.info(f"Using pre-built image {agent_server_image}")
             workspace = DockerWorkspace(
                 server_image=agent_server_image,
                 working_dir="/workspace",
diff --git a/benchmarks/swefficiency/run_infer.py b/benchmarks/swefficiency/run_infer.py
@@ -11,7 +11,7 @@
 from benchmarks.swefficiency.config import DOCKER_DEFAULTS, INFER_DEFAULTS
 from benchmarks.swefficiency.workspace import ResourceLimitedDockerWorkspace
 from benchmarks.utils.args_parser import get_parser
-from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.build_utils import ensure_local_image
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
 from benchmarks.utils.dataset import get_dataset
@@ -208,30 +208,14 @@ def prepare_workspace(
         logger.info(f"Agent server image: {agent_server_image}")
 
         if self.metadata.workspace_type == "docker":
-            # Build agent-server image from base swefficiency image
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "0").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-
-            if not SKIP_BUILD:
-                logger.info(
-                    f"Building workspace from {base_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while..."
-                )
-                output = build_image(
-                    base_image=base_docker_image,
-                    target_image=EVAL_AGENT_SERVER_IMAGE,
-                    custom_tag=custom_tag,
-                    target=build_target,
-                    push=False,
-                )
-                logger.info(f"Image build output: {output}")
-                assert output.error is None, f"Image build failed: {output.error}"
-                if agent_server_image not in output.tags:
-                    raise RuntimeError(
-                        f"Built image tags {output.tags} do not include expected tag "
-                        f"{agent_server_image}"
-                    )
+            built = ensure_local_image(
+                agent_server_image=agent_server_image,
+                base_image=base_docker_image,
+                custom_tag=custom_tag,
+                target=build_target,
+            )
+            if not built:
+                logger.info(f"Using pre-built image {agent_server_image}")
 
             # Get CPU group for resource limiting
             cpu_group = self._acquire_cpu_group()
diff --git a/benchmarks/swtbench/run_infer.py b/benchmarks/swtbench/run_infer.py
@@ -18,7 +18,7 @@
     get_default_on_result_writer,
 )
 from benchmarks.utils.fake_user_response import run_conversation_with_fake_user_response
-from benchmarks.utils.image_utils import image_exists
+from benchmarks.utils.image_utils import create_docker_workspace, image_exists
 from benchmarks.utils.llm_config import load_llm_config
 from benchmarks.utils.models import (
     EvalInstance,
@@ -31,7 +31,7 @@
 from openhands.sdk.workspace import RemoteWorkspace
 from openhands.tools.delegate import DelegateTool
 from openhands.tools.preset.default import get_default_tools
-from openhands.workspace import APIRemoteWorkspace, DockerDevWorkspace, DockerWorkspace
+from openhands.workspace import APIRemoteWorkspace
 
 
 logger = get_logger(__name__)
@@ -171,30 +171,12 @@ def prepare_workspace(
             agent_server_image = (
                 f"{EVAL_AGENT_SERVER_IMAGE}:{SDK_SHORT_SHA}-{custom_tag}{suffix}"
             )
-            SKIP_BUILD = os.getenv("SKIP_BUILD", "1").lower() in ("1", "true", "yes")
-            logger.info(f"SKIP_BUILD={SKIP_BUILD}")
-            if not SKIP_BUILD:
-                logger.info(
-                    f"Building workspace from {official_docker_image} "
-                    f"for instance {instance.id}. "
-                    "This may take a while...\n"
-                    "You can run benchmarks/swtbench/build_images.py and set "
-                    "SKIP_BUILD=1 to skip building and use pre-built "
-                    "agent-server image."
-                )
-                # For SWT-bench, we use DockerDevWorkspace with base_image
-                workspace = DockerDevWorkspace(
-                    base_image=official_docker_image,
-                    working_dir="/workspace",
-                    target=build_target,
-                    forward_env=forward_env or [],
-                )
-            else:
-                workspace = DockerWorkspace(
-                    server_image=agent_server_image,
-                    working_dir="/workspace",
-                    forward_env=forward_env or [],
-                )
+            workspace = create_docker_workspace(
+                agent_server_image=agent_server_image,
+                base_image=official_docker_image,
+                build_target=build_target,
+                forward_env=forward_env,
+            )
         elif self.metadata.workspace_type == "remote":
             runtime_api_key = os.getenv("RUNTIME_API_KEY")
             sdk_short_sha = os.getenv("SDK_SHORT_SHA", SDK_SHORT_SHA)
diff --git a/benchmarks/utils/build_utils.py b/benchmarks/utils/build_utils.py
@@ -307,6 +307,46 @@ def build_image(
     return BuildOutput(base_image=base_image, tags=tags, error=None)
 
 
+def ensure_local_image(
+    agent_server_image: str,
+    base_image: str,
+    custom_tag: str,
+    target: TargetType = "source-minimal",
+) -> bool:
+    """Build an agent-server image locally if it doesn't already exist.
+
+    Returns True if a build occurred, False if the image already existed.
+    Set FORCE_BUILD=1 to skip auto-detection and always rebuild.
+    """
+    from benchmarks.utils.image_utils import local_image_exists
+
+    force_build = os.getenv("FORCE_BUILD", "0").lower() in ("1", "true", "yes")
+    if not force_build and local_image_exists(agent_server_image):
+        logger.info(f"Using pre-built image {agent_server_image}")
+        return False
+
+    if force_build:
+        logger.info(f"FORCE_BUILD set, building image from {base_image}...")
+    else:
+        logger.info(f"Building image from {base_image}...")
+    output = build_image(
+        base_image=base_image,
+        target_image=EVAL_AGENT_SERVER_IMAGE,
+        custom_tag=custom_tag,
+        target=target,
+        push=False,
+    )
+    logger.info(f"Image build output: {output}")
+    if output.error is not None:
+        raise RuntimeError(f"Image build failed: {output.error}")
+    if agent_server_image not in output.tags:
+        raise RuntimeError(
+            f"Built image tags {output.tags} do not include expected tag "
+            f"{agent_server_image}"
+        )
+    return True
+
+
 def _build_with_logging(
     log_dir: Path,
     base_image: str,
diff --git a/benchmarks/utils/image_utils.py b/benchmarks/utils/image_utils.py
diff --git a/tests/test_image_utils.py b/tests/test_image_utils.py
