fix: pin SDK to proven-working SHA and fix build regression (#504)

Three-part fix for the 500-image build regression introduced in #456:

1. Pin SDK submodule to 30819566 (proven in 34-min 500-image build)
   - The regression between bde715c1 and this SHA has been fixed upstream
   - Images with this SHA already exist in the GHCR registry
   - Restores fast warm-cache builds (~34 minutes vs 5+ hours)

2. Replace openhands.sdk.get_logger with stdlib logging in build modules
   - build_utils.py, buildx_utils.py, image_utils.py, build_images.py
   - Prevents Rich console state from being inherited across
     ProcessPoolExecutor forks (deadlock fix)

3. Add cold-cache survivability improvements to CI workflows
   - timeout-minutes: 180 on both swebench and swtbench build jobs
   - Post-build disk/timing instrumentation for observability
   - Preflight BuildKit prune + disk check for swtbench (was missing)
   - BUILDKIT_RESET_ON_FAILURE for swtbench build step

Fixes #504
Refs #502, #503

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: simonrosenberg

.github/workflows/build-swebench-images.yml

@@ -78,6 +78,7 @@ jobs:
 
     runs-on:
       labels: blacksmith-32vcpu-ubuntu-2204
+    timeout-minutes: 180
 
     # Allow pushing to GHCR and commenting on issues
     permissions:
@@ -86,6 +87,11 @@ jobs:
       issues: write
 
     steps:
+      - name: Record build start time
+        run: |
+          echo "BUILD_START=$(date +%s)" >> "$GITHUB_ENV"
+          echo "Build started at $(date -u)"
+
       - name: Determine checkout ref
         id: checkout-ref
         run: |
@@ -249,6 +255,21 @@ jobs:
           BUILDKIT_PROGRESS: plain
           BUILDKIT_RESET_ON_FAILURE: 1
 
+      - name: Post-build disk and timing report
+        if: always()
+        run: |
+          set -euo pipefail
+          BUILD_END=$(date +%s)
+          ELAPSED=$(( BUILD_END - ${BUILD_START:-$BUILD_END} ))
+          echo "## Build Timing" >> "$GITHUB_STEP_SUMMARY"
+          echo "**Elapsed:** $((ELAPSED / 60))m $((ELAPSED % 60))s" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "## Disk Usage After Build" >> "$GITHUB_STEP_SUMMARY"
+          df -h / /var/lib/buildkit 2>/dev/null | tee -a "$GITHUB_STEP_SUMMARY" || true
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          docker buildx du --verbose 2>/dev/null | head -40 | tee -a "$GITHUB_STEP_SUMMARY" || true
+
       - name: Archive build logs
         if: always()
         run: |

.github/workflows/build-swtbench-images.yml

@@ -72,6 +72,7 @@ jobs:
 
     runs-on:
       labels: blacksmith-32vcpu-ubuntu-2204
+    timeout-minutes: 180
 
     permissions:
       contents: read
@@ -88,6 +89,11 @@ jobs:
       SELECT_FILE: ''
 
     steps:
+      - name: Record build start time
+        run: |
+          echo "BUILD_START=$(date +%s)" >> "$GITHUB_ENV"
+          echo "Build started at $(date -u)"
+
       - name: Determine checkout ref
         id: checkout-ref
         run: |
@@ -134,6 +140,35 @@ jobs:
         run: |
           make build
 
+      - name: "Preflight: prune cache and verify BuildKit disk"
+        run: |
+          set -euo pipefail
+          KEEP_GB=60
+          echo "Pruning BuildKit cache (target max-storage ${KEEP_GB} GiB, no filters)..."
+          if ! docker buildx prune --all --force --max-storage ${KEEP_GB}g; then
+            docker buildx prune --all --force --keep-storage ${KEEP_GB}g || true
+          fi
+
+          if df -B1 /var/lib/buildkit > /tmp/buildkit_df 2>/dev/null; then
+            LINE=$(tail -n1 /tmp/buildkit_df)
+            TOTAL=$(echo "$LINE" | awk '{print $2}')
+            USED=$(echo "$LINE" | awk '{print $3}')
+            FREE=$(echo "$LINE" | awk '{print $4}')
+            if [ -n "$TOTAL" ] && [ -n "$FREE" ]; then
+              PCT=$(( 100 * USED / TOTAL ))
+              echo "BuildKit disk: used ${USED} / ${TOTAL} bytes (${PCT}%); free ${FREE} bytes"
+              MIN=$((75 * 1024 * 1024 * 1024))
+              if [ "$FREE" -lt "$MIN" ]; then
+                echo "::error::Not enough free space on /var/lib/buildkit (${FREE} bytes free, need >= ${MIN})"
+                exit 1
+              fi
+            else
+              echo "Warning: unable to parse df output for /var/lib/buildkit"
+            fi
+          else
+            echo "Warning: /var/lib/buildkit not found; skipping disk check"
+          fi
+
       - name: Build and push SWT-Bench images
         run: |
           set -euo pipefail
@@ -181,6 +216,7 @@ jobs:
         env:
           DOCKER_BUILDKIT: 1
           BUILDKIT_PROGRESS: plain
+          BUILDKIT_RESET_ON_FAILURE: 1
 
       - name: Build prebaked eval env images
         if: ${{ inputs.build-eval-env == 'true' }}
@@ -240,6 +276,21 @@ jobs:
           docker ps -a || true
           docker system df || true
 
+      - name: Post-build disk and timing report
+        if: always()
+        run: |
+          set -euo pipefail
+          BUILD_END=$(date +%s)
+          ELAPSED=$(( BUILD_END - ${BUILD_START:-$BUILD_END} ))
+          echo "## Build Timing" >> "$GITHUB_STEP_SUMMARY"
+          echo "**Elapsed:** $((ELAPSED / 60))m $((ELAPSED % 60))s" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "## Disk Usage After Build" >> "$GITHUB_STEP_SUMMARY"
+          df -h / /var/lib/buildkit 2>/dev/null | tee -a "$GITHUB_STEP_SUMMARY" || true
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          docker buildx du --verbose 2>/dev/null | head -40 | tee -a "$GITHUB_STEP_SUMMARY" || true
+
       - name: Archive build logs
         if: always()
         run: |

benchmarks/swebench/build_images.py

@@ -23,10 +23,12 @@
 )
 from benchmarks.utils.dataset import get_dataset
 from benchmarks.utils.image_utils import remote_image_exists
-from openhands.sdk import get_logger
 
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 WRAPPER_DOCKERFILE = Path(__file__).with_name("Dockerfile.swebench-deps")
 
 

benchmarks/swtbench/build_eval_env_images.py

@@ -13,10 +13,12 @@
 from benchmarks.swtbench.image_utils import ensure_swt_bench_repo
 from benchmarks.utils.dataset import get_dataset
 from benchmarks.utils.image_utils import remote_image_exists
-from openhands.sdk import get_logger
 
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def select_instance_ids(

benchmarks/swtbench/image_utils.py

@@ -9,10 +9,12 @@
 from typing import Iterable
 
 from benchmarks.swtbench.config import EVAL_DEFAULTS
-from openhands.sdk import get_logger
 
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def ensure_swt_bench_repo(cache_dir: Path | None = None) -> Path:

benchmarks/utils/build_utils.py

@@ -29,10 +29,12 @@
 )
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.image_utils import local_image_exists, remote_image_exists
-from openhands.sdk import get_logger
 
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class BuildOutput(BaseModel):

benchmarks/utils/buildx_utils.py

@@ -11,10 +11,11 @@
 import time
 from pathlib import Path
 
-from openhands.sdk import get_logger
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _read_reset_state(path: Path) -> dict[str, float]:

benchmarks/utils/image_utils.py

@@ -14,10 +14,11 @@
 
 import requests
 
-from openhands.sdk import get_logger
+# Use stdlib logging instead of openhands.sdk.get_logger to avoid initializing
+# Rich console state before ProcessPoolExecutor forks (causes deadlocks).
+import logging
 
-
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 ACCEPT = ",".join(