fix(agent-sandbox): wait for sandbox api reachability after ready

Author: johannhartmann

clients/python/agentic-sandbox-client/k8s_agent_sandbox/sandbox_client.py

@@ -17,7 +17,6 @@
 file I/O) with the sandbox environment, including optional OpenTelemetry tracing.
 """
 
-import json
 import os
 import sys
 import time
@@ -106,6 +105,8 @@ def __init__(
         sandbox_ready_timeout: int = 180,
         gateway_ready_timeout: int = 180,
         port_forward_ready_timeout: int = 30,
+        api_ready_timeout: float = 15.0,
+        api_probe_interval: float = 0.5,
         enable_tracing: bool = False,
         trace_service_name: str = "sandbox-client",
         claim_name: str | None = None,
@@ -133,6 +134,8 @@ def __init__(
         self.sandbox_ready_timeout = sandbox_ready_timeout
         self.gateway_ready_timeout = gateway_ready_timeout
         self.port_forward_ready_timeout = port_forward_ready_timeout
+        self.api_ready_timeout = api_ready_timeout
+        self.api_probe_interval = api_probe_interval
         self.delete_on_exit = delete_on_exit
         self._custom_claim_name = claim_name
 
@@ -491,6 +494,54 @@ def _wait_for_gateway_ip(self):
                 f" an IP within {self.gateway_ready_timeout} seconds."
             )
 
+    @trace_span("wait_for_api_ready")
+    def _wait_for_api_ready(self):
+        """Wait until the configured API URL is DNS-resolvable and HTTP-reachable."""
+        if not self.base_url:
+            raise RuntimeError("Cannot wait for API; base_url is not configured.")
+
+        parsed_url = urllib.parse.urlparse(self.base_url)
+        host = parsed_url.hostname
+        if not host:
+            raise RuntimeError(f"Invalid base URL, missing host: '{self.base_url}'")
+
+        port = parsed_url.port
+        if port is None:
+            port = 443 if parsed_url.scheme == "https" else 80
+
+        deadline = time.monotonic() + self.api_ready_timeout
+        last_error: Exception | None = None
+
+        logging.info(f"Waiting for sandbox API reachability at '{self.base_url}'...")
+
+        while time.monotonic() < deadline:
+            try:
+                socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+                response = requests.get(
+                    self.base_url,
+                    timeout=1.5,
+                    allow_redirects=False,
+                )
+                if response.status_code < 500:
+                    logging.info(
+                        f"Sandbox API is reachable at '{self.base_url}' "
+                        f"(status {response.status_code})."
+                    )
+                    return
+                last_error = RuntimeError(
+                    f"Received HTTP {response.status_code} from sandbox API."
+                )
+            except (socket.gaierror, requests.exceptions.RequestException) as e:
+                last_error = e
+
+            time.sleep(self.api_probe_interval)
+
+        self.__exit__(None, None, None)
+        raise TimeoutError(
+            f"Sandbox API '{self.base_url}' did not become reachable within "
+            f"{self.api_ready_timeout} seconds. Last error: {last_error}"
+        )
+
     def __enter__(self) -> 'SandboxClient':
         trace_context_str = ""
         # We can't use the "with trace..." context management. This is the equivalent.
@@ -515,6 +566,7 @@ def __enter__(self) -> 'SandboxClient':
             # Case 3: No Gateway, No URL -> Developer Mode (Port Forward to Router)
             self._start_and_wait_for_port_forward()
 
+        self._wait_for_api_ready()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):

clients/python/agentic-sandbox-client/test_sandbox_client_unit.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import socket
 from unittest.mock import MagicMock
 
 import pytest
@@ -25,6 +26,7 @@ def _build_client(
     custom_objects_api: MagicMock,
     template_name: str = "python-template",
     claim_name: str | None = None,
+    api_url: str | None = None,
 ) -> SandboxClient:
     def _raise_incluster() -> None:
         raise sandbox_client_module.config.ConfigException("not running in cluster")
@@ -39,6 +41,7 @@ def _raise_incluster() -> None:
     return SandboxClient(
         template_name=template_name,
         claim_name=claim_name,
+        api_url=api_url,
         delete_on_exit=False,
     )
 
@@ -92,3 +95,97 @@ def test_delete_clears_claim_name(monkeypatch: pytest.MonkeyPatch):
     assert sandbox_client.delete() is True
     assert sandbox_client.claim_name is None
     custom_objects_api.delete_namespaced_custom_object.assert_called_once()
+
+
+def test_enter_waits_for_api_ready(monkeypatch: pytest.MonkeyPatch):
+    custom_objects_api = MagicMock()
+    sandbox_client = _build_client(
+        monkeypatch,
+        custom_objects_api,
+        api_url="http://sb-abc.sandbox-router",
+    )
+
+    setup_claim = MagicMock()
+    wait_for_ready = MagicMock()
+    wait_for_api = MagicMock()
+    wait_for_gateway = MagicMock()
+    start_port_forward = MagicMock()
+
+    monkeypatch.setattr(sandbox_client, "_setup_claim", setup_claim)
+    monkeypatch.setattr(sandbox_client, "_wait_for_sandbox_ready", wait_for_ready)
+    monkeypatch.setattr(sandbox_client, "_wait_for_api_ready", wait_for_api)
+    monkeypatch.setattr(sandbox_client, "_wait_for_gateway_ip", wait_for_gateway)
+    monkeypatch.setattr(
+        sandbox_client, "_start_and_wait_for_port_forward", start_port_forward
+    )
+
+    assert sandbox_client.__enter__() is sandbox_client
+    setup_claim.assert_called_once()
+    wait_for_ready.assert_called_once()
+    wait_for_api.assert_called_once()
+    wait_for_gateway.assert_not_called()
+    start_port_forward.assert_not_called()
+
+
+def test_wait_for_api_ready_retries_until_resolvable_and_reachable(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    custom_objects_api = MagicMock()
+    sandbox_client = _build_client(
+        monkeypatch,
+        custom_objects_api,
+        api_url="http://sb-abc.sandbox-router",
+    )
+    sandbox_client.api_ready_timeout = 1
+    sandbox_client.api_probe_interval = 0
+
+    dns_attempts = {"count": 0}
+    http_attempts = {"count": 0}
+
+    def fake_getaddrinfo(host: str, port: int, type: int):
+        assert host == "sb-abc.sandbox-router"
+        assert port == 80
+        assert type == socket.SOCK_STREAM
+        dns_attempts["count"] += 1
+        if dns_attempts["count"] < 3:
+            raise socket.gaierror(-3, "temporary failure in name resolution")
+        return [("ok",)]
+
+    def fake_get(url: str, timeout: float, allow_redirects: bool):
+        assert url == "http://sb-abc.sandbox-router"
+        assert timeout == 1.5
+        assert allow_redirects is False
+        http_attempts["count"] += 1
+        if http_attempts["count"] == 1:
+            raise sandbox_client_module.requests.exceptions.ConnectionError(
+                "connection refused"
+            )
+        response = MagicMock()
+        response.status_code = 404
+        return response
+
+    monkeypatch.setattr(sandbox_client_module.socket, "getaddrinfo", fake_getaddrinfo)
+    monkeypatch.setattr(sandbox_client_module.requests, "get", fake_get)
+    monkeypatch.setattr(sandbox_client_module.time, "sleep", lambda _: None)
+
+    sandbox_client._wait_for_api_ready()
+    assert dns_attempts["count"] == 4
+    assert http_attempts["count"] == 2
+
+
+def test_wait_for_api_ready_times_out(monkeypatch: pytest.MonkeyPatch):
+    custom_objects_api = MagicMock()
+    sandbox_client = _build_client(
+        monkeypatch,
+        custom_objects_api,
+        api_url="http://sb-timeout.sandbox-router",
+    )
+    sandbox_client.api_ready_timeout = 0
+
+    cleanup = MagicMock()
+    monkeypatch.setattr(sandbox_client, "__exit__", cleanup)
+
+    with pytest.raises(TimeoutError):
+        sandbox_client._wait_for_api_ready()
+
+    cleanup.assert_called_once_with(None, None, None)