feat: fallback url when hitting cold starts (#1113)

* feat: add cold start fallback for superagent models

Add automatic fallback to always-on endpoint when primary Cloud Run
endpoints experience cold starts.

- Add timeout-based fallback (default 5s) using AbortController (TS) / asyncio (Python)
- Add DEFAULT_FALLBACK_URL constant pointing to https://superagent.sh/api/fallback
- Add getFallbackUrl() helper with priority: client option > env var > default
- Add FallbackOptions interface/dataclass for configuration
- Add enableFallback, fallbackTimeoutMs/fallback_timeout, fallbackUrl options to ClientConfig
- Support SUPERAGENT_FALLBACK_URL environment variable override
- Add unit tests for fallback configuration and logic
- Bump version to 0.1.4

Co-authored-by: Cursor <[email protected]>

* chore: bump cli and mcp to use safety-agent v0.1.4

- cli: 0.1.3 → 0.1.4
- mcp: 0.1.2 → 0.1.3
- Update safety-agent dependency to ^0.1.4

Co-authored-by: Cursor <[email protected]>

---------

Co-authored-by: Cursor <[email protected]>

Author: homanp

cli/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "safety-agent-cli",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "CLI for Superagent - validate prompts and tool calls for security",
   "type": "module",
   "main": "./dist/index.js",
@@ -42,6 +42,6 @@
     "node": ">=18"
   },
   "dependencies": {
-    "safety-agent": "^0.1.3"
+    "safety-agent": "^0.1.4"
   }
 }

cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "safety-agent-mcp",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "MCP server for Superagent.sh API integration - security guardrails, PII redaction, and claim verification",
   "type": "module",
   "main": "dist/index.js",
@@ -33,7 +33,7 @@
   "license": "MIT",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.6.1",
-    "safety-agent": "^0.1.3",
+    "safety-agent": "^0.1.4",
     "zod": "^3.23.8"
   },
   "devDependencies": {

mcp/package-lock.json

@@ -1,6 +1,6 @@
 [project]
 name = "safety-agent"
-version = "0.1.3"
+version = "0.1.4"
 description = "A lightweight Python guardrail SDK for content safety"
 readme = "README.md"
 license = "MIT"

mcp/package.json

@@ -27,7 +27,7 @@
     ProcessedInput,
     MultimodalContentPart,
 )
-from .providers import call_provider, parse_model, DEFAULT_GUARD_MODEL
+from .providers import call_provider, parse_model, DEFAULT_GUARD_MODEL, FallbackOptions
 from .prompts.guard import build_guard_user_message, build_guard_system_prompt
 from .prompts.redact import build_redact_system_prompt, build_redact_user_message
 from .schemas import GUARD_RESPONSE_FORMAT, REDACT_RESPONSE_FORMAT
@@ -214,6 +214,11 @@ def __init__(self, config: ClientConfig | None = None):
             )
 
         self._api_key = api_key
+        self._fallback_options = FallbackOptions(
+            enable_fallback=config.enable_fallback if config else None,
+            fallback_timeout=config.fallback_timeout if config else None,
+            fallback_url=config.fallback_url if config else None,
+        )
 
     def _post_usage(self, usage: TokenUsage) -> None:
         """Post usage metrics to Superagent dashboard (fire and forget)."""
@@ -258,7 +263,9 @@ async def _guard_single_text(
         response_format = (
             GUARD_RESPONSE_FORMAT if _supports_structured_output(model) else None
         )
-        response = await call_provider(model, messages, response_format)
+        response = await call_provider(
+            model, messages, response_format, self._fallback_options
+        )
         content = response.choices[0].message.content
 
         if not content:
@@ -314,7 +321,9 @@ async def _guard_image(
         response_format = (
             GUARD_RESPONSE_FORMAT if _supports_structured_output(model) else None
         )
-        response = await call_provider(model, messages, response_format)
+        response = await call_provider(
+            model, messages, response_format, self._fallback_options
+        )
         content = response.choices[0].message.content
 
         if not content:
@@ -498,7 +507,9 @@ async def redact(
         response_format = (
             REDACT_RESPONSE_FORMAT if _supports_structured_output(model) else None
         )
-        response = await call_provider(model, messages, response_format)
+        response = await call_provider(
+            model, messages, response_format, self._fallback_options
+        )
         content = response.choices[0].message.content
 
         if not content:
@@ -695,17 +706,37 @@ def create_client(
     api_key: str | None = None,
     *,
     config: ClientConfig | None = None,
+    enable_fallback: bool | None = None,
+    fallback_timeout: float | None = None,
+    fallback_url: str | None = None,
 ) -> SafetyClient:
     """
     Create a new Safety Agent client.
 
     Args:
         api_key: API key for Superagent usage tracking
         config: Optional client configuration
+        enable_fallback: Enable fallback to always-on endpoint on cold start timeout.
+            Default: True for superagent provider.
+        fallback_timeout: Timeout in seconds before falling back. Default: 5.0.
+        fallback_url: Custom fallback URL. If not provided, uses env var or default.
 
     Returns:
         SafetyClient instance
     """
-    if api_key:
-        config = ClientConfig(api_key=api_key)
+    if config is None:
+        config = ClientConfig(
+            api_key=api_key,
+            enable_fallback=enable_fallback,
+            fallback_timeout=fallback_timeout,
+            fallback_url=fallback_url,
+        )
+    elif api_key:
+        # Override api_key if provided directly
+        config = ClientConfig(
+            api_key=api_key,
+            enable_fallback=enable_fallback or config.enable_fallback,
+            fallback_timeout=fallback_timeout or config.fallback_timeout,
+            fallback_url=fallback_url or config.fallback_url,
+        )
     return SafetyClient(config)

sdk/python/pyproject.toml

@@ -2,7 +2,9 @@
 Provider registry and utilities
 """
 
+import asyncio
 import os
+from dataclasses import dataclass
 from typing import Any
 
 import httpx
@@ -18,7 +20,26 @@
 from .openrouter import openrouter_provider
 from .openai_compatible import openai_compatible_provider
 from .vercel import vercel_provider
-from .superagent import superagent_provider
+from .superagent import (
+    superagent_provider,
+    get_fallback_url,
+    DEFAULT_FALLBACK_TIMEOUT,
+    DEFAULT_FALLBACK_URL,
+)
+
+
+@dataclass
+class FallbackOptions:
+    """Options for fallback behavior on cold starts."""
+
+    enable_fallback: bool | None = None
+    """Enable fallback to always-on endpoint on timeout. Default: True for superagent provider."""
+
+    fallback_timeout: float | None = None
+    """Timeout in seconds before falling back. Default: 5.0."""
+
+    fallback_url: str | None = None
+    """Custom fallback URL. If not provided, uses env var or default."""
 
 # Default model for guard() when no model is specified
 DEFAULT_GUARD_MODEL = "superagent/guard-1.7b"
@@ -82,6 +103,7 @@ async def call_provider(
     model_string: str,
     messages: list[ChatMessage],
     response_format: ResponseFormat | None = None,
+    fallback_options: FallbackOptions | None = None,
 ) -> AnalysisResponse:
     """Call an LLM provider with the given messages."""
     parsed = parse_model(model_string)
@@ -108,7 +130,67 @@ async def call_provider(
         payload = json.dumps(request_body)
         headers = provider.get_signed_headers(url, "POST", payload, api_key)
 
-    # Make request
+    # Determine if fallback is enabled (default: True for superagent provider)
+    is_superagent = parsed.provider == "superagent"
+    fallback_opts = fallback_options or FallbackOptions()
+    enable_fallback = fallback_opts.enable_fallback if fallback_opts.enable_fallback is not None else is_superagent
+    fallback_timeout = fallback_opts.fallback_timeout or DEFAULT_FALLBACK_TIMEOUT
+    fallback_url = get_fallback_url(fallback_opts.fallback_url)
+
+    # Check if fallback is enabled and URL is available
+    fallback_available = (
+        enable_fallback
+        and fallback_url
+        and fallback_url != "FALLBACK_ENDPOINT_PLACEHOLDER"
+    )
+
+    if fallback_available:
+        # Use timeout-based fallback
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await asyncio.wait_for(
+                    client.post(
+                        url,
+                        headers=headers,
+                        json=request_body,
+                        timeout=60.0,
+                    ),
+                    timeout=fallback_timeout,
+                )
+
+                if response.status_code != 200:
+                    raise RuntimeError(
+                        f"Provider API error ({response.status_code}): {response.text}"
+                    )
+
+                response_data = response.json()
+                return provider.transform_response(response_data)
+
+        except asyncio.TimeoutError:
+            # Retry on fallback endpoint
+            print(
+                f"Primary endpoint timed out after {fallback_timeout}s, "
+                f"falling back to always-on endpoint"
+            )
+
+            async with httpx.AsyncClient() as client:
+                fallback_response = await client.post(
+                    fallback_url,
+                    headers=headers,
+                    json=request_body,
+                    timeout=60.0,
+                )
+
+                if fallback_response.status_code != 200:
+                    raise RuntimeError(
+                        f"Fallback provider API error ({fallback_response.status_code}): "
+                        f"{fallback_response.text}"
+                    )
+
+                fallback_data = fallback_response.json()
+                return provider.transform_response(fallback_data)
+
+    # No fallback - standard request
     async with httpx.AsyncClient() as client:
         response = await client.post(
             url,
@@ -133,4 +215,5 @@ async def call_provider(
     "parse_model",
     "get_provider",
     "call_provider",
+    "FallbackOptions",
 ]

sdk/python/src/safety_agent/client.py

@@ -3,6 +3,7 @@
 """
 
 import json
+import os
 import re
 from typing import Any
 
@@ -16,6 +17,23 @@
     "guard-4b": "https://superagent-guard-medium-408394858807.us-central1.run.app/api/chat",
 }
 
+# Default fallback URL for cold start mitigation.
+# This always-on endpoint handles requests when the primary endpoint has a cold start.
+DEFAULT_FALLBACK_URL = "https://superagent.sh/api/fallback"
+
+# Default timeout in seconds before falling back to the always-on endpoint.
+DEFAULT_FALLBACK_TIMEOUT = 5.0
+
+
+def get_fallback_url(client_option: str | None = None) -> str:
+    """
+    Get the fallback URL based on priority:
+    1. Client option (highest priority)
+    2. Environment variable SUPERAGENT_FALLBACK_URL
+    3. Default constant (lowest priority)
+    """
+    return client_option or os.environ.get("SUPERAGENT_FALLBACK_URL") or DEFAULT_FALLBACK_URL
+
 
 class SuperagentProvider:
     """Superagent provider configuration using Ollama-style API."""

sdk/python/src/safety_agent/providers/__init__.py

@@ -18,6 +18,15 @@ class ClientConfig:
     api_key: str | None = None
     """API key for Superagent usage tracking. Defaults to SUPERAGENT_API_KEY env var."""
 
+    enable_fallback: bool | None = None
+    """Enable fallback to always-on endpoint on cold start timeout. Default: True for superagent provider."""
+
+    fallback_timeout: float | None = None
+    """Timeout in seconds before falling back to always-on endpoint. Default: 5.0."""
+
+    fallback_url: str | None = None
+    """Custom fallback URL. If not provided, uses SUPERAGENT_FALLBACK_URL env var or built-in default."""
+
 
 # =============================================================================
 # Model Types