diff --git a/docs.json b/docs.json index b01de064..f38212da 100644 --- a/docs.json +++ b/docs.json @@ -66,6 +66,7 @@ "product/ai-gateway/cache-simple-and-semantic", "product/ai-gateway/fallbacks", "product/ai-gateway/automatic-retries", + "product/ai-gateway/circuit-breaker", "product/ai-gateway/realtime-api", "product/ai-gateway/load-balancing", "product/ai-gateway/canary-testing", @@ -1007,6 +1008,7 @@ "product/ai-gateway/cache-simple-and-semantic", "product/ai-gateway/fallbacks", "product/ai-gateway/automatic-retries", + "product/ai-gateway/circuit-breaker", "product/ai-gateway/realtime-api", "product/ai-gateway/load-balancing", "product/ai-gateway/canary-testing", @@ -2849,6 +2851,10 @@ "source": "/key-features/automatic-retries", "destination": "/product/ai-gateway/automatic-retries" }, + { + "source": "/key-features/circuit-breaker", + "destination": "/product/ai-gateway/circuit-breaker" + }, { "source": "/key-features/custom-metadata", "destination": "/product/observability/metadata" diff --git a/introduction/feature-overview.mdx b/introduction/feature-overview.mdx index dac7d4ff..7006706f 100644 --- a/introduction/feature-overview.mdx +++ b/introduction/feature-overview.mdx @@ -23,6 +23,9 @@ Connect to 250+ AI models using a single consistent API. Set up load balancers, Implement automatic retries for improved resilience + + Configure per-strategy circuit protection and failure handling + Distribute workload efficiently across multiple models diff --git a/product/ai-gateway.mdx b/product/ai-gateway.mdx index 9b08876e..b94ac143 100644 --- a/product/ai-gateway.mdx +++ b/product/ai-gateway.mdx @@ -35,6 +35,10 @@ description: The world's fastest AI Gateway with advanced routing & integrated G Setup automatic retry strategies + + Configure per-strategy circuit protection and failure handling + + Load balance between various API Keys to counter rate-limits diff --git a/product/ai-gateway/circuit-breaker.mdx b/product/ai-gateway/circuit-breaker.mdx new file mode 100644 index 00000000..780af545 --- /dev/null +++ b/product/ai-gateway/circuit-breaker.mdx @@ -0,0 +1,93 @@ +--- +title: 'Circuit Breaker' +description: 'Configure per-strategy circuit protection and failure handling' +--- + + +This feature is available on all Portkey [plans](https://portkey.ai/pricing). + + +## Circuit Breaker Config Schema + +Each `strategy` in a config may define a `cb_config` with the following fields: + +- **failure_threshold**: Number of failures after which the circuit opens. +- **failure_threshold_percentage** *(optional)*: Percentage failure rate to trip the circuit. +- **cooldown_interval**: Time (in milliseconds) to wait before allowing retries. A minimum of 30 seconds is enforced. +- **failure_status_codes** *(optional)*: Specific HTTP status codes considered as failures. If not provided, all status codes >500 are considered as failures. +- **minimum_requests** *(optional)*: Minimum number of requests required before failure rate is evaluated. + + +- Strategies can inherit `cb_config` from a parent if not set at their level. +- Targets inherit `cb_config` from their parent strategy. + + + +Strategies using the `conditional` mode are **not considered** in circuit breaker logic. + + +## Example Config + +```json +{ + "retry": { + "attempts": 3 + }, + "strategy": { + "mode": "fallback", + "cb_config": { + "failure_threshold_percentage": 20, + "minimum_requests": 10, + "cooldown_interval": 60000, + "failure_status_codes": [ + 401, + 429, + 500, + ] + } + }, + "targets": [ + { + "virtual_key": "virtual-key-1" + }, + { + "virtual_key": "virtual-key-2" + } + ] +} +``` + +## Circuit State Evaluation + +Circuit breaker logic is evaluated per strategy path. It tracks: + +- Number of failures and successes. +- The time of the first failure. +- Whether the total requests meet the `minimum_requests` threshold and calculates failure rate as percentage + +The circuit is **OPEN**(unhealthy) if: +- The failure count exceeds the `failure_threshold`, or +- The failure rate exceeds `failure_threshold_percentage`. + +Once **OPEN**, requests to the affected targets are blocked until the `cooldown_interval` has passed. + +When the cooldown period ends, the circuit is **CLOSED** automatically, and failure counters are reset. + +## Runtime Behavior + +- At each strategy level, the circuit breaker evaluates the status of all sibling targets. +- Unhealthy targets (circuit OPEN) are removed from the target list before strategy execution. +- If no healthy targets remain, circuit breaker is bypassed and all targets are used for routing. + +```mermaid +flowchart TD + A[User sends request] --> B[Gateway selects a strategy] + B --> C[Strategy has multiple targets] + C --> D{All targets OPEN?} + D -->|Yes| E[Ignore circuit breakers] + E --> F[Use all targets for routing] + D -->|No| G["Use only CLOSED (healthy) targets"] + F --> H[Route to best target] + G --> H + H --> I[Process target] +``` \ No newline at end of file