Merge branch 'Portkey-AI:main' into patch-3

Author: horochx

.gitignore

@@ -142,3 +142,4 @@ build
 plugins/**/.creds.json
 plugins/**/creds.json
 plugins/**/.parameters.json
+src/handlers/tests/.creds.json

CLAUDE.md

@@ -0,0 +1,92 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This is the **Portkey AI Gateway** - a fast, reliable AI gateway that routes requests to 250+ LLMs with sub-1ms latency. It's built with Hono framework for TypeScript/JavaScript and can be deployed to multiple environments including Cloudflare Workers, Node.js servers, and Docker containers.
+
+## Development Commands
+
+### Core Development
+- `npm run dev` - Start development server using Wrangler (Cloudflare Workers)
+- `npm run dev:node` - Start development server using Node.js
+- `npm run build` - Build the project for production
+- `npm run build-plugins` - Build the plugin system
+
+### Testing
+- `npm run test:gateway` - Run tests for the main gateway code (src/)
+- `npm run test:plugins` - Run tests for plugins
+- `jest src/` - Run specific gateway tests
+- `jest plugins/` - Run specific plugin tests
+
+### Code Quality
+- `npm run format` - Format code with Prettier
+- `npm run format:check` - Check code formatting
+- `npm run pretty` - Alternative format command
+
+### Deployment
+- `npm run deploy` - Deploy to Cloudflare Workers
+- `npm run start:node` - Start production Node.js server
+
+## Architecture
+
+### Core Components
+
+**Main Application (`src/index.ts`)**
+- Hono-based HTTP server with middleware pipeline
+- Handles multiple AI provider integrations
+- Routes: `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, etc.
+
+**Provider System (`src/providers/`)**
+- Modular provider implementations (OpenAI, Anthropic, Azure, etc.)
+- Each provider has standardized interface: `api.ts`, `chatComplete.ts`, `embed.ts`
+- Provider configs define supported features and transformations
+
+**Middleware Pipeline**
+- `requestValidator` - Validates incoming requests
+- `hooks` - Pre/post request hooks
+- `memoryCache` - Response caching
+- `logger` - Request/response logging
+- `portkey` - Core Portkey-specific middleware for routing, guardrails, etc.
+
+**Plugin System (`plugins/`)**
+- Guardrail plugins for content filtering, PII detection, etc.
+- Each plugin has `manifest.json` defining capabilities
+- Plugins are built separately with `npm run build-plugins`
+
+### Key Concepts
+
+**Configs** - JSON configurations that define:
+- Provider routing and fallbacks
+- Load balancing strategies
+- Guardrails and content filtering
+- Caching and retry policies
+
+**Handlers** - Route-specific request processors in `src/handlers/`
+- Each AI API endpoint has dedicated handler
+- Stream handling for real-time responses
+- WebSocket support for realtime APIs
+
+## File Structure
+
+- `src/providers/` - AI provider integrations
+- `src/handlers/` - API endpoint handlers
+- `src/middlewares/` - Request/response middleware
+- `plugins/` - Guardrail and validation plugins
+- `cookbook/` - Example integrations and use cases
+- `conf.json` - Runtime configuration
+
+## Testing Strategy
+
+Tests are organized by component:
+- `src/tests/` - Core gateway functionality tests
+- `src/handlers/__tests__/` - Handler-specific tests
+- `plugins/*/**.test.ts` - Plugin tests
+- Test timeout: 30 seconds (configured in jest.config.js)
+
+## Configuration
+
+The gateway uses `conf.json` for runtime configuration. Sample config available in `conf_sample.json`.
+
+Key environment variables and configuration handled through Hono's adapter system for multi-environment deployment.

Dockerfile

@@ -9,7 +9,7 @@ COPY package*.json ./
 COPY patches ./
 
 # Upgrade system packages
-RUN apk update && apk upgrade --no-cache
+RUN apk upgrade --no-cache
 
 # Upgrade npm to version 10.9.2
 RUN npm install -g [email protected]
@@ -29,7 +29,7 @@ RUN npm run build \
 FROM node:20-alpine
 
 # Upgrade system packages
-RUN apk update && apk upgrade --no-cache
+RUN apk upgrade --no-cache
 
 # Upgrade npm to version 10.9.2
 RUN npm install -g [email protected]

conf.json

@@ -8,7 +8,8 @@
     "patronus",
     "pangea",
     "promptsecurity",
-    "panw-prisma-airs"
+    "panw-prisma-airs",
+    "walledai"
   ],
   "credentials": {
     "portkey": {

cookbook/monitoring-agents/Autogen_with_Telemetry.ipynb

@@ -74,20 +74,20 @@
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "Requirement already satisfied: pyautogen in /usr/local/lib/python3.10/dist-packages (0.2.28)\n",
+            "Requirement already satisfied: ag2 in /usr/local/lib/python3.10/dist-packages (0.2.28)\n",
             "Collecting portkey-ai\n",
             "  Downloading portkey_ai-1.3.2-py3-none-any.whl (86 kB)\n",
             "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.3/86.3 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hRequirement already satisfied: diskcache in /usr/local/lib/python3.10/dist-packages (from pyautogen) (5.6.3)\n",
-            "Requirement already satisfied: docker in /usr/local/lib/python3.10/dist-packages (from pyautogen) (7.1.0)\n",
-            "Requirement already satisfied: flaml in /usr/local/lib/python3.10/dist-packages (from pyautogen) (2.1.2)\n",
-            "Requirement already satisfied: numpy<2,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from pyautogen) (1.25.2)\n",
-            "Requirement already satisfied: openai>=1.3 in /usr/local/lib/python3.10/dist-packages (from pyautogen) (1.30.5)\n",
-            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from pyautogen) (24.0)\n",
-            "Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /usr/local/lib/python3.10/dist-packages (from pyautogen) (2.7.1)\n",
-            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from pyautogen) (1.0.1)\n",
-            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from pyautogen) (2.4.0)\n",
-            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from pyautogen) (0.7.0)\n",
+            "\u001b[?25hRequirement already satisfied: diskcache in /usr/local/lib/python3.10/dist-packages (from ag2) (5.6.3)\n",
+            "Requirement already satisfied: docker in /usr/local/lib/python3.10/dist-packages (from ag2) (7.1.0)\n",
+            "Requirement already satisfied: flaml in /usr/local/lib/python3.10/dist-packages (from ag2) (2.1.2)\n",
+            "Requirement already satisfied: numpy<2,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from ag2) (1.25.2)\n",
+            "Requirement already satisfied: openai>=1.3 in /usr/local/lib/python3.10/dist-packages (from ag2) (1.30.5)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from ag2) (24.0)\n",
+            "Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /usr/local/lib/python3.10/dist-packages (from ag2) (2.7.1)\n",
+            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from ag2) (1.0.1)\n",
+            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from ag2) (2.4.0)\n",
+            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from ag2) (0.7.0)\n",
             "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from portkey-ai) (0.27.0)\n",
             "Collecting mypy<2.0,>=0.991 (from portkey-ai)\n",
             "  Downloading mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.7 MB)\n",
@@ -96,28 +96,28 @@
             "Collecting mypy-extensions>=1.0.0 (from mypy<2.0,>=0.991->portkey-ai)\n",
             "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
             "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from mypy<2.0,>=0.991->portkey-ai) (2.0.1)\n",
-            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->pyautogen) (3.7.1)\n",
-            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.3->pyautogen) (1.7.0)\n",
-            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->pyautogen) (1.3.1)\n",
-            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->pyautogen) (4.66.4)\n",
+            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->ag2) (3.7.1)\n",
+            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.3->ag2) (1.7.0)\n",
+            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->ag2) (1.3.1)\n",
+            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3->ag2) (4.66.4)\n",
             "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->portkey-ai) (2024.2.2)\n",
             "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->portkey-ai) (1.0.5)\n",
             "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->portkey-ai) (3.7)\n",
             "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->portkey-ai) (0.14.0)\n",
-            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen) (0.7.0)\n",
-            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen) (2.18.2)\n",
-            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->pyautogen) (2.31.0)\n",
-            "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->pyautogen) (2.0.7)\n",
-            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->pyautogen) (2024.5.15)\n",
-            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen) (1.2.1)\n",
-            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->docker->pyautogen) (3.3.2)\n",
+            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=2.6.0,<3,>=1.10->ag2) (0.7.0)\n",
+            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=2.6.0,<3,>=1.10->ag2) (2.18.2)\n",
+            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->ag2) (2.31.0)\n",
+            "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->ag2) (2.0.7)\n",
+            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->ag2) (2024.5.15)\n",
+            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.3->ag2) (1.2.1)\n",
+            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->docker->ag2) (3.3.2)\n",
             "Installing collected packages: mypy-extensions, mypy, portkey-ai\n",
             "Successfully installed mypy-1.10.0 mypy-extensions-1.0.0 portkey-ai-1.3.2\n"
           ]
         }
       ],
       "source": [
-        "!pip install -qU pyautogen portkey-ai"
+        "!pip install -qU ag2 portkey-ai"
       ]
     },
     {