Merge branch 'main' into allowed-models-plugin

Author: VisargD

.github/workflows/run_tests.yml

@@ -0,0 +1,75 @@
+name: Gateway Tests
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  gateway-tests:
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run tests') }}
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - name: Checkout head
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Start gateway and run tests
+        id: run-tests
+        continue-on-error: true
+        run: |
+          npm run build/start-server.js &
+          echo "Waiting for gateway to start..."
+          while ! curl -s http://localhost:8787 > /dev/null; do
+            sleep 1
+          done
+          echo "Gateway is ready. Running tests..."
+          npm run test:gateway
+
+      - name: Update PR Check
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const { owner, repo } = context.repo;
+            const issue_number = context.issue.number;
+            
+            try {
+              const { data: pull_request } = await github.rest.pulls.get({
+                owner,
+                repo,
+                pull_number: issue_number,
+              });
+              
+              const sha = pull_request.head.sha;
+              
+              await github.rest.checks.create({
+                owner,
+                repo,
+                name: 'Gateway Tests (Comment Triggered)',
+                head_sha: sha,
+                status: 'completed',
+                conclusion: '${{ steps.run-tests.outcome }}',
+                output: {
+                  title: 'Gateway Test Results',
+                  summary: 'Gateway tests have completed.',
+                  text: 'These tests were triggered by a comment on the PR.'
+                },
+              });
+              
+              console.log('Check run created successfully');
+            } catch (error) {
+              console.error('Error creating check run:', error);
+              core.setFailed(`Action failed with error: ${error}`);
+            }
+
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}

README.md

@@ -67,7 +67,7 @@ The AI Gateway's enterprise version offers enterprise-ready capabilities for **o
 
 The enterprise deployment architecture, supported platforms is available here - [**Enterprise Private Cloud Deployments**](https://docs.portkey.ai/docs/product/enterprise-offering/private-cloud-deployments)
 
-<a href="https://app.portkey.ai/signup"><img src="https://portkey.ai/blog/content/images/2024/08/Get-API-Key--5-.png" height=50 alt="Book an enterprise AI gateway demo" /></a><br>
+<a href="https://portkey.sh/demo-22"><img src="https://portkey.ai/blog/content/images/2024/08/Get-API-Key--5-.png" height=50 alt="Book an enterprise AI gateway demo" /></a><br>
 
 <br>
 
@@ -264,11 +264,11 @@ Make your AI app more <ins>reliable</ins> and <ins>forward compatible</ins>, whi
 ✅&nbsp; Secure Key Management - for role-based access control and tracking <br>
 ✅&nbsp; Simple & Semantic Caching - to serve repeat queries faster & save costs <br>
 ✅&nbsp; Access Control & Inbound Rules - to control which IPs and Geos can connect to your deployments <br>
-✅&nbsp; PII Redaction - to automatically remove sensitive data from your requests to prevent indavertent exposure <br>
+✅&nbsp; PII Redaction - to automatically remove sensitive data from your requests to prevent inadvertent exposure <br>
 ✅&nbsp; SOC2, ISO, HIPAA, GDPR Compliances - for best security practices <br>
 ✅&nbsp; Professional Support - along with feature prioritization <br>
 
-[Schedule a call to discuss enterprise deployments](https://calendly.com/rohit-portkey/noam)
+[Schedule a call to discuss enterprise deployments](https://portkey.sh/demo-22)
 
 <br>
 

cookbook/integrations/vercel/vercel-app

@@ -1,6 +1,6 @@
 {
   "name": "@portkey-ai/gateway",
-  "version": "1.7.7",
+  "version": "1.8.0",
   "description": "A fast AI gateway by Portkey",
   "repository": {
     "type": "git",
@@ -34,7 +34,8 @@
     "start:node": "node build/start-server.js",
     "format": "prettier --write \"./**/*.{js,jsx,ts,tsx,json}\"",
     "format:check": "prettier --check \"./**/*.{js,jsx,ts,tsx,json}\"",
-    "test": "ts-jest",
+    "test:gateway": "jest src/",
+    "test:plugins": "jest plugins/",
     "pre-push": "npm run build && node start-test.js",
     "prepare": "node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky"
   },

package-lock.json

@@ -13,6 +13,7 @@ export const HEADER_KEYS: Record<string, string> = {
   CUSTOM_HOST: `x-${POWERED_BY}-custom-host`,
   REQUEST_TIMEOUT: `x-${POWERED_BY}-request-timeout`,
   STRICT_OPEN_AI_COMPLIANCE: `x-${POWERED_BY}-strict-open-ai-compliance`,
+  CONTENT_TYPE: `Content-Type`,
 };
 
 export const RESPONSE_HEADER_KEYS: Record<string, string> = {
@@ -138,3 +139,52 @@ export const MULTIPART_FORM_DATA_ENDPOINTS: endpointStrings[] = [
   'createTranscription',
   'createTranslation',
 ];
+
+export const fileExtensionMimeTypeMap = {
+  mp4: 'video/mp4',
+  jpeg: 'image/jpeg',
+  jpg: 'image/jpeg',
+  png: 'image/png',
+  bmp: 'image/bmp',
+  tiff: 'image/tiff',
+  webp: 'image/webp',
+  pdf: 'application/pdf',
+  csv: 'text/csv',
+  doc: 'application/msword',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  xls: 'application/vnd.ms-excel',
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  html: 'text/html',
+  md: 'text/markdown',
+  mp3: 'audio/mp3',
+  wav: 'audio/wav',
+  txt: 'text/plain',
+  mov: 'video/mov',
+  mpeg: 'video/mpeg',
+  mpg: 'video/mpg',
+  avi: 'video/avi',
+  wmv: 'video/wmv',
+  mpegps: 'video/mpegps',
+  flv: 'video/flv',
+};
+
+export const imagesMimeTypes = [
+  fileExtensionMimeTypeMap.jpeg,
+  fileExtensionMimeTypeMap.jpg,
+  fileExtensionMimeTypeMap.png,
+  fileExtensionMimeTypeMap.bmp,
+  fileExtensionMimeTypeMap.tiff,
+  fileExtensionMimeTypeMap.webp,
+];
+
+export const documentMimeTypes = [
+  fileExtensionMimeTypeMap.pdf,
+  fileExtensionMimeTypeMap.csv,
+  fileExtensionMimeTypeMap.doc,
+  fileExtensionMimeTypeMap.docx,
+  fileExtensionMimeTypeMap.xls,
+  fileExtensionMimeTypeMap.xlsx,
+  fileExtensionMimeTypeMap.html,
+  fileExtensionMimeTypeMap.md,
+  fileExtensionMimeTypeMap.txt,
+];

package.json

@@ -11,9 +11,9 @@ import {
   OPEN_AI,
   AZURE_AI_INFERENCE,
   ANTHROPIC,
-  MULTIPART_FORM_DATA_ENDPOINTS,
   CONTENT_TYPES,
   HUGGING_FACE,
+  STABILITY_AI,
 } from '../globals';
 import Providers from '../providers';
 import { ProviderAPIConfig, endpointStrings } from '../providers/types';
@@ -269,6 +269,7 @@ export async function tryPostProxy(
     : (providerOption.urlToFetch as string);
 
   const headers = await apiConfig.headers({
+    c,
     providerOptions: providerOption,
     fn,
     transformedRequestBody: params,
@@ -520,10 +521,12 @@ export async function tryPost(
   const url = `${baseUrl}${endpoint}`;
 
   const headers = await apiConfig.headers({
+    c,
     providerOptions: providerOption,
     fn,
     transformedRequestBody,
     transformedRequestUrl: url,
+    gatewayRequestBody: params,
   });
 
   // Construct the base object for the POST request
@@ -535,9 +538,10 @@ export async function tryPost(
     requestHeaders
   );
 
-  fetchOptions.body = MULTIPART_FORM_DATA_ENDPOINTS.includes(fn)
-    ? (transformedRequestBody as FormData)
-    : JSON.stringify(transformedRequestBody);
+  fetchOptions.body =
+    headers[HEADER_KEYS.CONTENT_TYPE] === CONTENT_TYPES.MULTIPART_FORM_DATA
+      ? (transformedRequestBody as FormData)
+      : JSON.stringify(transformedRequestBody);
 
   providerOption.retry = {
     attempts: providerOption.retry?.attempts ?? 0,
@@ -638,8 +642,8 @@ export async function tryPost(
 
   // Prerequest validator (For virtual key budgets)
   const preRequestValidator = c.get('preRequestValidator');
-  let preRequestValidatorResponse = preRequestValidator
-    ? await preRequestValidator(env(c), providerOption, requestHeaders)
+  const preRequestValidatorResponse = preRequestValidator
+    ? await preRequestValidator(c, providerOption, requestHeaders, params)
     : undefined;
   if (!!preRequestValidatorResponse) {
     return createResponse(preRequestValidatorResponse, undefined, false);
@@ -1009,9 +1013,24 @@ export function constructConfigFromRequestHeaders(
     resourceName: requestHeaders[`x-${POWERED_BY}-azure-resource-name`],
     deploymentId: requestHeaders[`x-${POWERED_BY}-azure-deployment-id`],
     apiVersion: requestHeaders[`x-${POWERED_BY}-azure-api-version`],
+    azureAuthMode: requestHeaders[`x-${POWERED_BY}-azure-auth-mode`],
+    azureManagedClientId:
+      requestHeaders[`x-${POWERED_BY}-azure-managed-client-id`],
+    azureEntraClientId: requestHeaders[`x-${POWERED_BY}-azure-entra-client-id`],
+    azureEntraClientSecret:
+      requestHeaders[`x-${POWERED_BY}-azure-entra-client-secret`],
+    azureEntraTenantId: requestHeaders[`x-${POWERED_BY}-azure-entra-tenant-id`],
     azureModelName: requestHeaders[`x-${POWERED_BY}-azure-model-name`],
   };
 
+  const stabilityAiConfig = {
+    stabilityClientId: requestHeaders[`x-${POWERED_BY}-stability-client-id`],
+    stabilityClientUserId:
+      requestHeaders[`x-${POWERED_BY}-stability-client-user-id`],
+    stabilityClientVersion:
+      requestHeaders[`x-${POWERED_BY}-stability-client-version`],
+  };
+
   const azureAiInferenceConfig = {
     azureDeploymentName:
       requestHeaders[`x-${POWERED_BY}-azure-deployment-name`],
@@ -1027,6 +1046,9 @@ export function constructConfigFromRequestHeaders(
     awsSecretAccessKey: requestHeaders[`x-${POWERED_BY}-aws-secret-access-key`],
     awsSessionToken: requestHeaders[`x-${POWERED_BY}-aws-session-token`],
     awsRegion: requestHeaders[`x-${POWERED_BY}-aws-region`],
+    awsRoleArn: requestHeaders[`x-${POWERED_BY}-aws-role-arn`],
+    awsAuthType: requestHeaders[`x-${POWERED_BY}-aws-auth-type`],
+    awsExternalId: requestHeaders[`x-${POWERED_BY}-aws-external-id`],
   };
 
   const workersAiConfig = {
@@ -1128,6 +1150,12 @@ export function constructConfigFromRequestHeaders(
           ...anthropicConfig,
         };
       }
+      if (parsedConfigJson.provider === STABILITY_AI) {
+        parsedConfigJson = {
+          ...parsedConfigJson,
+          ...stabilityAiConfig,
+        };
+      }
     }
     return convertKeysToCamelCase(parsedConfigJson, [
       'override_params',
@@ -1158,6 +1186,8 @@ export function constructConfigFromRequestHeaders(
       huggingfaceConfig),
     mistralFimCompletion:
       requestHeaders[`x-${POWERED_BY}-mistral-fim-completion`],
+    ...(requestHeaders[`x-${POWERED_BY}-provider`] === STABILITY_AI &&
+      stabilityAiConfig),
   };
 }
 

src/globals.ts

@@ -33,7 +33,10 @@ function getPayloadFromAWSChunk(chunk: Uint8Array): string {
 
   const payloadLength = chunkLength - headersEnd - 4; // Subtracting 4 for the message crc
   const payload = chunk.slice(headersEnd, headersEnd + payloadLength);
-  return decoder.decode(payload);
+  const decodedJson = JSON.parse(decoder.decode(payload));
+  return decodedJson.bytes
+    ? Buffer.from(decodedJson.bytes, 'base64').toString()
+    : JSON.stringify(decodedJson);
 }
 
 function concatenateUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
@@ -60,10 +63,7 @@ export async function* readAWSStream(
           const data = buffer.subarray(0, expectedLength);
           buffer = buffer.subarray(expectedLength);
           expectedLength = readUInt32BE(buffer, 0);
-          const payload = Buffer.from(
-            JSON.parse(getPayloadFromAWSChunk(data)).bytes,
-            'base64'
-          ).toString();
+          const payload = getPayloadFromAWSChunk(data);
           if (transformFunction) {
             const transformedChunk = transformFunction(
               payload,
@@ -96,11 +96,7 @@ export async function* readAWSStream(
       buffer = buffer.subarray(expectedLength);
 
       expectedLength = readUInt32BE(buffer, 0);
-      const payload = Buffer.from(
-        JSON.parse(getPayloadFromAWSChunk(data)).bytes,
-        'base64'
-      ).toString();
-
+      const payload = getPayloadFromAWSChunk(data);
       if (transformFunction) {
         const transformedChunk = transformFunction(
           payload,

src/handlers/handlerUtils.ts

@@ -2,14 +2,20 @@ import { ProviderAPIConfig } from '../types';
 
 const AnthropicAPIConfig: ProviderAPIConfig = {
   getBaseURL: () => 'https://api.anthropic.com/v1',
-  headers: ({ providerOptions, fn }) => {
+  headers: ({ providerOptions, fn, gatewayRequestBody }) => {
     const headers: Record<string, string> = {
       'X-API-Key': `${providerOptions.apiKey}`,
     };
 
+    // Accept anthropic_beta and anthropic_version in body to support enviroments which cannot send it in headers.
     const betaHeader =
-      providerOptions?.['anthropicBeta'] ?? 'messages-2023-12-15';
-    const version = providerOptions?.['anthropicVersion'] ?? '2023-06-01';
+      providerOptions?.['anthropicBeta'] ??
+      gatewayRequestBody?.['anthropic_beta'] ??
+      'messages-2023-12-15';
+    const version =
+      providerOptions?.['anthropicVersion'] ??
+      gatewayRequestBody?.['anthropic_version'] ??
+      '2023-06-01';
 
     if (fn === 'chatComplete') {
       headers['anthropic-beta'] = betaHeader;