Merge branch 'main' into feat/model-provider-api

Author: VisargD

.github/workflows/run_tests.yml

@@ -1,23 +1,29 @@
-name: Run Gateway tests
+name: Gateway Tests
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened]
+  issue_comment:
+    types: [created]
 
 jobs:
   gateway-tests:
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run tests') }}
     runs-on: ubuntu-latest
     environment: production
     steps:
       - name: Checkout head
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - name: Install dependencies
         run: npm ci
+
       - name: Build
         run: npm run build
+
       - name: Start gateway and run tests
+        id: run-tests
+        continue-on-error: true
         run: |
           npm run build/start-server.js &
           echo "Waiting for gateway to start..."
@@ -27,6 +33,43 @@ jobs:
           echo "Gateway is ready. Running tests..."
           npm run test:gateway
 
+      - name: Update PR Check
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const { owner, repo } = context.repo;
+            const issue_number = context.issue.number;
+            
+            try {
+              const { data: pull_request } = await github.rest.pulls.get({
+                owner,
+                repo,
+                pull_number: issue_number,
+              });
+              
+              const sha = pull_request.head.sha;
+              
+              await github.rest.checks.create({
+                owner,
+                repo,
+                name: 'Gateway Tests (Comment Triggered)',
+                head_sha: sha,
+                status: 'completed',
+                conclusion: '${{ steps.run-tests.outcome }}',
+                output: {
+                  title: 'Gateway Test Results',
+                  summary: 'Gateway tests have completed.',
+                  text: 'These tests were triggered by a comment on the PR.'
+                },
+              });
+              
+              console.log('Check run created successfully');
+            } catch (error) {
+              console.error('Error creating check run:', error);
+              core.setFailed(`Action failed with error: ${error}`);
+            }
+
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}

README.md

@@ -67,7 +67,7 @@ The AI Gateway's enterprise version offers enterprise-ready capabilities for **o
 
 The enterprise deployment architecture, supported platforms is available here - [**Enterprise Private Cloud Deployments**](https://docs.portkey.ai/docs/product/enterprise-offering/private-cloud-deployments)
 
-<a href="https://app.portkey.ai/signup"><img src="https://portkey.ai/blog/content/images/2024/08/Get-API-Key--5-.png" height=50 alt="Book an enterprise AI gateway demo" /></a><br>
+<a href="https://portkey.sh/demo-22"><img src="https://portkey.ai/blog/content/images/2024/08/Get-API-Key--5-.png" height=50 alt="Book an enterprise AI gateway demo" /></a><br>
 
 <br>
 
@@ -268,7 +268,7 @@ Make your AI app more <ins>reliable</ins> and <ins>forward compatible</ins>, whi
 ✅&nbsp; SOC2, ISO, HIPAA, GDPR Compliances - for best security practices <br>
 ✅&nbsp; Professional Support - along with feature prioritization <br>
 
-[Schedule a call to discuss enterprise deployments](https://calendly.com/rohit-portkey/noam)
+[Schedule a call to discuss enterprise deployments](https://portkey.sh/demo-22)
 
 <br>
 

cookbook/integrations/vercel/vercel-app

@@ -10,6 +10,7 @@ import { handler as logHandler } from './log';
 import { handler as allUppercaseHandler } from './alluppercase';
 import { handler as endsWithHandler } from './endsWith';
 import { handler as allLowerCaseHandler } from './alllowercase';
+import { handler as modelWhitelistHandler } from './modelWhitelist';
 
 import { z } from 'zod';
 import { PluginContext, PluginParameters } from '../types';
@@ -802,3 +803,36 @@ describe('allLowercase handler', () => {
     expect(result.verdict).toBe(false);
   });
 });
+
+describe('modelWhitelist handler', () => {
+  it('should return true verdict when the model requested is part of the whitelist', async () => {
+    const context: PluginContext = {
+      request: { json: { model: 'gemini-1.5-flash-001' } },
+    };
+
+    const parameters: PluginParameters = {
+      models: ['gemini-1.5-flash-001'],
+    };
+    const eventType = 'beforeRequestHook';
+
+    const result = await modelWhitelistHandler(context, parameters, eventType);
+
+    expect(result.error).toBe(null);
+    expect(result.verdict).toBe(true);
+  });
+  it('should return false verdict when the model requested is not part of the whitelist', async () => {
+    const context: PluginContext = {
+      request: { json: { model: 'gemini-1.5-pro-001' } },
+    };
+
+    const parameters: PluginParameters = {
+      models: ['gemini-1.5-flash-001'],
+    };
+    const eventType = 'beforeRequestHook';
+
+    const result = await modelWhitelistHandler(context, parameters, eventType);
+
+    expect(result.error).toBe(null);
+    expect(result.verdict).toBe(false);
+  });
+});

plugins/default/default.test.ts

@@ -475,6 +475,37 @@
         }
       ],
       "parameters": {}
+    },
+    {
+      "name": "Model whitelisting",
+      "id": "modelwhitelist",
+      "type": "guardrail",
+      "supportedHooks": ["beforeRequestHook"],
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Check if the model in the request is part of the allowed model list."
+        }
+      ],
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "models": {
+            "type": "array",
+            "label": "Model list",
+            "description": [
+              {
+                "type": "subHeading",
+                "text": "Enter the allowed models."
+              }
+            ],
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": ["models"]
+      }
     }
   ]
 }

plugins/default/manifest.json

@@ -0,0 +1,25 @@
+import {
+  HookEventType,
+  PluginContext,
+  PluginHandler,
+  PluginParameters,
+} from '../types';
+
+export const handler: PluginHandler = async (
+  context: PluginContext,
+  parameters: PluginParameters,
+  eventType: HookEventType
+) => {
+  let error = null;
+  let verdict = false;
+
+  try {
+    const modelList = parameters.models;
+    let requestModel = context.request?.json.model;
+    verdict = modelList.includes(requestModel);
+  } catch (e) {
+    error = e as Error;
+  }
+
+  return { error, verdict };
+};

plugins/default/modelWhitelist.ts

@@ -12,6 +12,7 @@ import { handler as defaultcontainsCode } from './default/containsCode';
 import { handler as defaultalluppercase } from './default/alluppercase';
 import { handler as defaultalllowercase } from './default/alllowercase';
 import { handler as defaultendsWith } from './default/endsWith';
+import { handler as defaultmodelWhitelist } from './default/modelWhitelist';
 import { handler as portkeymoderateContent } from './portkey/moderateContent';
 import { handler as portkeylanguage } from './portkey/language';
 import { handler as portkeypii } from './portkey/pii';
@@ -48,6 +49,7 @@ export const plugins = {
     alluppercase: defaultalluppercase,
     alllowercase: defaultalllowercase,
     endsWith: defaultendsWith,
+    modelWhitelist: defaultmodelWhitelist,
   },
   portkey: {
     moderateContent: portkeymoderateContent,

plugins/index.ts

@@ -642,8 +642,8 @@ export async function tryPost(
 
   // Prerequest validator (For virtual key budgets)
   const preRequestValidator = c.get('preRequestValidator');
-  let preRequestValidatorResponse = preRequestValidator
-    ? await preRequestValidator(env(c), providerOption, requestHeaders)
+  const preRequestValidatorResponse = preRequestValidator
+    ? await preRequestValidator(c, providerOption, requestHeaders, params)
     : undefined;
   if (!!preRequestValidatorResponse) {
     return createResponse(preRequestValidatorResponse, undefined, false);

src/handlers/handlerUtils.ts

@@ -87,6 +87,24 @@ export const AzureOpenAIChatCompleteConfig: ProviderConfig = {
   response_format: {
     param: 'response_format',
   },
+  store: {
+    param: 'store',
+  },
+  metadata: {
+    param: 'metadata',
+  },
+  modalities: {
+    param: 'modalities',
+  },
+  audio: {
+    param: 'audio',
+  },
+  seed: {
+    param: 'seed',
+  },
+  prediction: {
+    param: 'prediction',
+  },
 };
 
 interface AzureOpenAIChatCompleteResponse extends ChatCompletionResponse {}

src/providers/azure-openai/chatComplete.ts

@@ -35,6 +35,9 @@ export function transformGenerationConfig(params: Params) {
       schema = derefer(schema);
       delete schema['$defs'];
     }
+    if (Object.hasOwn(schema, '$schema')) {
+      delete schema['$schema'];
+    }
     generationConfig['responseSchema'] = schema;
   }