Merge branch 'main' into chore/video-support-bedrock

Author: VisargD

plugins/default/regexReplace.ts

@@ -0,0 +1,92 @@
+import {
+  HookEventType,
+  PluginContext,
+  PluginHandler,
+  PluginParameters,
+} from '../types';
+import { getCurrentContentPart, setCurrentContentPart } from '../utils';
+
+export const handler: PluginHandler = async (
+  context: PluginContext,
+  parameters: PluginParameters,
+  eventType: HookEventType
+) => {
+  let error = null;
+  let verdict = true;
+  let data: any = null;
+  const transformedData: Record<string, any> = {
+    request: {
+      json: null,
+    },
+    response: {
+      json: null,
+    },
+  };
+  let transformed = false;
+
+  try {
+    const regexPattern = parameters.rule;
+    const redactText = parameters.redactText || '[REDACTED]';
+    const failOnDetection = parameters.failOnDetection || false;
+
+    const { content, textArray } = getCurrentContentPart(context, eventType);
+
+    if (!regexPattern) {
+      throw new Error('Missing regex pattern');
+    }
+    if (!content) {
+      throw new Error('Missing text to match');
+    }
+
+    const regex = new RegExp(regexPattern, 'g');
+
+    // Process all text items in the array
+    let hasMatches = false;
+    const mappedTextArray: Array<string | null> = [];
+    textArray.forEach((text) => {
+      if (!text) {
+        mappedTextArray.push(null);
+        return;
+      }
+
+      // Reset regex for each text when using global flag
+      regex.lastIndex = 0;
+
+      const matches = text.match(regex);
+      if (matches && matches.length > 0) {
+        hasMatches = true;
+      }
+      const replacedText = text.replace(regex, redactText);
+      mappedTextArray.push(replacedText);
+    });
+
+    // Handle transformation
+    if (hasMatches) {
+      setCurrentContentPart(
+        context,
+        eventType,
+        transformedData,
+        mappedTextArray
+      );
+      transformed = true;
+    }
+    if (failOnDetection && hasMatches) {
+      verdict = false;
+    }
+    data = {
+      regexPattern,
+      verdict,
+      explanation: transformed
+        ? `Pattern '${regexPattern}' matched and was replaced with '${redactText}'`
+        : `The regex pattern '${regexPattern}' did not match any text.`,
+    };
+  } catch (e: any) {
+    error = e;
+    data = {
+      explanation: `An error occurred while processing the regex: ${e.message}`,
+      regexPattern: parameters.rule,
+    };
+  }
+
+  return { error, verdict, data, transformedData, transformed };
+};

plugins/index.ts

@@ -50,6 +50,7 @@ import { handler as panwPrismaAirsintercept } from './panw-prisma-airs/intercept
 import { handler as defaultjwt } from './default/jwt';
 import { handler as defaultrequiredMetadataKeys } from './default/requiredMetadataKeys';
 import { handler as walledaiguardrails } from './walledai/guardrails';
+import { handler as defaultregexReplace } from './default/regexReplace';
 
 export const plugins = {
   default: {
@@ -70,6 +71,7 @@ export const plugins = {
     modelWhitelist: defaultmodelWhitelist,
     jwt: defaultjwt,
     requiredMetadataKeys: defaultrequiredMetadataKeys,
+    regexReplace: defaultregexReplace,
   },
   portkey: {
     moderateContent: portkeymoderateContent,

src/providers/anthropic/chatComplete.ts

@@ -606,6 +606,9 @@ export const AnthropicChatCompleteStreamChunkTransform: (
   streamState,
   strictOpenAiCompliance
 ) => {
+  if (streamState.toolIndex == undefined) {
+    streamState.toolIndex = -1;
+  }
   let chunk = responseChunk.trim();
   if (
     chunk.startsWith('event: ping') ||
@@ -724,9 +727,7 @@ export const AnthropicChatCompleteStreamChunkTransform: (
     parsedChunk.type === 'content_block_start' &&
     parsedChunk.content_block?.type === 'tool_use';
   if (isToolBlockStart) {
-    streamState.toolIndex = streamState.toolIndex
-      ? streamState.toolIndex + 1
-      : 0;
+    streamState.toolIndex = streamState.toolIndex + 1;
   }
   const isToolBlockDelta: boolean =
     parsedChunk.type === 'content_block_delta' &&

src/providers/anthropic/complete.ts

@@ -1,9 +1,16 @@
 import { ANTHROPIC } from '../../globals';
 import { Params } from '../../types/requestBody';
 import { CompletionResponse, ErrorResponse, ProviderConfig } from '../types';
-import { generateInvalidProviderResponseError } from '../utils';
+import {
+  generateInvalidProviderResponseError,
+  transformFinishReason,
+} from '../utils';
+import {
+  ANTHROPIC_STOP_REASON,
+  AnthropicStreamState,
+  AnthropicErrorResponse,
+} from './types';
 import { AnthropicErrorResponseTransform } from './utils';
-import { AnthropicErrorResponse } from './types';
 
 // TODO: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model.
 
@@ -57,7 +64,7 @@ export const AnthropicCompleteConfig: ProviderConfig = {
 
 interface AnthropicCompleteResponse {
   completion: string;
-  stop_reason: string;
+  stop_reason: ANTHROPIC_STOP_REASON;
   model: string;
   truncated: boolean;
   stop: null | string;
@@ -68,10 +75,20 @@ interface AnthropicCompleteResponse {
 // TODO: The token calculation is wrong atm
 export const AnthropicCompleteResponseTransform: (
   response: AnthropicCompleteResponse | AnthropicErrorResponse,
-  responseStatus: number
-) => CompletionResponse | ErrorResponse = (response, responseStatus) => {
-  if (responseStatus !== 200 && 'error' in response) {
-    return AnthropicErrorResponseTransform(response);
+  responseStatus: number,
+  responseHeaders: Headers,
+  strictOpenAiCompliance: boolean
+) => CompletionResponse | ErrorResponse = (
+  response,
+  responseStatus,
+  _responseHeaders,
+  strictOpenAiCompliance
+) => {
+  if (responseStatus !== 200) {
+    const errorResposne = AnthropicErrorResponseTransform(
+      response as AnthropicErrorResponse
+    );
+    if (errorResposne) return errorResposne;
   }
 
   if ('completion' in response) {
@@ -86,7 +103,10 @@ export const AnthropicCompleteResponseTransform: (
           text: response.completion,
           index: 0,
           logprobs: null,
-          finish_reason: response.stop_reason,
+          finish_reason: transformFinishReason(
+            response.stop_reason,
+            strictOpenAiCompliance
+          ),
         },
       ],
     };
@@ -96,8 +116,16 @@ export const AnthropicCompleteResponseTransform: (
 };
 
 export const AnthropicCompleteStreamChunkTransform: (
-  response: string
-) => string | undefined = (responseChunk) => {
+  response: string,
+  fallbackId: string,
+  streamState: AnthropicStreamState,
+  strictOpenAiCompliance: boolean
+) => string | undefined = (
+  responseChunk,
+  fallbackId,
+  streamState,
+  strictOpenAiCompliance
+) => {
   let chunk = responseChunk.trim();
   if (chunk.startsWith('event: ping')) {
     return;
@@ -110,6 +138,9 @@ export const AnthropicCompleteStreamChunkTransform: (
     return chunk;
   }
   const parsedChunk: AnthropicCompleteResponse = JSON.parse(chunk);
+  const finishReason = parsedChunk.stop_reason
+    ? transformFinishReason(parsedChunk.stop_reason, strictOpenAiCompliance)
+    : null;
   return (
     `data: ${JSON.stringify({
       id: parsedChunk.log_id,
@@ -122,7 +153,7 @@ export const AnthropicCompleteStreamChunkTransform: (
           text: parsedChunk.completion,
           index: 0,
           logprobs: null,
-          finish_reason: parsedChunk.stop_reason,
+          finish_reason: finishReason,
         },
       ],
     })}` + '\n\n'

src/providers/bedrock/complete.ts

@@ -1,9 +1,13 @@
 import { BEDROCK } from '../../globals';
 import { Params } from '../../types/requestBody';
 import { CompletionResponse, ErrorResponse, ProviderConfig } from '../types';
-import { generateInvalidProviderResponseError } from '../utils';
+import {
+  generateInvalidProviderResponseError,
+  transformFinishReason,
+} from '../utils';
 import { BedrockErrorResponseTransform } from './chatComplete';
 import { BedrockErrorResponse } from './embed';
+import { TITAN_STOP_REASON as TITAN_COMPLETION_REASON } from './types';
 
 export const BedrockAnthropicCompleteConfig: ProviderConfig = {
   prompt: {
@@ -380,7 +384,7 @@ export interface BedrockTitanCompleteResponse {
   results: {
     tokenCount: number;
     outputText: string;
-    completionReason: string;
+    completionReason: TITAN_COMPLETION_REASON;
   }[];
 }
 
@@ -420,7 +424,10 @@ export const BedrockTitanCompleteResponseTransform: (
         text: generation.outputText,
         index: index,
         logprobs: null,
-        finish_reason: generation.completionReason,
+        finish_reason: transformFinishReason(
+          generation.completionReason,
+          strictOpenAiCompliance
+        ),
       })),
       usage: {
         prompt_tokens: response.inputTextTokenCount,
@@ -437,7 +444,7 @@ export interface BedrockTitanStreamChunk {
   outputText: string;
   index: number;
   totalOutputTextTokenCount: number;
-  completionReason: string | null;
+  completionReason: TITAN_COMPLETION_REASON | null;
   'amazon-bedrock-invocationMetrics': {
     inputTokenCount: number;
     outputTokenCount: number;
@@ -462,6 +469,12 @@ export const BedrockTitanCompleteStreamChunkTransform: (
   let chunk = responseChunk.trim();
   chunk = chunk.trim();
   const parsedChunk: BedrockTitanStreamChunk = JSON.parse(chunk);
+  const finishReason = parsedChunk.completionReason
+    ? transformFinishReason(
+        parsedChunk.completionReason,
+        _strictOpenAiCompliance
+      )
+    : null;
 
   return [
     `data: ${JSON.stringify({
@@ -490,7 +503,7 @@ export const BedrockTitanCompleteStreamChunkTransform: (
           text: '',
           index: 0,
           logprobs: null,
-          finish_reason: parsedChunk.completionReason,
+          finish_reason: finishReason,
         },
       ],
       usage: {

src/providers/bedrock/types.ts

@@ -108,7 +108,7 @@ export interface BedrockChatCompletionResponse {
       content: BedrockContentItem[];
     };
   };
-  stopReason: BEDROCK_STOP_REASON;
+  stopReason: BEDROCK_CONVERSE_STOP_REASON;
   usage: {
     inputTokens: number;
     outputTokens: number;
@@ -171,7 +171,7 @@ export type BedrockContentItem = {
 };
 
 export interface BedrockStreamState {
-  stopReason?: BEDROCK_STOP_REASON;
+  stopReason?: BEDROCK_CONVERSE_STOP_REASON;
   currentToolCallIndex?: number;
   currentContentBlockIndex?: number;
 }
@@ -201,7 +201,7 @@ export interface BedrockChatCompleteStreamChunk {
       input?: object;
     };
   };
-  stopReason?: BEDROCK_STOP_REASON;
+  stopReason?: BEDROCK_CONVERSE_STOP_REASON;
   metrics?: {
     latencyMs: number;
   };
@@ -214,13 +214,22 @@ export interface BedrockChatCompleteStreamChunk {
     cacheWriteInputTokenCount?: number;
     cacheWriteInputTokens?: number;
   };
+  message?: string;
 }
 
-export enum BEDROCK_STOP_REASON {
+export enum BEDROCK_CONVERSE_STOP_REASON {
   end_turn = 'end_turn',
   tool_use = 'tool_use',
   max_tokens = 'max_tokens',
   stop_sequence = 'stop_sequence',
   guardrail_intervened = 'guardrail_intervened',
   content_filtered = 'content_filtered',
 }
+
+export enum TITAN_STOP_REASON {
+  FINISHED = 'FINISHED',
+  LENGTH = 'LENGTH',
+  STOP_CRITERIA_MET = 'STOP_CRITERIA_MET',
+  RAG_QUERY_WHEN_RAG_DISABLED = 'RAG_QUERY_WHEN_RAG_DISABLED',
+  CONTENT_FILTERED = 'CONTENT_FILTERED',
+}

src/providers/dashscope/api.ts

@@ -1,7 +1,7 @@
 import { ProviderAPIConfig } from '../types';
 
 export const dashscopeAPIConfig: ProviderAPIConfig = {
-  getBaseURL: () => 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+  getBaseURL: () => 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1',
   headers({ providerOptions }) {
     const { apiKey } = providerOptions;
     return { Authorization: `Bearer ${apiKey}` };

src/providers/dashscope/index.ts

@@ -8,7 +8,30 @@ import { ProviderConfigs } from '../types';
 import { dashscopeAPIConfig } from './api';
 
 export const DashScopeConfig: ProviderConfigs = {
-  chatComplete: chatCompleteParams([], { model: 'qwen-turbo' }),
+  chatComplete: chatCompleteParams(
+    [],
+    { model: 'qwen-turbo' },
+    {
+      top_k: {
+        param: 'top_k',
+      },
+      repetition_penalty: {
+        param: 'repetition_penalty',
+      },
+      stop: {
+        param: 'stop',
+      },
+      enable_search: {
+        param: 'enable_search',
+      },
+      enable_thinking: {
+        param: 'enable_thinking',
+      },
+      thinking_budget: {
+        param: 'thinking_budget',
+      },
+    }
+  ),
   embed: embedParams([], { model: 'text-embedding-v1' }),
   api: dashscopeAPIConfig,
   responseTransforms: responseTransformers(DASHSCOPE, {