Merge branch 'main' into feature/add-qualifire-integration

Author: yuval-qf

src/providers/anthropic/chatComplete.ts

@@ -606,6 +606,9 @@ export const AnthropicChatCompleteStreamChunkTransform: (
   streamState,
   strictOpenAiCompliance
 ) => {
+  if (streamState.toolIndex == undefined) {
+    streamState.toolIndex = -1;
+  }
   let chunk = responseChunk.trim();
   if (
     chunk.startsWith('event: ping') ||
@@ -724,9 +727,7 @@ export const AnthropicChatCompleteStreamChunkTransform: (
     parsedChunk.type === 'content_block_start' &&
     parsedChunk.content_block?.type === 'tool_use';
   if (isToolBlockStart) {
-    streamState.toolIndex = streamState.toolIndex
-      ? streamState.toolIndex + 1
-      : 0;
+    streamState.toolIndex = streamState.toolIndex + 1;
   }
   const isToolBlockDelta: boolean =
     parsedChunk.type === 'content_block_delta' &&

src/providers/anthropic/complete.ts

@@ -1,9 +1,16 @@
 import { ANTHROPIC } from '../../globals';
 import { Params } from '../../types/requestBody';
 import { CompletionResponse, ErrorResponse, ProviderConfig } from '../types';
-import { generateInvalidProviderResponseError } from '../utils';
+import {
+  generateInvalidProviderResponseError,
+  transformFinishReason,
+} from '../utils';
+import {
+  ANTHROPIC_STOP_REASON,
+  AnthropicStreamState,
+  AnthropicErrorResponse,
+} from './types';
 import { AnthropicErrorResponseTransform } from './utils';
-import { AnthropicErrorResponse } from './types';
 
 // TODO: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model.
 
@@ -57,7 +64,7 @@ export const AnthropicCompleteConfig: ProviderConfig = {
 
 interface AnthropicCompleteResponse {
   completion: string;
-  stop_reason: string;
+  stop_reason: ANTHROPIC_STOP_REASON;
   model: string;
   truncated: boolean;
   stop: null | string;
@@ -68,10 +75,20 @@ interface AnthropicCompleteResponse {
 // TODO: The token calculation is wrong atm
 export const AnthropicCompleteResponseTransform: (
   response: AnthropicCompleteResponse | AnthropicErrorResponse,
-  responseStatus: number
-) => CompletionResponse | ErrorResponse = (response, responseStatus) => {
-  if (responseStatus !== 200 && 'error' in response) {
-    return AnthropicErrorResponseTransform(response);
+  responseStatus: number,
+  responseHeaders: Headers,
+  strictOpenAiCompliance: boolean
+) => CompletionResponse | ErrorResponse = (
+  response,
+  responseStatus,
+  _responseHeaders,
+  strictOpenAiCompliance
+) => {
+  if (responseStatus !== 200) {
+    const errorResposne = AnthropicErrorResponseTransform(
+      response as AnthropicErrorResponse
+    );
+    if (errorResposne) return errorResposne;
   }
 
   if ('completion' in response) {
@@ -86,7 +103,10 @@ export const AnthropicCompleteResponseTransform: (
           text: response.completion,
           index: 0,
           logprobs: null,
-          finish_reason: response.stop_reason,
+          finish_reason: transformFinishReason(
+            response.stop_reason,
+            strictOpenAiCompliance
+          ),
         },
       ],
     };
@@ -96,8 +116,16 @@ export const AnthropicCompleteResponseTransform: (
 };
 
 export const AnthropicCompleteStreamChunkTransform: (
-  response: string
-) => string | undefined = (responseChunk) => {
+  response: string,
+  fallbackId: string,
+  streamState: AnthropicStreamState,
+  strictOpenAiCompliance: boolean
+) => string | undefined = (
+  responseChunk,
+  fallbackId,
+  streamState,
+  strictOpenAiCompliance
+) => {
   let chunk = responseChunk.trim();
   if (chunk.startsWith('event: ping')) {
     return;
@@ -110,6 +138,9 @@ export const AnthropicCompleteStreamChunkTransform: (
     return chunk;
   }
   const parsedChunk: AnthropicCompleteResponse = JSON.parse(chunk);
+  const finishReason = parsedChunk.stop_reason
+    ? transformFinishReason(parsedChunk.stop_reason, strictOpenAiCompliance)
+    : null;
   return (
     `data: ${JSON.stringify({
       id: parsedChunk.log_id,
@@ -122,7 +153,7 @@ export const AnthropicCompleteStreamChunkTransform: (
           text: parsedChunk.completion,
           index: 0,
           logprobs: null,
-          finish_reason: parsedChunk.stop_reason,
+          finish_reason: finishReason,
         },
       ],
     })}` + '\n\n'

src/providers/bedrock/complete.ts

@@ -1,9 +1,13 @@
 import { BEDROCK } from '../../globals';
 import { Params } from '../../types/requestBody';
 import { CompletionResponse, ErrorResponse, ProviderConfig } from '../types';
-import { generateInvalidProviderResponseError } from '../utils';
+import {
+  generateInvalidProviderResponseError,
+  transformFinishReason,
+} from '../utils';
 import { BedrockErrorResponseTransform } from './chatComplete';
 import { BedrockErrorResponse } from './embed';
+import { TITAN_STOP_REASON as TITAN_COMPLETION_REASON } from './types';
 
 export const BedrockAnthropicCompleteConfig: ProviderConfig = {
   prompt: {
@@ -380,7 +384,7 @@ export interface BedrockTitanCompleteResponse {
   results: {
     tokenCount: number;
     outputText: string;
-    completionReason: string;
+    completionReason: TITAN_COMPLETION_REASON;
   }[];
 }
 
@@ -420,7 +424,10 @@ export const BedrockTitanCompleteResponseTransform: (
         text: generation.outputText,
         index: index,
         logprobs: null,
-        finish_reason: generation.completionReason,
+        finish_reason: transformFinishReason(
+          generation.completionReason,
+          strictOpenAiCompliance
+        ),
       })),
       usage: {
         prompt_tokens: response.inputTextTokenCount,
@@ -437,7 +444,7 @@ export interface BedrockTitanStreamChunk {
   outputText: string;
   index: number;
   totalOutputTextTokenCount: number;
-  completionReason: string | null;
+  completionReason: TITAN_COMPLETION_REASON | null;
   'amazon-bedrock-invocationMetrics': {
     inputTokenCount: number;
     outputTokenCount: number;
@@ -462,6 +469,12 @@ export const BedrockTitanCompleteStreamChunkTransform: (
   let chunk = responseChunk.trim();
   chunk = chunk.trim();
   const parsedChunk: BedrockTitanStreamChunk = JSON.parse(chunk);
+  const finishReason = parsedChunk.completionReason
+    ? transformFinishReason(
+        parsedChunk.completionReason,
+        _strictOpenAiCompliance
+      )
+    : null;
 
   return [
     `data: ${JSON.stringify({
@@ -490,7 +503,7 @@ export const BedrockTitanCompleteStreamChunkTransform: (
           text: '',
           index: 0,
           logprobs: null,
-          finish_reason: parsedChunk.completionReason,
+          finish_reason: finishReason,
         },
       ],
       usage: {

src/providers/bedrock/types.ts

@@ -108,7 +108,7 @@ export interface BedrockChatCompletionResponse {
       content: BedrockContentItem[];
     };
   };
-  stopReason: BEDROCK_STOP_REASON;
+  stopReason: BEDROCK_CONVERSE_STOP_REASON;
   usage: {
     inputTokens: number;
     outputTokens: number;
@@ -156,7 +156,7 @@ export type BedrockContentItem = {
 };
 
 export interface BedrockStreamState {
-  stopReason?: BEDROCK_STOP_REASON;
+  stopReason?: BEDROCK_CONVERSE_STOP_REASON;
   currentToolCallIndex?: number;
   currentContentBlockIndex?: number;
 }
@@ -186,7 +186,7 @@ export interface BedrockChatCompleteStreamChunk {
       input?: object;
     };
   };
-  stopReason?: BEDROCK_STOP_REASON;
+  stopReason?: BEDROCK_CONVERSE_STOP_REASON;
   metrics?: {
     latencyMs: number;
   };
@@ -199,13 +199,22 @@ export interface BedrockChatCompleteStreamChunk {
     cacheWriteInputTokenCount?: number;
     cacheWriteInputTokens?: number;
   };
+  message?: string;
 }
 
-export enum BEDROCK_STOP_REASON {
+export enum BEDROCK_CONVERSE_STOP_REASON {
   end_turn = 'end_turn',
   tool_use = 'tool_use',
   max_tokens = 'max_tokens',
   stop_sequence = 'stop_sequence',
   guardrail_intervened = 'guardrail_intervened',
   content_filtered = 'content_filtered',
 }
+
+export enum TITAN_STOP_REASON {
+  FINISHED = 'FINISHED',
+  LENGTH = 'LENGTH',
+  STOP_CRITERIA_MET = 'STOP_CRITERIA_MET',
+  RAG_QUERY_WHEN_RAG_DISABLED = 'RAG_QUERY_WHEN_RAG_DISABLED',
+  CONTENT_FILTERED = 'CONTENT_FILTERED',
+}

src/providers/dashscope/api.ts

@@ -1,7 +1,7 @@
 import { ProviderAPIConfig } from '../types';
 
 export const dashscopeAPIConfig: ProviderAPIConfig = {
-  getBaseURL: () => 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+  getBaseURL: () => 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1',
   headers({ providerOptions }) {
     const { apiKey } = providerOptions;
     return { Authorization: `Bearer ${apiKey}` };

src/providers/dashscope/index.ts

@@ -8,7 +8,30 @@ import { ProviderConfigs } from '../types';
 import { dashscopeAPIConfig } from './api';
 
 export const DashScopeConfig: ProviderConfigs = {
-  chatComplete: chatCompleteParams([], { model: 'qwen-turbo' }),
+  chatComplete: chatCompleteParams(
+    [],
+    { model: 'qwen-turbo' },
+    {
+      top_k: {
+        param: 'top_k',
+      },
+      repetition_penalty: {
+        param: 'repetition_penalty',
+      },
+      stop: {
+        param: 'stop',
+      },
+      enable_search: {
+        param: 'enable_search',
+      },
+      enable_thinking: {
+        param: 'enable_thinking',
+      },
+      thinking_budget: {
+        param: 'thinking_budget',
+      },
+    }
+  ),
   embed: embedParams([], { model: 'text-embedding-v1' }),
   api: dashscopeAPIConfig,
   responseTransforms: responseTransformers(DASHSCOPE, {

src/providers/deepseek/chatComplete.ts

@@ -9,7 +9,9 @@ import {
 import {
   generateErrorResponse,
   generateInvalidProviderResponseError,
+  transformFinishReason,
 } from '../utils';
+import { DEEPSEEK_STOP_REASON } from './types';
 
 export const DeepSeekChatCompleteConfig: ProviderConfig = {
   model: {
@@ -127,8 +129,15 @@ interface DeepSeekStreamChunk {
 
 export const DeepSeekChatCompleteResponseTransform: (
   response: DeepSeekChatCompleteResponse | DeepSeekErrorResponse,
-  responseStatus: number
-) => ChatCompletionResponse | ErrorResponse = (response, responseStatus) => {
+  responseStatus: number,
+  responseHeaders: Headers,
+  strictOpenAiCompliance: boolean
+) => ChatCompletionResponse | ErrorResponse = (
+  response,
+  responseStatus,
+  _responseHeaders,
+  strictOpenAiCompliance
+) => {
   if ('message' in response && responseStatus !== 200) {
     return generateErrorResponse(
       {
@@ -154,7 +163,10 @@ export const DeepSeekChatCompleteResponseTransform: (
           role: c.message.role,
           content: c.message.content,
         },
-        finish_reason: c.finish_reason,
+        finish_reason: transformFinishReason(
+          c.finish_reason as DEEPSEEK_STOP_REASON,
+          strictOpenAiCompliance
+        ),
       })),
       usage: {
         prompt_tokens: response.usage?.prompt_tokens,
@@ -168,15 +180,31 @@ export const DeepSeekChatCompleteResponseTransform: (
 };
 
 export const DeepSeekChatCompleteStreamChunkTransform: (
-  response: string
-) => string = (responseChunk) => {
+  response: string,
+  fallbackId: string,
+  streamState: any,
+  strictOpenAiCompliance: boolean,
+  gatewayRequest: Params
+) => string | string[] = (
+  responseChunk,
+  fallbackId,
+  _streamState,
+  strictOpenAiCompliance,
+  _gatewayRequest
+) => {
   let chunk = responseChunk.trim();
   chunk = chunk.replace(/^data: /, '');
   chunk = chunk.trim();
   if (chunk === '[DONE]') {
     return `data: ${chunk}\n\n`;
   }
   const parsedChunk: DeepSeekStreamChunk = JSON.parse(chunk);
+  const finishReason = parsedChunk.choices[0].finish_reason
+    ? transformFinishReason(
+        parsedChunk.choices[0].finish_reason as DEEPSEEK_STOP_REASON,
+        strictOpenAiCompliance
+      )
+    : null;
   return (
     `data: ${JSON.stringify({
       id: parsedChunk.id,
@@ -188,7 +216,7 @@ export const DeepSeekChatCompleteStreamChunkTransform: (
         {
           index: parsedChunk.choices[0].index,
           delta: parsedChunk.choices[0].delta,
-          finish_reason: parsedChunk.choices[0].finish_reason,
+          finish_reason: finishReason,
         },
       ],
       usage: parsedChunk.usage,

src/providers/deepseek/types.ts

@@ -0,0 +1,7 @@
+export enum DEEPSEEK_STOP_REASON {
+  stop = 'stop',
+  length = 'length',
+  tool_calls = 'tool_calls',
+  content_filter = 'content_filter',
+  insufficient_system_resource = 'insufficient_system_resource',
+}