Merge pull request #1165 from narengogi/chore/openai-compliance-improvements

improve strict openai compliance

Author: VisargD

src/providers/anthropic/chatComplete.ts

@@ -14,8 +14,9 @@ import {
 import {
   generateErrorResponse,
   generateInvalidProviderResponseError,
+  transformFinishReason,
 } from '../utils';
-import { AnthropicStreamState } from './types';
+import { ANTHROPIC_STOP_REASON, AnthropicStreamState } from './types';
 
 // TODO: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model.
 
@@ -456,7 +457,7 @@ export interface AnthropicChatCompleteResponse {
   type: string;
   role: string;
   content: AnthropicContentItem[];
-  stop_reason: string;
+  stop_reason: ANTHROPIC_STOP_REASON;
   model: string;
   stop_sequence: null | string;
   usage: {
@@ -474,7 +475,7 @@ export interface AnthropicChatCompleteStreamResponse {
     type?: string;
     text?: string;
     partial_json?: string;
-    stop_reason?: string;
+    stop_reason?: ANTHROPIC_STOP_REASON;
   };
   content_block?: {
     type: string;
@@ -590,7 +591,10 @@ export const AnthropicChatCompleteResponseTransform: (
           },
           index: 0,
           logprobs: null,
-          finish_reason: response.stop_reason,
+          finish_reason: transformFinishReason(
+            response.stop_reason,
+            strictOpenAiCompliance
+          ),
         },
       ],
       usage: {
@@ -703,6 +707,7 @@ export const AnthropicChatCompleteStreamChunkTransform: (
     );
   }
 
+  // final chunk
   if (parsedChunk.type === 'message_delta' && parsedChunk.usage) {
     const totalTokens =
       (streamState?.usage?.prompt_tokens ?? 0) +
@@ -720,7 +725,10 @@ export const AnthropicChatCompleteStreamChunkTransform: (
           {
             index: 0,
             delta: {},
-            finish_reason: parsedChunk.delta?.stop_reason,
+            finish_reason: transformFinishReason(
+              parsedChunk.delta?.stop_reason,
+              strictOpenAiCompliance
+            ),
           },
         ],
         usage: {
@@ -791,7 +799,7 @@ export const AnthropicChatCompleteStreamChunkTransform: (
           },
           index: 0,
           logprobs: null,
-          finish_reason: parsedChunk.delta?.stop_reason ?? null,
+          finish_reason: null,
         },
       ],
     })}` + '\n\n'

src/providers/anthropic/types.ts

@@ -8,3 +8,12 @@ export type AnthropicStreamState = {
   };
   model?: string;
 };
+
+// https://docs.anthropic.com/en/api/messages#response-stop-reason
+export enum ANTHROPIC_STOP_REASON {
+  max_tokens = 'max_tokens',
+  stop_sequence = 'stop_sequence',
+  tool_use = 'tool_use',
+  end_turn = 'end_turn',
+  pause_turn = 'pause_turn',
+}

src/providers/bedrock/chatComplete.ts

@@ -20,13 +20,15 @@ import {
 import {
   generateErrorResponse,
   generateInvalidProviderResponseError,
+  transformFinishReason,
 } from '../utils';
 import {
   BedrockAI21CompleteResponse,
   BedrockCohereCompleteResponse,
   BedrockCohereStreamChunk,
 } from './complete';
 import { BedrockErrorResponse } from './embed';
+import { BEDROCK_STOP_REASON } from './types';
 import {
   getBedrockErrorChunk,
   transformAdditionalModelRequestFields,
@@ -452,7 +454,7 @@ interface BedrockChatCompletionResponse {
       content: BedrockContentItem[];
     };
   };
-  stopReason: string;
+  stopReason: BEDROCK_STOP_REASON;
   usage: {
     inputTokens: number;
     outputTokens: number;
@@ -553,7 +555,10 @@ export const BedrockChatCompleteResponseTransform: (
               content_blocks: contentBlocks,
             }),
           },
-          finish_reason: response.stopReason,
+          finish_reason: transformFinishReason(
+            response.stopReason,
+            strictOpenAiCompliance
+          ),
         },
       ],
       usage: {
@@ -608,7 +613,7 @@ export interface BedrockChatCompleteStreamChunk {
       input?: object;
     };
   };
-  stopReason?: string;
+  stopReason?: BEDROCK_STOP_REASON;
   metrics?: {
     latencyMs: number;
   };
@@ -624,7 +629,7 @@ export interface BedrockChatCompleteStreamChunk {
 }
 
 interface BedrockStreamState {
-  stopReason?: string;
+  stopReason?: BEDROCK_STOP_REASON;
   currentToolCallIndex?: number;
 }
 
@@ -653,6 +658,7 @@ export const BedrockChatCompleteStreamChunkTransform: (
     streamState.currentToolCallIndex = -1;
   }
 
+  // final chunk
   if (parsedChunk.usage) {
     const shouldSendCacheUsage =
       parsedChunk.usage.cacheWriteInputTokens ||
@@ -668,7 +674,10 @@ export const BedrockChatCompleteStreamChunkTransform: (
           {
             index: 0,
             delta: {},
-            finish_reason: streamState.stopReason,
+            finish_reason: transformFinishReason(
+              streamState.stopReason,
+              strictOpenAiCompliance
+            ),
           },
         ],
         usage: {

src/providers/bedrock/types.ts

@@ -78,3 +78,12 @@ export interface BedrockInferenceProfile {
   status: string;
   type: string;
 }
+
+export enum BEDROCK_STOP_REASON {
+  end_turn = 'end_turn',
+  tool_use = 'tool_use',
+  max_tokens = 'max_tokens',
+  stop_sequence = 'stop_sequence',
+  guardrail_intervened = 'guardrail_intervened',
+  content_filtered = 'content_filtered',
+}

src/providers/types.ts

@@ -1,5 +1,7 @@
 import { Context } from 'hono';
 import { Message, Options, Params } from '../types/requestBody';
+import { ANTHROPIC_STOP_REASON } from './anthropic/types';
+import { BEDROCK_STOP_REASON } from './bedrock/types';
 
 /**
  * Configuration for a parameter.
@@ -400,3 +402,15 @@ export interface StreamContentBlock {
     data?: string;
   };
 }
+
+export enum FINISH_REASON {
+  stop = 'stop',
+  length = 'length',
+  tool_calls = 'tool_calls',
+  content_filter = 'content_filter',
+  function_call = 'function_call',
+}
+
+export type PROVIDER_FINISH_REASON =
+  | ANTHROPIC_STOP_REASON
+  | BEDROCK_STOP_REASON;

src/providers/utils.ts

@@ -1,4 +1,5 @@
-import { ErrorResponse } from './types';
+import { FINISH_REASON, ErrorResponse, PROVIDER_FINISH_REASON } from './types';
+import { finishReasonMap } from './utils/finishReasonMap';
 
 export const generateInvalidProviderResponseError: (
   response: Record<string, any>,
@@ -57,3 +58,23 @@ export function splitString(input: string, separator: string): SplitResult {
     after: input.substring(sepIndex + 1),
   };
 }
+
+/*
+  Transforms the finish reason from the provider to the finish reason used by the OpenAI API.
+  If the finish reason is not found in the map, it will return the stop reason.
+  If the strictOpenAiCompliance is true, it will return the finish reason from the map.
+  If the strictOpenAiCompliance is false, it will return the finish reason from the provider.
+  NOTE: this function always returns a finish reason
+*/
+export const transformFinishReason = (
+  finishReason?: PROVIDER_FINISH_REASON,
+  strictOpenAiCompliance?: boolean
+): FINISH_REASON | PROVIDER_FINISH_REASON => {
+  if (!finishReason) return FINISH_REASON.stop;
+  if (!strictOpenAiCompliance) return finishReason;
+  const transformedFinishReason = finishReasonMap.get(finishReason);
+  if (!transformedFinishReason) {
+    return FINISH_REASON.stop;
+  }
+  return transformedFinishReason;
+};

src/providers/utils/finishReasonMap.ts

@@ -0,0 +1,19 @@
+import { ANTHROPIC_STOP_REASON } from '../anthropic/types';
+import { FINISH_REASON, PROVIDER_FINISH_REASON } from '../types';
+import { BEDROCK_STOP_REASON } from '../bedrock/types';
+
+export const finishReasonMap = new Map<PROVIDER_FINISH_REASON, FINISH_REASON>([
+  // https://docs.anthropic.com/en/api/messages#response-stop-reason
+  [ANTHROPIC_STOP_REASON.stop_sequence, FINISH_REASON.stop],
+  [ANTHROPIC_STOP_REASON.end_turn, FINISH_REASON.stop],
+  [ANTHROPIC_STOP_REASON.pause_turn, FINISH_REASON.stop],
+  [ANTHROPIC_STOP_REASON.tool_use, FINISH_REASON.tool_calls],
+  [ANTHROPIC_STOP_REASON.max_tokens, FINISH_REASON.length],
+  // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html#API_runtime_Converse_ResponseSyntax
+  [BEDROCK_STOP_REASON.end_turn, FINISH_REASON.stop],
+  [BEDROCK_STOP_REASON.tool_use, FINISH_REASON.tool_calls],
+  [BEDROCK_STOP_REASON.max_tokens, FINISH_REASON.length],
+  [BEDROCK_STOP_REASON.stop_sequence, FINISH_REASON.stop],
+  [BEDROCK_STOP_REASON.guardrail_intervened, FINISH_REASON.content_filter],
+  [BEDROCK_STOP_REASON.content_filtered, FINISH_REASON.content_filter],
+]);