Merge pull request #1168 from narengogi/feature/v1-messages-route

Feature: Beta support for /v1/messages route

Author: VisargD

src/handlers/messagesHandler.ts

@@ -0,0 +1,55 @@
+import { RouterError } from '../errors/RouterError';
+import {
+  constructConfigFromRequestHeaders,
+  tryTargetsRecursively,
+} from './handlerUtils';
+import { Context } from 'hono';
+
+/**
+ * Handles the '/messages' API request by selecting the appropriate provider(s) and making the request to them.
+ *
+ * @param {Context} c - The Cloudflare Worker context.
+ * @returns {Promise<Response>} - The response from the provider.
+ * @throws Will throw an error if no provider options can be determined or if the request to the provider(s) fails.
+ * @throws Will throw an 500 error if the handler fails due to some reasons
+ */
+export async function messagesHandler(c: Context): Promise<Response> {
+  try {
+    let request = await c.req.json();
+    let requestHeaders = Object.fromEntries(c.req.raw.headers);
+    const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders);
+    const tryTargetsResponse = await tryTargetsRecursively(
+      c,
+      camelCaseConfig ?? {},
+      request,
+      requestHeaders,
+      'messages',
+      'POST',
+      'config'
+    );
+
+    return tryTargetsResponse;
+  } catch (err: any) {
+    console.log('messages error', err.message);
+    let statusCode = 500;
+    let errorMessage = 'Something went wrong';
+
+    if (err instanceof RouterError) {
+      statusCode = 400;
+      errorMessage = err.message;
+    }
+
+    return new Response(
+      JSON.stringify({
+        status: 'failure',
+        message: errorMessage,
+      }),
+      {
+        status: statusCode,
+        headers: {
+          'content-type': 'application/json',
+        },
+      }
+    );
+  }
+}

src/handlers/responseHandlers.ts

@@ -17,6 +17,7 @@ import {
 import { HookSpan } from '../middlewares/hooks';
 import { env } from 'hono/adapter';
 import { OpenAIModelResponseJSONToStreamGenerator } from '../providers/open-ai-base/createModelResponse';
+import { anthropicMessagesJsonToStreamGenerator } from '../providers/anthropic-base/utils/streamGenerator';
 
 /**
  * Handles various types of responses based on the specified parameters
@@ -81,6 +82,9 @@ export async function responseHandler(
         responseTransformerFunction =
           OpenAIChatCompleteJSONToStreamResponseTransform;
         break;
+      case 'messages':
+        responseTransformerFunction = anthropicMessagesJsonToStreamGenerator;
+        break;
       case 'createModelResponse':
         responseTransformerFunction = OpenAIModelResponseJSONToStreamGenerator;
         break;

src/index.ts

@@ -31,6 +31,7 @@ import { realTimeHandler } from './handlers/realtimeHandler';
 import filesHandler from './handlers/filesHandler';
 import batchesHandler from './handlers/batchesHandler';
 import finetuneHandler from './handlers/finetuneHandler';
+import { messagesHandler } from './handlers/messagesHandler';
 
 // Config
 import conf from '../conf.json';
@@ -117,6 +118,11 @@ app.onError((err, c) => {
   return c.json({ status: 'failure', message: err.message });
 });
 
+/**
+ * POST route for '/v1/messages' in anthropic format
+ */
+app.post('/v1/messages', requestValidator, messagesHandler);
+
 /**
  * POST route for '/v1/chat/completions'.
  * Handles requests by passing them to the chatCompletionsHandler.

src/providers/anthropic-base/constants.ts

@@ -0,0 +1,51 @@
+export const ANTHROPIC_MESSAGE_START_EVENT = JSON.stringify({
+  type: 'message_start',
+  message: {
+    id: '',
+    type: 'message',
+    role: 'assistant',
+    model: '',
+    content: [],
+    stop_reason: null,
+    stop_sequence: null,
+    usage: {
+      input_tokens: 0,
+      cache_creation_input_tokens: 0,
+      cache_read_input_tokens: 0,
+      output_tokens: 0,
+    },
+  },
+});
+
+export const ANTHROPIC_MESSAGE_DELTA_EVENT = JSON.stringify({
+  type: 'message_delta',
+  delta: {
+    stop_reason: '',
+    stop_sequence: null,
+  },
+  usage: {
+    input_tokens: 0,
+    output_tokens: 0,
+    cache_read_input_tokens: 0,
+    cache_creation_input_tokens: 0,
+  },
+});
+
+export const ANTHROPIC_MESSAGE_STOP_EVENT = {
+  type: 'message_stop',
+};
+
+export const ANTHROPIC_CONTENT_BLOCK_STOP_EVENT = JSON.stringify({
+  type: 'content_block_stop',
+  index: 0,
+});
+
+export const ANTHROPIC_CONTENT_BLOCK_START_EVENT = JSON.stringify({
+  type: 'content_block_start',
+  index: 1,
+  // handle other content block types here
+  content_block: {
+    type: 'text',
+    text: '',
+  },
+});

src/providers/anthropic-base/messages.ts

@@ -0,0 +1,95 @@
+import { ProviderConfig } from '../types';
+
+export const messagesBaseConfig: ProviderConfig = {
+  model: {
+    param: 'model',
+    required: true,
+  },
+  messages: {
+    param: 'messages',
+    required: true,
+  },
+  max_tokens: {
+    param: 'max_tokens',
+    required: true,
+  },
+  container: {
+    param: 'container',
+    required: false,
+  },
+  mcp_servers: {
+    param: 'mcp_servers',
+    required: false,
+  },
+  metadata: {
+    param: 'metadata',
+    required: false,
+  },
+  service_tier: {
+    param: 'service_tier',
+    required: false,
+  },
+  stop_sequences: {
+    param: 'stop_sequences',
+    required: false,
+  },
+  stream: {
+    param: 'stream',
+    required: false,
+  },
+  system: {
+    param: 'system',
+  },
+  temperature: {
+    param: 'temperature',
+    required: false,
+  },
+  thinking: {
+    param: 'thinking',
+    required: false,
+  },
+  tool_choice: {
+    param: 'tool_choice',
+    required: false,
+  },
+  tools: {
+    param: 'tools',
+    required: false,
+  },
+  top_k: {
+    param: 'top_k',
+    required: false,
+  },
+  top_p: {
+    param: 'top_p',
+    required: false,
+  },
+};
+
+export const getMessagesConfig = ({
+  exclude = [],
+  defaultValues = {},
+  extra = {},
+}: {
+  exclude?: string[];
+  defaultValues?: Record<
+    keyof typeof messagesBaseConfig,
+    string | number | boolean
+  >;
+  extra?: ProviderConfig;
+}): ProviderConfig => {
+  const baseParams = { ...messagesBaseConfig };
+  if (defaultValues) {
+    Object.keys(defaultValues).forEach((key) => {
+      if (!Array.isArray(baseParams[key])) {
+        baseParams[key].default = defaultValues[key];
+      }
+    });
+  }
+  exclude.forEach((key) => {
+    // not checking if the key exists as if it doesnt, a build failure is expected
+    delete baseParams[key];
+  });
+
+  return { ...baseParams, ...extra };
+};

src/providers/anthropic-base/types.ts

@@ -0,0 +1,32 @@
+export interface AnthropicMessageStartEvent {
+  type: 'message_start';
+  message: {
+    id: string;
+    type: 'message';
+    role: 'assistant';
+    model: string;
+    content: any[];
+    stop_reason: string | null;
+    stop_sequence: string | null;
+    usage?: {
+      input_tokens: number;
+      cache_creation_input_tokens: number;
+      cache_read_input_tokens: number;
+      output_tokens: number;
+    };
+  };
+}
+
+export interface AnthropicMessageDeltaEvent {
+  type: 'message_delta';
+  delta: {
+    stop_reason: string;
+    stop_sequence: string | null;
+  };
+  usage: {
+    input_tokens?: number;
+    output_tokens: number;
+    cache_read_input_tokens?: number;
+    cache_creation_input_tokens?: number;
+  };
+}