Merge pull request #1167 from RooVetGit/cte/claude-3.7-thinking

Claude 3.7 thinking

Author: cte

package-lock.json

@@ -304,7 +304,7 @@
 	},
 	"dependencies": {
 		"@anthropic-ai/bedrock-sdk": "^0.10.2",
-		"@anthropic-ai/sdk": "^0.26.0",
+		"@anthropic-ai/sdk": "^0.37.0",
 		"@anthropic-ai/vertex-sdk": "^0.4.1",
 		"@aws-sdk/client-bedrock-runtime": "^3.706.0",
 		"@google/generative-ai": "^0.18.0",

package.json

@@ -1,50 +1,13 @@
+// npx jest src/api/providers/__tests__/anthropic.test.ts
+
 import { AnthropicHandler } from "../anthropic"
 import { ApiHandlerOptions } from "../../../shared/api"
-import { ApiStream } from "../../transform/stream"
-import { Anthropic } from "@anthropic-ai/sdk"
 
-// Mock Anthropic client
-const mockBetaCreate = jest.fn()
 const mockCreate = jest.fn()
+
 jest.mock("@anthropic-ai/sdk", () => {
 	return {
 		Anthropic: jest.fn().mockImplementation(() => ({
-			beta: {
-				promptCaching: {
-					messages: {
-						create: mockBetaCreate.mockImplementation(async () => ({
-							async *[Symbol.asyncIterator]() {
-								yield {
-									type: "message_start",
-									message: {
-										usage: {
-											input_tokens: 100,
-											output_tokens: 50,
-											cache_creation_input_tokens: 20,
-											cache_read_input_tokens: 10,
-										},
-									},
-								}
-								yield {
-									type: "content_block_start",
-									index: 0,
-									content_block: {
-										type: "text",
-										text: "Hello",
-									},
-								}
-								yield {
-									type: "content_block_delta",
-									delta: {
-										type: "text_delta",
-										text: " world",
-									},
-								}
-							},
-						})),
-					},
-				},
-			},
 			messages: {
 				create: mockCreate.mockImplementation(async (options) => {
 					if (!options.stream) {
@@ -65,16 +28,26 @@ jest.mock("@anthropic-ai/sdk", () => {
 								type: "message_start",
 								message: {
 									usage: {
-										input_tokens: 10,
-										output_tokens: 5,
+										input_tokens: 100,
+										output_tokens: 50,
+										cache_creation_input_tokens: 20,
+										cache_read_input_tokens: 10,
 									},
 								},
 							}
 							yield {
 								type: "content_block_start",
+								index: 0,
 								content_block: {
 									type: "text",
-									text: "Test response",
+									text: "Hello",
+								},
+							}
+							yield {
+								type: "content_block_delta",
+								delta: {
+									type: "text_delta",
+									text: " world",
 								},
 							}
 						},
@@ -95,7 +68,6 @@ describe("AnthropicHandler", () => {
 			apiModelId: "claude-3-5-sonnet-20241022",
 		}
 		handler = new AnthropicHandler(mockOptions)
-		mockBetaCreate.mockClear()
 		mockCreate.mockClear()
 	})
 
@@ -126,17 +98,6 @@ describe("AnthropicHandler", () => {
 
 	describe("createMessage", () => {
 		const systemPrompt = "You are a helpful assistant."
-		const messages: Anthropic.Messages.MessageParam[] = [
-			{
-				role: "user",
-				content: [
-					{
-						type: "text" as const,
-						text: "Hello!",
-					},
-				],
-			},
-		]
 
 		it("should handle prompt caching for supported models", async () => {
 			const stream = handler.createMessage(systemPrompt, [
@@ -173,9 +134,8 @@ describe("AnthropicHandler", () => {
 			expect(textChunks[0].text).toBe("Hello")
 			expect(textChunks[1].text).toBe(" world")
 
-			// Verify beta API was used
-			expect(mockBetaCreate).toHaveBeenCalled()
-			expect(mockCreate).not.toHaveBeenCalled()
+			// Verify API
+			expect(mockCreate).toHaveBeenCalled()
 		})
 	})
 

src/api/providers/__tests__/anthropic.test.ts

@@ -1,5 +1,7 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
+import { CacheControlEphemeral } from "@anthropic-ai/sdk/resources"
+import { BetaThinkingConfigParam } from "@anthropic-ai/sdk/resources/beta"
 import {
 	anthropicDefaultModelId,
 	AnthropicModelId,
@@ -12,60 +14,73 @@ import { ApiStream } from "../transform/stream"
 
 const ANTHROPIC_DEFAULT_TEMPERATURE = 0
 
+const THINKING_MODELS = ["claude-3-7-sonnet-20250219"]
+
 export class AnthropicHandler implements ApiHandler, SingleCompletionHandler {
 	private options: ApiHandlerOptions
 	private client: Anthropic
 
 	constructor(options: ApiHandlerOptions) {
 		this.options = options
+
 		this.client = new Anthropic({
 			apiKey: this.options.apiKey,
 			baseURL: this.options.anthropicBaseUrl || undefined,
 		})
 	}
 
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		let stream: AnthropicStream<Anthropic.Beta.PromptCaching.Messages.RawPromptCachingBetaMessageStreamEvent>
+		let stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>
+		const cacheControl: CacheControlEphemeral = { type: "ephemeral" }
 		const modelId = this.getModel().id
+		const maxTokens = this.getModel().info.maxTokens || 8192
+		let temperature = this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE
+		let thinking: BetaThinkingConfigParam | undefined = undefined
+
+		if (THINKING_MODELS.includes(modelId)) {
+			thinking = this.options.anthropicThinking
+				? { type: "enabled", budget_tokens: this.options.anthropicThinking }
+				: { type: "disabled" }
+
+			temperature = 1.0
+		}
 
 		switch (modelId) {
-			// 'latest' alias does not support cache_control
 			case "claude-3-7-sonnet-20250219":
 			case "claude-3-5-sonnet-20241022":
 			case "claude-3-5-haiku-20241022":
 			case "claude-3-opus-20240229":
 			case "claude-3-haiku-20240307": {
-				/*
-				The latest message will be the new user message, one before will be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request..
-				*/
+				/**
+				 * The latest message will be the new user message, one before will
+				 * be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request..
+				 */
 				const userMsgIndices = messages.reduce(
 					(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
 					[] as number[],
 				)
+
 				const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
 				const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
-				stream = await this.client.beta.promptCaching.messages.create(
+
+				stream = await this.client.messages.create(
 					{
 						model: modelId,
-						max_tokens: this.getModel().info.maxTokens || 8192,
-						temperature: this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE,
-						system: [{ text: systemPrompt, type: "text", cache_control: { type: "ephemeral" } }], // setting cache breakpoint for system prompt so new tasks can reuse it
+						max_tokens: maxTokens,
+						temperature,
+						thinking,
+						// Setting cache breakpoint for system prompt so new tasks can reuse it.
+						system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }],
 						messages: messages.map((message, index) => {
 							if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) {
 								return {
 									...message,
 									content:
 										typeof message.content === "string"
-											? [
-													{
-														type: "text",
-														text: message.content,
-														cache_control: { type: "ephemeral" },
-													},
-												]
+											? [{ type: "text", text: message.content, cache_control: cacheControl }]
 											: message.content.map((content, contentIndex) =>
 													contentIndex === message.content.length - 1
-														? { ...content, cache_control: { type: "ephemeral" } }
+														? { ...content, cache_control: cacheControl }
 														: content,
 												),
 								}
@@ -114,54 +129,63 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler {
 		for await (const chunk of stream) {
 			switch (chunk.type) {
 				case "message_start":
-					// tells us cache reads/writes/input/output
+					// Tells us cache reads/writes/input/output.
 					const usage = chunk.message.usage
+
 					yield {
 						type: "usage",
 						inputTokens: usage.input_tokens || 0,
 						outputTokens: usage.output_tokens || 0,
 						cacheWriteTokens: usage.cache_creation_input_tokens || undefined,
 						cacheReadTokens: usage.cache_read_input_tokens || undefined,
 					}
+
 					break
 				case "message_delta":
-					// tells us stop_reason, stop_sequence, and output tokens along the way and at the end of the message
-
+					// Tells us stop_reason, stop_sequence, and output tokens
+					// along the way and at the end of the message.
 					yield {
 						type: "usage",
 						inputTokens: 0,
 						outputTokens: chunk.usage.output_tokens || 0,
 					}
+
 					break
 				case "message_stop":
-					// no usage data, just an indicator that the message is done
+					// No usage data, just an indicator that the message is done.
 					break
 				case "content_block_start":
 					switch (chunk.content_block.type) {
-						case "text":
-							// we may receive multiple text blocks, in which case just insert a line break between them
+						case "thinking":
+							// We may receive multiple text blocks, in which
+							// case just insert a line break between them.
 							if (chunk.index > 0) {
-								yield {
-									type: "text",
-									text: "\n",
-								}
+								yield { type: "reasoning", text: "\n" }
 							}
-							yield {
-								type: "text",
-								text: chunk.content_block.text,
+
+							yield { type: "reasoning", text: chunk.content_block.thinking }
+							break
+						case "text":
+							// We may receive multiple text blocks, in which
+							// case just insert a line break between them.
+							if (chunk.index > 0) {
+								yield { type: "text", text: "\n" }
 							}
+
+							yield { type: "text", text: chunk.content_block.text }
 							break
 					}
 					break
 				case "content_block_delta":
 					switch (chunk.delta.type) {
+						case "thinking_delta":
+							yield { type: "reasoning", text: chunk.delta.thinking }
+							break
 						case "text_delta":
-							yield {
-								type: "text",
-								text: chunk.delta.text,
-							}
+							yield { type: "text", text: chunk.delta.text }
 							break
 					}
+
 					break
 				case "content_block_stop":
 					break
@@ -171,10 +195,12 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler {
 
 	getModel(): { id: AnthropicModelId; info: ModelInfo } {
 		const modelId = this.options.apiModelId
+
 		if (modelId && modelId in anthropicModels) {
 			const id = modelId as AnthropicModelId
 			return { id, info: anthropicModels[id] }
 		}
+
 		return { id: anthropicDefaultModelId, info: anthropicModels[anthropicDefaultModelId] }
 	}
 
@@ -189,14 +215,17 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler {
 			})
 
 			const content = response.content[0]
+
 			if (content.type === "text") {
 				return content.text
 			}
+
 			return ""
 		} catch (error) {
 			if (error instanceof Error) {
 				throw new Error(`Anthropic completion error: ${error.message}`)
 			}
+
 			throw error
 		}
 	}

src/api/providers/anthropic.ts

@@ -9,7 +9,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import { ApiHandler, SingleCompletionHandler } from "../"
 import { ApiHandlerOptions, BedrockModelId, ModelInfo, bedrockDefaultModelId, bedrockModels } from "../../shared/api"
 import { ApiStream } from "../transform/stream"
-import { convertToBedrockConverseMessages, convertToAnthropicMessage } from "../transform/bedrock-converse-format"
+import { convertToBedrockConverseMessages } from "../transform/bedrock-converse-format"
 
 const BEDROCK_DEFAULT_TEMPERATURE = 0.3