diff --git a/packages/agent/package.json b/packages/agent/package.json index 9e9da1f..7bb5c9e 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -44,7 +44,7 @@ "author": "Ben Houston", "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0.16.0", + "@anthropic-ai/sdk": "^0.37", "@mozilla/readability": "^0.5.0", "@playwright/test": "^1.50.1", "@vitest/browser": "^3.0.5", diff --git a/packages/agent/src/core/llm/provider.ts b/packages/agent/src/core/llm/provider.ts index dd63486..379bbef 100644 --- a/packages/agent/src/core/llm/provider.ts +++ b/packages/agent/src/core/llm/provider.ts @@ -31,14 +31,6 @@ export interface LLMProvider { * @returns Response with text and/or tool calls */ generateText(options: GenerateOptions): Promise; - - /** - * Get the number of tokens in a given text - * - * @param text Text to count tokens for - * @returns Number of tokens - */ - countTokens(text: string): Promise; } // Provider factory registry diff --git a/packages/agent/src/core/llm/providers/anthropic.ts b/packages/agent/src/core/llm/providers/anthropic.ts index 8be118a..835fac7 100644 --- a/packages/agent/src/core/llm/providers/anthropic.ts +++ b/packages/agent/src/core/llm/providers/anthropic.ts @@ -3,6 +3,7 @@ */ import Anthropic from '@anthropic-ai/sdk'; +import { TokenUsage } from '../../tokens.js'; import { LLMProvider } from '../provider.js'; import { GenerateOptions, @@ -19,6 +20,73 @@ export interface AnthropicOptions extends ProviderOptions { baseUrl?: string; } +// a function that takes a list of messages and returns a list of messages but with the last message having a cache_control of ephemeral +function addCacheControlToTools(messages: T[]): T[] { + return messages.map((m, i) => ({ + ...m, + ...(i === messages.length - 1 + ? { cache_control: { type: 'ephemeral' } } + : {}), + })); +} + +function addCacheControlToContentBlocks( + content: Anthropic.Messages.TextBlock[], +): Anthropic.Messages.TextBlock[] { + return content.map((c, i) => { + if (i === content.length - 1) { + if ( + c.type === 'text' || + c.type === 'document' || + c.type === 'image' || + c.type === 'tool_use' || + c.type === 'tool_result' || + c.type === 'thinking' || + c.type === 'redacted_thinking' + ) { + return { ...c, cache_control: { type: 'ephemeral' } }; + } + } + return c; + }); +} +function addCacheControlToMessages( + messages: Anthropic.Messages.MessageParam[], +): Anthropic.Messages.MessageParam[] { + return messages.map((m, i) => { + if (typeof m.content === 'string') { + return { + ...m, + content: [ + { + type: 'text', + text: m.content, + cache_control: { type: 'ephemeral' }, + }, + ], + }; + } + return { + ...m, + content: + i >= messages.length - 2 + ? addCacheControlToContentBlocks( + m.content as Anthropic.Messages.TextBlock[], + ) + : m.content, + }; + }); +} + +function tokenUsageFromMessage(message: Anthropic.Message) { + const usage = new TokenUsage(); + usage.input = message.usage.input_tokens; + usage.cacheWrites = message.usage.cache_creation_input_tokens ?? 0; + usage.cacheReads = message.usage.cache_read_input_tokens ?? 0; + usage.output = message.usage.output_tokens; + return usage; +} + /** * Anthropic provider implementation */ @@ -50,41 +118,41 @@ export class AnthropicProvider implements LLMProvider { * Generate text using Anthropic API */ async generateText(options: GenerateOptions): Promise { - const { - messages, - functions, - temperature = 0.7, - maxTokens, - stopSequences, - topP, - } = options; + const { messages, functions, temperature = 0.7, maxTokens, topP } = options; // Extract system message const systemMessage = messages.find((msg) => msg.role === 'system'); const nonSystemMessages = messages.filter((msg) => msg.role !== 'system'); const formattedMessages = this.formatMessages(nonSystemMessages); + const tools = addCacheControlToTools( + (functions ?? []).map((fn) => ({ + name: fn.name, + description: fn.description, + input_schema: fn.parameters as Anthropic.Tool.InputSchema, + })), + ); + try { const requestOptions: Anthropic.MessageCreateParams = { model: this.model, - messages: formattedMessages, + messages: addCacheControlToMessages(formattedMessages), temperature, max_tokens: maxTokens || 1024, - ...(stopSequences && { stop_sequences: stopSequences }), - ...(topP && { top_p: topP }), - ...(systemMessage && { system: systemMessage.content }), + system: systemMessage?.content + ? [ + { + type: 'text', + text: systemMessage?.content, + cache_control: { type: 'ephemeral' }, + }, + ] + : undefined, + top_p: topP, + tools, + stream: false, }; - // Add tools if provided - if (functions && functions.length > 0) { - const tools = functions.map((fn) => ({ - name: fn.name, - description: fn.description, - input_schema: fn.parameters, - })); - (requestOptions as any).tools = tools; - } - const response = await this.client.messages.create(requestOptions); // Extract content and tool calls @@ -92,15 +160,13 @@ export class AnthropicProvider implements LLMProvider { response.content.find((c) => c.type === 'text')?.text || ''; const toolCalls = response.content .filter((c) => { - const contentType = (c as any).type; + const contentType = c.type; return contentType === 'tool_use'; }) .map((c) => { - const toolUse = c as any; + const toolUse = c as Anthropic.Messages.ToolUseBlock; return { - id: - toolUse.id || - `tool-${Math.random().toString(36).substring(2, 11)}`, + id: toolUse.id, name: toolUse.name, content: JSON.stringify(toolUse.input), }; @@ -109,6 +175,7 @@ export class AnthropicProvider implements LLMProvider { return { text: content, toolCalls: toolCalls, + tokenUsage: tokenUsageFromMessage(response), }; } catch (error) { throw new Error( @@ -117,20 +184,12 @@ export class AnthropicProvider implements LLMProvider { } } - /** - * Count tokens in a text using Anthropic's tokenizer - * Note: This is a simplified implementation - */ - async countTokens(text: string): Promise { - // In a real implementation, you would use Anthropic's tokenizer - // This is a simplified approximation - return Math.ceil(text.length / 3.5); - } - /** * Format messages for Anthropic API */ - private formatMessages(messages: Message[]): any[] { + private formatMessages( + messages: Message[], + ): Anthropic.Messages.MessageParam[] { // Format messages for Anthropic API return messages.map((msg) => { if (msg.role === 'user') { diff --git a/packages/agent/src/core/llm/types.ts b/packages/agent/src/core/llm/types.ts index 5cea886..e278d86 100644 --- a/packages/agent/src/core/llm/types.ts +++ b/packages/agent/src/core/llm/types.ts @@ -2,6 +2,9 @@ * Core message types for LLM interactions */ +import { JsonSchema7Type } from 'zod-to-json-schema'; + +import { TokenUsage } from '../tokens'; import { ToolCall } from '../types'; /** @@ -67,7 +70,7 @@ export type Message = export interface FunctionDefinition { name: string; description: string; - parameters: Record; // JSON Schema object + parameters: JsonSchema7Type; // JSON Schema object } /** @@ -76,6 +79,7 @@ export interface FunctionDefinition { export interface LLMResponse { text: string; toolCalls: ToolCall[]; + tokenUsage: TokenUsage; } /** diff --git a/packages/agent/src/core/toolAgent/toolAgentCore.ts b/packages/agent/src/core/toolAgent/toolAgentCore.ts index 37bde8f..0ac4be7 100644 --- a/packages/agent/src/core/toolAgent/toolAgentCore.ts +++ b/packages/agent/src/core/toolAgent/toolAgentCore.ts @@ -76,7 +76,12 @@ export const toolAgent = async ( maxTokens: config.maxTokens, }; - const { text, toolCalls } = await generateText(provider, generateOptions); + const { text, toolCalls, tokenUsage } = await generateText( + provider, + generateOptions, + ); + + tokenTracker.tokenUsage.add(tokenUsage); if (!text.length && toolCalls.length === 0) { // Only consider it empty if there's no text AND no tool calls diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7fe9a0e..90bccf1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,8 +79,8 @@ importers: packages/agent: dependencies: '@anthropic-ai/sdk': - specifier: ^0.16.0 - version: 0.16.1 + specifier: ^0.37 + version: 0.37.0 '@mozilla/readability': specifier: ^0.5.0 version: 0.5.0 @@ -191,8 +191,8 @@ importers: packages: - '@anthropic-ai/sdk@0.16.1': - resolution: {integrity: sha512-vHgvfWEyFy5ktqam56Nrhv8MVa7EJthsRYNi+1OrFFfyrj9tR2/aji1QbVbQjYU/pPhPFaYrdCEC/MLPFrmKwA==} + '@anthropic-ai/sdk@0.37.0': + resolution: {integrity: sha512-tHjX2YbkUBwEgg0JZU3EFSSAQPoK4qQR/NFYa8Vtzd5UAyXzZksCw2In69Rml4R/TyHPBfRYaLK35XiOe33pjw==} '@asamuzakjp/css-color@2.8.3': resolution: {integrity: sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==} @@ -1589,9 +1589,6 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - base-64@0.1.0: - resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} - before-after-hook@3.0.2: resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} @@ -1658,9 +1655,6 @@ packages: chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} - charenc@0.0.2: - resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} - check-error@2.1.1: resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} engines: {node: '>= 16'} @@ -1807,9 +1801,6 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} - crypt@0.0.2: - resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} - crypto-random-string@4.0.0: resolution: {integrity: sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==} engines: {node: '>=12'} @@ -1889,9 +1880,6 @@ packages: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} - digest-fetch@1.3.0: - resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} - dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -2594,9 +2582,6 @@ packages: resolution: {integrity: sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==} engines: {node: '>= 0.4'} - is-buffer@1.1.6: - resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} - is-bun-module@1.3.0: resolution: {integrity: sha512-DgXeu5UWI0IsMQundYb5UAOzm6G2eVnarJ0byP6Tm55iZNKceD59LNPA2L4VvsScTtHcw0yEkVwSf7PC+QoLSA==} @@ -2938,9 +2923,6 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} - md5@2.3.0: - resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} - meow@12.1.1: resolution: {integrity: sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==} engines: {node: '>=16.10'} @@ -4172,10 +4154,6 @@ packages: resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} engines: {node: '>=18'} - web-streams-polyfill@3.3.3: - resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} - engines: {node: '>= 8'} - web-streams-polyfill@4.0.0-beta.3: resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} engines: {node: '>= 14'} @@ -4331,17 +4309,15 @@ packages: snapshots: - '@anthropic-ai/sdk@0.16.1': + '@anthropic-ai/sdk@0.37.0': dependencies: '@types/node': 18.19.76 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.6.0 - digest-fetch: 1.3.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 node-fetch: 2.7.0 - web-streams-polyfill: 3.3.3 transitivePeerDependencies: - encoding @@ -5900,8 +5876,6 @@ snapshots: balanced-match@1.0.2: {} - base-64@0.1.0: {} - before-after-hook@3.0.2: {} better-path-resolve@1.0.0: @@ -5971,8 +5945,6 @@ snapshots: chardet@0.7.0: {} - charenc@0.0.2: {} - check-error@2.1.1: {} ci-info@3.9.0: {} @@ -6118,8 +6090,6 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 - crypt@0.0.2: {} - crypto-random-string@4.0.0: dependencies: type-fest: 1.4.0 @@ -6188,11 +6158,6 @@ snapshots: detect-indent@6.1.0: {} - digest-fetch@1.3.0: - dependencies: - base-64: 0.1.0 - md5: 2.3.0 - dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -7046,8 +7011,6 @@ snapshots: call-bound: 1.0.3 has-tostringtag: 1.0.2 - is-buffer@1.1.6: {} - is-bun-module@1.3.0: dependencies: semver: 7.7.1 @@ -7389,12 +7352,6 @@ snapshots: math-intrinsics@1.1.0: {} - md5@2.3.0: - dependencies: - charenc: 0.0.2 - crypt: 0.0.2 - is-buffer: 1.1.6 - meow@12.1.1: {} meow@13.2.0: {} @@ -8563,8 +8520,6 @@ snapshots: dependencies: xml-name-validator: 5.0.0 - web-streams-polyfill@3.3.3: {} - web-streams-polyfill@4.0.0-beta.3: {} webidl-conversions@3.0.1: {}