fix: special tokens should not break task processing (#7540)

Author: pwilkin

src/utils/__tests__/tiktoken.spec.ts

@@ -25,6 +25,15 @@ describe("tiktoken", () => {
 		expect(result).toBe(0)
 	})
 
+	it("should not throw on text content with special tokens", async () => {
+		const content: Anthropic.Messages.ContentBlockParam[] = [
+			{ type: "text", text: "something<|endoftext|>something" },
+		]
+
+		const result = await tiktoken(content)
+		expect(result).toBeGreaterThan(0)
+	})
+
 	it("should handle missing text content", async () => {
 		// Using 'as any' to bypass TypeScript's type checking for this test case
 		// since we're specifically testing how the function handles undefined text

src/utils/tiktoken.ts

@@ -24,7 +24,7 @@ export async function tiktoken(content: Anthropic.Messages.ContentBlockParam[]):
 			const text = block.text || ""
 
 			if (text.length > 0) {
-				const tokens = encoder.encode(text)
+				const tokens = encoder.encode(text, undefined, [])
 				totalTokens += tokens.length
 			}
 		} else if (block.type === "image") {