Merge pull request #6722 from continuedev/nate/anthropic-stop-reason

fix: increase default max_tokens for Anthropic API to 32k

Author: RomneyDa

packages/openai-adapters/src/apis/Anthropic.ts

@@ -55,6 +55,14 @@ export class AnthropicApi implements BaseLlmApi {
     return cachingStrategy(cleanBody);
   }
 
+  private maxTokensForModel(model: string): number {
+    if (model.includes("haiku")) {
+      return 8192;
+    }
+
+    return 32_000;
+  }
+
   public _convertToCleanAnthropicBody(oaiBody: ChatCompletionCreateParams) {
     let stop = undefined;
     if (oaiBody.stop && Array.isArray(oaiBody.stop)) {
@@ -81,7 +89,7 @@ export class AnthropicApi implements BaseLlmApi {
         : systemMessage,
       top_p: oaiBody.top_p,
       temperature: oaiBody.temperature,
-      max_tokens: oaiBody.max_tokens ?? 4096, // max_tokens is required
+      max_tokens: oaiBody.max_tokens ?? this.maxTokensForModel(oaiBody.model), // max_tokens is required
       model: oaiBody.model,
       stop_sequences: stop,
       stream: oaiBody.stream,

packages/openai-adapters/src/test/anthropic-adapter.vitest.ts

@@ -1,4 +1,4 @@
-import { describe, test, vi, expect, afterEach } from "vitest";
+import { afterEach, describe, expect, test, vi } from "vitest";
 import { runAdapterTest } from "./adapter-test-utils.js";
 
 // Mock the fetch package
@@ -53,7 +53,7 @@ describe("Anthropic Adapter Tests", () => {
           messages: [{ role: "user", content: "hello" }],
           system: undefined,
           model: "claude-3-5-sonnet-20241022",
-          max_tokens: 4096,
+          max_tokens: 32000,
           stream: undefined,
         },
       },
@@ -107,7 +107,7 @@ describe("Anthropic Adapter Tests", () => {
           messages: [{ role: "user", content: "hello" }],
           system: undefined,
           model: "claude-3-5-sonnet-20241022",
-          max_tokens: 4096,
+          max_tokens: 32000,
           stream: true,
         },
       },
@@ -168,7 +168,7 @@ describe("Anthropic Adapter Tests", () => {
             },
           ],
           model: "claude-3-5-sonnet-20241022",
-          max_tokens: 4096,
+          max_tokens: 32000,
           stream: true,
         },
       },