From 4173863e2faa033b7311679b89c4e83bc1b95677 Mon Sep 17 00:00:00 2001 From: Dailyfocus <135451694+Dayleyfocus@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:58:40 -0700 Subject: [PATCH 1/2] Add Anthropic Compatible provider support Introduces a new 'anthropic-compatible' provider with schema, API handler, and UI components for custom Anthropic-compatible endpoints. Updates provider selection, model handling, and settings to support custom base URLs, auth tokens, and model configuration. --- packages/types/src/provider-settings.ts | 18 +- packages/types/src/providers/anthropic.ts | 1 + src/api/index.ts | 3 + src/api/providers/anthropic-compatible.ts | 322 ++++++++++++++++++ src/api/providers/index.ts | 1 + .../src/components/settings/ApiOptions.tsx | 10 +- .../src/components/settings/constants.ts | 1 + .../providers/AnthropicCompatible.tsx | 256 ++++++++++++++ .../components/settings/providers/index.ts | 1 + .../components/ui/hooks/useSelectedModel.ts | 6 + webview-ui/src/i18n/locales/en/settings.json | 1 + 11 files changed, 618 insertions(+), 2 deletions(-) create mode 100644 src/api/providers/anthropic-compatible.ts create mode 100644 webview-ui/src/components/settings/providers/AnthropicCompatible.tsx diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index 090dfe6693..3eb6b5d67e 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -33,6 +33,7 @@ import { export const providerNames = [ "anthropic", + "anthropic-compatible", "claude-code", "glama", "openrouter", @@ -126,6 +127,14 @@ const anthropicSchema = apiModelIdProviderModelSchema.extend({ anthropicBeta1MContext: z.boolean().optional(), // Enable 'context-1m-2025-08-07' beta for 1M context window }) +const anthropicCompatibleSchema = apiModelIdProviderModelSchema.extend({ + apiKey: z.string().optional(), + anthropicBaseUrl: z.string().optional(), + anthropicUseAuthToken: z.boolean().optional(), + anthropicBeta1MContext: z.boolean().optional(), // Enable 'context-1m-2025-08-07' beta for 1M context window + anthropicCustomModelInfo: modelInfoSchema.nullish(), +}) + const claudeCodeSchema = apiModelIdProviderModelSchema.extend({ claudeCodePath: z.string().optional(), claudeCodeMaxOutputTokens: z.number().int().min(1).max(200000).optional(), @@ -335,6 +344,7 @@ const defaultSchema = z.object({ export const providerSettingsSchemaDiscriminated = z.discriminatedUnion("apiProvider", [ anthropicSchema.merge(z.object({ apiProvider: z.literal("anthropic") })), + anthropicCompatibleSchema.merge(z.object({ apiProvider: z.literal("anthropic-compatible") })), claudeCodeSchema.merge(z.object({ apiProvider: z.literal("claude-code") })), glamaSchema.merge(z.object({ apiProvider: z.literal("glama") })), openRouterSchema.merge(z.object({ apiProvider: z.literal("openrouter") })), @@ -375,6 +385,7 @@ export const providerSettingsSchemaDiscriminated = z.discriminatedUnion("apiProv export const providerSettingsSchema = z.object({ apiProvider: providerNamesSchema.optional(), ...anthropicSchema.shape, + ...anthropicCompatibleSchema.shape, ...claudeCodeSchema.shape, ...glamaSchema.shape, ...openRouterSchema.shape, @@ -446,7 +457,7 @@ export const getModelId = (settings: ProviderSettings): string | undefined => { } // Providers that use Anthropic-style API protocol. -export const ANTHROPIC_STYLE_PROVIDERS: ProviderName[] = ["anthropic", "claude-code", "bedrock"] +export const ANTHROPIC_STYLE_PROVIDERS: ProviderName[] = ["anthropic", "anthropic-compatible", "claude-code", "bedrock"] export const getApiProtocol = (provider: ProviderName | undefined, modelId?: string): "anthropic" | "openai" => { if (provider && ANTHROPIC_STYLE_PROVIDERS.includes(provider)) { @@ -474,6 +485,11 @@ export const MODELS_BY_PROVIDER: Record< label: "Anthropic", models: Object.keys(anthropicModels), }, + "anthropic-compatible": { + id: "anthropic-compatible", + label: "Anthropic Compatible", + models: Object.keys(anthropicModels), + }, bedrock: { id: "bedrock", label: "Amazon Bedrock", diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 2cb38537a4..97636bec6d 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -4,6 +4,7 @@ import type { ModelInfo } from "../model.js" export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-20250514" +export const anthropicCompatibleDefaultModelId: AnthropicModelId = "claude-sonnet-4-20250514" export const anthropicModels = { "claude-sonnet-4-20250514": { diff --git a/src/api/index.ts b/src/api/index.ts index b50afbb023..13cbd8c35a 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -7,6 +7,7 @@ import { ApiStream } from "./transform/stream" import { GlamaHandler, AnthropicHandler, + AnthropicCompatibleHandler, AwsBedrockHandler, CerebrasHandler, OpenRouterHandler, @@ -92,6 +93,8 @@ export function buildApiHandler(configuration: ProviderSettings): ApiHandler { switch (apiProvider) { case "anthropic": return new AnthropicHandler(options) + case "anthropic-compatible": + return new AnthropicCompatibleHandler(options) case "claude-code": return new ClaudeCodeHandler(options) case "glama": diff --git a/src/api/providers/anthropic-compatible.ts b/src/api/providers/anthropic-compatible.ts new file mode 100644 index 0000000000..50a682184c --- /dev/null +++ b/src/api/providers/anthropic-compatible.ts @@ -0,0 +1,322 @@ +import { Anthropic } from "@anthropic-ai/sdk" +import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming" +import { CacheControlEphemeral } from "@anthropic-ai/sdk/resources" + +import { + type ModelInfo, + type AnthropicModelId, + anthropicDefaultModelId, + anthropicModels, + ANTHROPIC_DEFAULT_MAX_TOKENS, +} from "@roo-code/types" + +import type { ApiHandlerOptions } from "../../shared/api" + +import { ApiStream } from "../transform/stream" +import { getModelParams } from "../transform/model-params" + +import { BaseProvider } from "./base-provider" +import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" +import { calculateApiCostAnthropic } from "../../shared/cost" + +export class AnthropicCompatibleHandler extends BaseProvider implements SingleCompletionHandler { + private options: ApiHandlerOptions + private client: Anthropic + + constructor(options: ApiHandlerOptions) { + super() + this.options = options + + const apiKeyFieldName = + this.options.anthropicBaseUrl && this.options.anthropicUseAuthToken ? "authToken" : "apiKey" + + this.client = new Anthropic({ + baseURL: this.options.anthropicBaseUrl || undefined, + [apiKeyFieldName]: this.options.apiKey, + }) + } + + async *createMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): ApiStream { + let stream: AnthropicStream + const cacheControl: CacheControlEphemeral = { type: "ephemeral" } + let { id: modelId, betas = [], maxTokens, temperature, reasoning: thinking } = this.getModel() + + // Add 1M context beta flag if enabled for Claude Sonnet 4 + if (modelId === "claude-sonnet-4-20250514" && this.options.anthropicBeta1MContext) { + betas.push("context-1m-2025-08-07") + } + + switch (modelId) { + case "claude-sonnet-4-20250514": + case "claude-opus-4-1-20250805": + case "claude-opus-4-20250514": + case "claude-3-7-sonnet-20250219": + case "claude-3-5-sonnet-20241022": + case "claude-3-5-haiku-20241022": + case "claude-3-opus-20240229": + case "claude-3-haiku-20240307": { + /** + * The latest message will be the new user message, one before + * will be the assistant message from a previous request, and + * the user message before that will be a previously cached user + * message. So we need to mark the latest user message as + * ephemeral to cache it for the next request, and mark the + * second to last user message as ephemeral to let the server + * know the last message to retrieve from the cache for the + * current request. + */ + const userMsgIndices = messages.reduce( + (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), + [] as number[], + ) + + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + + stream = await this.client.messages.create( + { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + thinking, + // Setting cache breakpoint for system prompt so new tasks can reuse it. + system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + messages: messages.map((message, index) => { + if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { + return { + ...message, + content: + typeof message.content === "string" + ? [{ type: "text", text: message.content, cache_control: cacheControl }] + : message.content.map((content, contentIndex) => + contentIndex === message.content.length - 1 + ? { ...content, cache_control: cacheControl } + : content, + ), + } + } + return message + }), + stream: true, + }, + (() => { + // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 + // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers + // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 + + // Then check for models that support prompt caching + switch (modelId) { + case "claude-sonnet-4-20250514": + case "claude-opus-4-1-20250805": + case "claude-opus-4-20250514": + case "claude-3-7-sonnet-20250219": + case "claude-3-5-sonnet-20241022": + case "claude-3-5-haiku-20241022": + case "claude-3-opus-20240229": + case "claude-3-haiku-20240307": + betas.push("prompt-caching-2024-07-31") + return { headers: { "anthropic-beta": betas.join(",") } } + default: + return undefined + } + })(), + ) + break + } + default: { + stream = (await this.client.messages.create({ + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + system: [{ text: systemPrompt, type: "text" }], + messages, + stream: true, + })) as any + break + } + } + + let inputTokens = 0 + let outputTokens = 0 + let cacheWriteTokens = 0 + let cacheReadTokens = 0 + + for await (const chunk of stream) { + switch (chunk.type) { + case "message_start": { + // Tells us cache reads/writes/input/output. + const { + input_tokens = 0, + output_tokens = 0, + cache_creation_input_tokens, + cache_read_input_tokens, + } = chunk.message.usage + + yield { + type: "usage", + inputTokens: input_tokens, + outputTokens: output_tokens, + cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheReadTokens: cache_read_input_tokens || undefined, + } + + inputTokens += input_tokens + outputTokens += output_tokens + cacheWriteTokens += cache_creation_input_tokens || 0 + cacheReadTokens += cache_read_input_tokens || 0 + + break + } + case "message_delta": + // Tells us stop_reason, stop_sequence, and output tokens + // along the way and at the end of the message. + yield { + type: "usage", + inputTokens: 0, + outputTokens: chunk.usage.output_tokens || 0, + } + + break + case "message_stop": + // No usage data, just an indicator that the message is done. + break + case "content_block_start": + switch (chunk.content_block.type) { + case "thinking": + // We may receive multiple text blocks, in which + // case just insert a line break between them. + if (chunk.index > 0) { + yield { type: "reasoning", text: "\n" } + } + + yield { type: "reasoning", text: chunk.content_block.thinking } + break + case "text": + // We may receive multiple text blocks, in which + // case just insert a line break between them. + if (chunk.index > 0) { + yield { type: "text", text: "\n" } + } + + yield { type: "text", text: chunk.content_block.text } + break + } + break + case "content_block_delta": + switch (chunk.delta.type) { + case "thinking_delta": + yield { type: "reasoning", text: chunk.delta.thinking } + break + case "text_delta": + yield { type: "text", text: chunk.delta.text } + break + } + + break + case "content_block_stop": + break + } + } + + if (inputTokens > 0 || outputTokens > 0 || cacheWriteTokens > 0 || cacheReadTokens > 0) { + yield { + type: "usage", + inputTokens: 0, + outputTokens: 0, + totalCost: calculateApiCostAnthropic( + this.getModel().info, + inputTokens, + outputTokens, + cacheWriteTokens, + cacheReadTokens, + ), + } + } + } + + getModel() { + const modelId = this.options.apiModelId + let id = modelId && modelId in anthropicModels ? (modelId as AnthropicModelId) : anthropicDefaultModelId + let info: ModelInfo = anthropicModels[id] + + // If 1M context beta is enabled for Claude Sonnet 4, update the model info + if (id === "claude-sonnet-4-20250514" && this.options.anthropicBeta1MContext) { + // Use the tier pricing for 1M context + const tier = info.tiers?.[0] + if (tier) { + info = { + ...info, + contextWindow: tier.contextWindow, + inputPrice: tier.inputPrice, + outputPrice: tier.outputPrice, + cacheWritesPrice: tier.cacheWritesPrice, + cacheReadsPrice: tier.cacheReadsPrice, + } + } + } + + const params = getModelParams({ + format: "anthropic", + modelId: id, + model: info, + settings: this.options, + }) + + // The `:thinking` suffix indicates that the model is a "Hybrid" + // reasoning model and that reasoning is required to be enabled. + // The actual model ID honored by Anthropic's API does not have this + // suffix. + return { + id: id === "claude-3-7-sonnet-20250219:thinking" ? "claude-3-7-sonnet-20250219" : id, + info, + betas: id === "claude-3-7-sonnet-20250219:thinking" ? ["output-128k-2025-02-19"] : undefined, + ...params, + } + } + + async completePrompt(prompt: string) { + let { id: model, temperature } = this.getModel() + + const message = await this.client.messages.create({ + model, + max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, + thinking: undefined, + temperature, + messages: [{ role: "user", content: prompt }], + stream: false, + }) + + const content = message.content.find(({ type }) => type === "text") + return content?.type === "text" ? content.text : "" + } + + /** + * Counts tokens for the given content using Anthropic's API + * + * @param content The content blocks to count tokens for + * @returns A promise resolving to the token count + */ + override async countTokens(content: Array): Promise { + try { + // Use the current model + const { id: model } = this.getModel() + + const response = await this.client.messages.countTokens({ + model, + messages: [{ role: "user", content: content }], + }) + + return response.input_tokens + } catch (error) { + // Log error but fallback to tiktoken estimation + console.warn("Anthropic token counting failed, using fallback", error) + + // Use the base provider's implementation as fallback + return super.countTokens(content) + } + } +} diff --git a/src/api/providers/index.ts b/src/api/providers/index.ts index c3786c5f56..7f2b9b7e66 100644 --- a/src/api/providers/index.ts +++ b/src/api/providers/index.ts @@ -1,5 +1,6 @@ export { AnthropicVertexHandler } from "./anthropic-vertex" export { AnthropicHandler } from "./anthropic" +export { AnthropicCompatibleHandler } from "./anthropic-compatible" export { AwsBedrockHandler } from "./bedrock" export { CerebrasHandler } from "./cerebras" export { ChutesHandler } from "./chutes" diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index 80ecd75ae4..d0cce7b96d 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -63,6 +63,7 @@ import { import { Anthropic, + AnthropicCompatible, Bedrock, Cerebras, Chutes, @@ -491,6 +492,13 @@ const ApiOptions = ({ )} + {selectedProvider === "anthropic-compatible" && ( + + )} + {selectedProvider === "claude-code" && ( )} @@ -658,7 +666,7 @@ const ApiOptions = ({ )} - {selectedProviderModels.length > 0 && ( + {selectedProviderModels.length > 0 && selectedProvider !== "anthropic-compatible" && ( <>
diff --git a/webview-ui/src/components/settings/constants.ts b/webview-ui/src/components/settings/constants.ts index 9aa02bbf53..24e71739a0 100644 --- a/webview-ui/src/components/settings/constants.ts +++ b/webview-ui/src/components/settings/constants.ts @@ -49,6 +49,7 @@ export const MODELS_BY_PROVIDER: Partial void +} + +export const AnthropicCompatible = ({ apiConfiguration, setApiConfigurationField }: AnthropicCompatibleProps) => { + const { t } = useAppTranslation() + + const [anthropicBaseUrlSelected, setAnthropicBaseUrlSelected] = useState(!!apiConfiguration?.anthropicBaseUrl) + + const handleInputChange = useCallback( + ( + field: K, + transform: (event: E) => ProviderSettings[K] = inputEventTransform, + ) => + (event: E | Event) => { + setApiConfigurationField(field, transform(event as E)) + }, + [setApiConfigurationField], + ) + + return ( + <> + + + +
+ {t("settings:providers.apiKeyStorageNotice")} +
+ {!apiConfiguration?.apiKey && ( + + {t("settings:providers.getAnthropicApiKey")} + + )} + + + +
+ Enter the model name (e.g., claude-3-5-sonnet-20241022) +
+
+ { + setAnthropicBaseUrlSelected(checked) + + if (!checked) { + setApiConfigurationField("anthropicBaseUrl", "") + setApiConfigurationField("anthropicUseAuthToken", false) + } + }}> + {t("settings:providers.useCustomBaseUrl")} + + {anthropicBaseUrlSelected && ( + <> + + + {t("settings:providers.anthropicUseAuthToken")} + + + )} +
+ +
+
+ Custom Model Configuration +
+ +
+ { + const value = apiConfiguration?.anthropicCustomModelInfo?.contextWindow + + if (!value) { + return "var(--vscode-input-border)" + } + + return value > 0 ? "var(--vscode-charts-green)" : "var(--vscode-errorForeground)" + })(), + }} + onInput={handleInputChange("anthropicCustomModelInfo", (e) => { + const value = (e.target as HTMLInputElement).value + const parsed = parseInt(value) + + return { + ...(apiConfiguration?.anthropicCustomModelInfo || openAiModelInfoSaneDefaults), + contextWindow: isNaN(parsed) ? openAiModelInfoSaneDefaults.contextWindow : parsed, + } + })} + placeholder="200000" + className="w-full"> + + +
+ Enter the context window size (max tokens the model can process) +
+
+ +
+
+ { + return { + ...(apiConfiguration?.anthropicCustomModelInfo || openAiModelInfoSaneDefaults), + supportsImages: checked, + } + })}> + Supports Images + + + + +
+
+ Enable image support for this model +
+
+ +
+
+ { + return { + ...(apiConfiguration?.anthropicCustomModelInfo || openAiModelInfoSaneDefaults), + supportsComputerUse: checked, + } + })}> + Supports Computer Use + + + + +
+
+ Enable computer use capabilities for this model +
+
+ +
+
+ { + return { + ...(apiConfiguration?.anthropicCustomModelInfo || openAiModelInfoSaneDefaults), + supportsPromptCache: checked, + } + })}> + Supports Prompt Caching + + + + +
+
+ Enable prompt caching for this model +
+
+ +
+ { + const value = apiConfiguration?.anthropicCustomModelInfo?.maxTokens + + if (!value) { + return "var(--vscode-input-border)" + } + + return value > 0 ? "var(--vscode-charts-green)" : "var(--vscode-errorForeground)" + })(), + }} + onInput={handleInputChange("anthropicCustomModelInfo", (e) => { + const value = parseInt((e.target as HTMLInputElement).value) + + return { + ...(apiConfiguration?.anthropicCustomModelInfo || openAiModelInfoSaneDefaults), + maxTokens: isNaN(value) ? undefined : value, + } + })} + placeholder="8192" + className="w-full"> + + +
+ Maximum number of tokens the model can generate +
+
+ + +
+ + ) +} diff --git a/webview-ui/src/components/settings/providers/index.ts b/webview-ui/src/components/settings/providers/index.ts index eedbba0c29..1b95be2b3e 100644 --- a/webview-ui/src/components/settings/providers/index.ts +++ b/webview-ui/src/components/settings/providers/index.ts @@ -1,4 +1,5 @@ export { Anthropic } from "./Anthropic" +export { AnthropicCompatible } from "./AnthropicCompatible" export { Bedrock } from "./Bedrock" export { Cerebras } from "./Cerebras" export { Chutes } from "./Chutes" diff --git a/webview-ui/src/components/ui/hooks/useSelectedModel.ts b/webview-ui/src/components/ui/hooks/useSelectedModel.ts index e9470e0902..f57b75cf4d 100644 --- a/webview-ui/src/components/ui/hooks/useSelectedModel.ts +++ b/webview-ui/src/components/ui/hooks/useSelectedModel.ts @@ -335,6 +335,12 @@ function getSelectedModel({ const info = routerModels["vercel-ai-gateway"]?.[id] return { id, info } } + case "anthropic-compatible": { + const id = apiConfiguration.apiModelId ?? anthropicDefaultModelId + const info = + apiConfiguration?.anthropicCustomModelInfo ?? anthropicModels[id as keyof typeof anthropicModels] + return { id, info } + } // case "anthropic": // case "human-relay": // case "fake-ai": diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 8479be7793..e4c60d46b2 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -263,6 +263,7 @@ "openRouterTransformsText": "Compress prompts and message chains to the context size (OpenRouter Transforms)", "anthropicApiKey": "Anthropic API Key", "getAnthropicApiKey": "Get Anthropic API Key", + "anthropicCompatibleApiKey": "Anthropic Compatible API Key", "anthropicUseAuthToken": "Pass Anthropic API Key as Authorization header instead of X-Api-Key", "anthropic1MContextBetaLabel": "Enable 1M context window (Beta)", "anthropic1MContextBetaDescription": "Extends context window to 1 million tokens for Claude Sonnet 4", From b6df473323c9d8eb4840cf8803179a1a6d8ce4a4 Mon Sep 17 00:00:00 2001 From: Dailyfocus <135451694+Dayleyfocus@users.noreply.github.com> Date: Wed, 3 Sep 2025 21:33:47 -0700 Subject: [PATCH 2/2] Update webview-ui/src/components/settings/providers/AnthropicCompatible.tsx Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- .../src/components/settings/providers/AnthropicCompatible.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webview-ui/src/components/settings/providers/AnthropicCompatible.tsx b/webview-ui/src/components/settings/providers/AnthropicCompatible.tsx index 787fde0d38..012fd7f61b 100644 --- a/webview-ui/src/components/settings/providers/AnthropicCompatible.tsx +++ b/webview-ui/src/components/settings/providers/AnthropicCompatible.tsx @@ -55,7 +55,7 @@ export const AnthropicCompatible = ({ apiConfiguration, setApiConfigurationField onInput={handleInputChange("apiModelId")} placeholder="claude-3-5-sonnet-20241022" className="w-full"> - +
Enter the model name (e.g., claude-3-5-sonnet-20241022)