Skip to content

Added a hardcoded list of computer use models for litellm as a fallba… #4052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 27, 2025
Merged

Added a hardcoded list of computer use models for litellm as a fallba… #4052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 199 additions & 0 deletions src/api/providers/fetchers/__tests__/litellm.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,4 +248,203 @@ describe("getLiteLLMModels", () => {

expect(result).toEqual({})
})

it("uses fallback computer use detection when supports_computer_use is not available", async () => {
const mockResponse = {
data: {
data: [
{
model_name: "claude-3-5-sonnet-latest",
model_info: {
max_tokens: 4096,
max_input_tokens: 200000,
supports_vision: true,
supports_prompt_caching: false,
// Note: no supports_computer_use field
},
litellm_params: {
model: "anthropic/claude-3-5-sonnet-latest", // This should match the fallback list
},
},
{
model_name: "gpt-4-turbo",
model_info: {
max_tokens: 8192,
max_input_tokens: 128000,
supports_vision: false,
supports_prompt_caching: false,
// Note: no supports_computer_use field
},
litellm_params: {
model: "openai/gpt-4-turbo", // This should NOT match the fallback list
},
},
],
},
}

mockedAxios.get.mockResolvedValue(mockResponse)

const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")

expect(result["claude-3-5-sonnet-latest"]).toEqual({
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: true, // Should be true due to fallback
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
description: "claude-3-5-sonnet-latest via LiteLLM proxy",
})

expect(result["gpt-4-turbo"]).toEqual({
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: false, // Should be false as it's not in fallback list
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
description: "gpt-4-turbo via LiteLLM proxy",
})
})

it("prioritizes explicit supports_computer_use over fallback detection", async () => {
const mockResponse = {
data: {
data: [
{
model_name: "claude-3-5-sonnet-latest",
model_info: {
max_tokens: 4096,
max_input_tokens: 200000,
supports_vision: true,
supports_prompt_caching: false,
supports_computer_use: false, // Explicitly set to false
},
litellm_params: {
model: "anthropic/claude-3-5-sonnet-latest", // This matches fallback list but should be ignored
},
},
{
model_name: "custom-model",
model_info: {
max_tokens: 8192,
max_input_tokens: 128000,
supports_vision: false,
supports_prompt_caching: false,
supports_computer_use: true, // Explicitly set to true
},
litellm_params: {
model: "custom/custom-model", // This would NOT match fallback list
},
},
{
model_name: "another-custom-model",
model_info: {
max_tokens: 8192,
max_input_tokens: 128000,
supports_vision: false,
supports_prompt_caching: false,
supports_computer_use: false, // Explicitly set to false
},
litellm_params: {
model: "custom/another-custom-model", // This would NOT match fallback list
},
},
],
},
}

mockedAxios.get.mockResolvedValue(mockResponse)

const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")

expect(result["claude-3-5-sonnet-latest"]).toEqual({
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: false, // False because explicitly set to false (fallback ignored)
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
description: "claude-3-5-sonnet-latest via LiteLLM proxy",
})

expect(result["custom-model"]).toEqual({
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: true, // True because explicitly set to true
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
description: "custom-model via LiteLLM proxy",
})

expect(result["another-custom-model"]).toEqual({
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: false, // False because explicitly set to false
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
description: "another-custom-model via LiteLLM proxy",
})
})

it("handles fallback detection with various model name formats", async () => {
const mockResponse = {
data: {
data: [
{
model_name: "vertex-claude",
model_info: {
max_tokens: 4096,
max_input_tokens: 200000,
supports_vision: true,
supports_prompt_caching: false,
},
litellm_params: {
model: "vertex_ai/claude-3-5-sonnet", // Should match fallback list
},
},
{
model_name: "openrouter-claude",
model_info: {
max_tokens: 4096,
max_input_tokens: 200000,
supports_vision: true,
supports_prompt_caching: false,
},
litellm_params: {
model: "openrouter/anthropic/claude-3.5-sonnet", // Should match fallback list
},
},
{
model_name: "bedrock-claude",
model_info: {
max_tokens: 4096,
max_input_tokens: 200000,
supports_vision: true,
supports_prompt_caching: false,
},
litellm_params: {
model: "anthropic.claude-3-5-sonnet-20241022-v2:0", // Should match fallback list
},
},
],
},
}

mockedAxios.get.mockResolvedValue(mockResponse)

const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")

expect(result["vertex-claude"].supportsComputerUse).toBe(true)
expect(result["openrouter-claude"].supportsComputerUse).toBe(true)
expect(result["bedrock-claude"].supportsComputerUse).toBe(true)
})
})
17 changes: 15 additions & 2 deletions src/api/providers/fetchers/litellm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import axios from "axios"

import { ModelRecord } from "../../../shared/api"
import { LITELLM_COMPUTER_USE_MODELS, ModelRecord } from "../../../shared/api"

/**
* Fetches available models from a LiteLLM server
Expand All @@ -23,6 +23,8 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
const response = await axios.get(`${baseUrl}/v1/model/info`, { headers, timeout: 5000 })
const models: ModelRecord = {}

const computerModels = Array.from(LITELLM_COMPUTER_USE_MODELS)

// Process the model info from the response
if (response.data && response.data.data && Array.isArray(response.data.data)) {
for (const model of response.data.data) {
Expand All @@ -32,12 +34,23 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise

if (!modelName || !modelInfo || !litellmModelName) continue

// Use explicit supports_computer_use if available, otherwise fall back to hardcoded list
let supportsComputerUse: boolean
if (modelInfo.supports_computer_use !== undefined) {
supportsComputerUse = Boolean(modelInfo.supports_computer_use)
} else {
// Fallback for older LiteLLM versions that don't have supports_computer_use field
supportsComputerUse = computerModels.some((computer_model) =>
litellmModelName.endsWith(computer_model),
)
}

models[modelName] = {
maxTokens: modelInfo.max_tokens || 8192,
contextWindow: modelInfo.max_input_tokens || 200000,
supportsImages: Boolean(modelInfo.supports_vision),
// litellm_params.model may have a prefix like openrouter/
supportsComputerUse: Boolean(modelInfo.supports_computer_use),
supportsComputerUse,
supportsPromptCache: Boolean(modelInfo.supports_prompt_caching),
inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined,
outputPrice: modelInfo.output_cost_per_token
Expand Down
33 changes: 33 additions & 0 deletions src/shared/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1260,6 +1260,39 @@ export const litellmDefaultModelInfo: ModelInfo = {
cacheWritesPrice: 3.75,
cacheReadsPrice: 0.3,
}

export const LITELLM_COMPUTER_USE_MODELS = new Set([
"claude-3-5-sonnet-latest",
"claude-opus-4-20250514",
"claude-sonnet-4-20250514",
"claude-3-7-sonnet-latest",
"claude-3-7-sonnet-20250219",
"claude-3-5-sonnet-20241022",
"vertex_ai/claude-3-5-sonnet",
"vertex_ai/claude-3-5-sonnet-v2",
"vertex_ai/claude-3-5-sonnet-v2@20241022",
"vertex_ai/claude-3-7-sonnet@20250219",
"vertex_ai/claude-opus-4@20250514",
"vertex_ai/claude-sonnet-4@20250514",
"openrouter/anthropic/claude-3.5-sonnet",
"openrouter/anthropic/claude-3.5-sonnet:beta",
"openrouter/anthropic/claude-3.7-sonnet",
"openrouter/anthropic/claude-3.7-sonnet:beta",
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",
"anthropic.claude-3-7-sonnet-20250219-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
"us.anthropic.claude-opus-4-20250514-v1:0",
"us.anthropic.claude-sonnet-4-20250514-v1:0",
"eu.anthropic.claude-3-5-sonnet-20241022-v2:0",
"eu.anthropic.claude-3-7-sonnet-20250219-v1:0",
"eu.anthropic.claude-opus-4-20250514-v1:0",
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
"snowflake/claude-3-5-sonnet",
])

// xAI
// https://docs.x.ai/docs/api-reference
export type XAIModelId = keyof typeof xaiModels
Expand Down
Loading