Litellm models can now be refreshed

Author: slytechnical

src/api/providers/fetchers/litellm.ts

@@ -7,6 +7,7 @@ import { COMPUTER_USE_MODELS, ModelRecord } from "../../../shared/api"
  * @param apiKey The API key for the LiteLLM server
  * @param baseUrl The base URL of the LiteLLM server
  * @returns A promise that resolves to a record of model IDs to model info
+ * @throws Will throw an error if the request fails or the response is not as expected.
  */
 export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise<ModelRecord> {
 	try {
@@ -17,8 +18,8 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
 		if (apiKey) {
 			headers["Authorization"] = `Bearer ${apiKey}`
 		}
-
-		const response = await axios.get(`${baseUrl}/v1/model/info`, { headers })
+		// Added timeout to prevent indefinite hanging
+		const response = await axios.get(`${baseUrl}/v1/model/info`, { headers, timeout: 5000 })
 		const models: ModelRecord = {}
 
 		const computerModels = Array.from(COMPUTER_USE_MODELS)
@@ -48,11 +49,25 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
 					description: `${modelName} via LiteLLM proxy`,
 				}
 			}
+		} else {
+			// If response.data.data is not in the expected format, consider it an error.
+			console.error("Error fetching LiteLLM models: Unexpected response format", response.data)
+			throw new Error("Failed to fetch LiteLLM models: Unexpected response format.")
 		}
 
 		return models
-	} catch (error) {
-		console.error("Error fetching LiteLLM models:", error)
-		return {}
+	} catch (error: any) {
+		console.error("Error fetching LiteLLM models:", error.message ? error.message : error)
+		if (axios.isAxiosError(error) && error.response) {
+			throw new Error(
+				`Failed to fetch LiteLLM models: ${error.response.status} ${error.response.statusText}. Check base URL and API key.`,
+			)
+		} else if (axios.isAxiosError(error) && error.request) {
+			throw new Error(
+				"Failed to fetch LiteLLM models: No response from server. Check LiteLLM server status and base URL.",
+			)
+		} else {
+			throw new Error(`Failed to fetch LiteLLM models: ${error.message || "An unknown error occurred."}`)
+		}
 	}
 }

src/api/providers/fetchers/modelCache.ts

@@ -13,7 +13,7 @@ import { getRequestyModels } from "./requesty"
 import { getGlamaModels } from "./glama"
 import { getUnboundModels } from "./unbound"
 import { getLiteLLMModels } from "./litellm"
-
+import { GetModelsOptions } from "../../../shared/api"
 const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
 
 async function writeModels(router: RouterName, data: ModelRecord) {
@@ -41,64 +41,58 @@ async function readModels(router: RouterName): Promise<ModelRecord | undefined>
  * @param baseUrl - Optional base URL for the provider (currently used only for LiteLLM).
  * @returns The models from the cache or the fetched models.
  */
-export const getModels = async (
-	router: RouterName,
-	apiKey: string | undefined = undefined,
-	baseUrl: string | undefined = undefined,
-): Promise<ModelRecord> => {
-	let models = memoryCache.get<ModelRecord>(router)
-
+export const getModels = async (options: GetModelsOptions): Promise<ModelRecord> => {
+	const { provider } = options
+	let models = memoryCache.get<ModelRecord>(provider)
 	if (models) {
-		// console.log(`[getModels] NodeCache hit for ${router} -> ${Object.keys(models).length}`)
 		return models
 	}
 
-	switch (router) {
-		case "openrouter":
-			models = await getOpenRouterModels()
-			break
-		case "requesty":
-			// Requesty models endpoint requires an API key for per-user custom policies
-			models = await getRequestyModels(apiKey)
-			break
-		case "glama":
-			models = await getGlamaModels()
-			break
-		case "unbound":
-			// Unbound models endpoint requires an API key to fetch application specific models
-			models = await getUnboundModels(apiKey)
-			break
-		case "litellm":
-			if (apiKey && baseUrl) {
-				models = await getLiteLLMModels(apiKey, baseUrl)
-			} else {
-				models = {}
-			}
-			break
-	}
+	try {
+		switch (provider) {
+			case "openrouter":
+				models = await getOpenRouterModels()
+				break
+			case "requesty":
+				// Requesty models endpoint requires an API key for per-user custom policies
+				models = await getRequestyModels(options.apiKey)
+				break
+			case "glama":
+				models = await getGlamaModels()
+				break
+			case "unbound":
+				// Unbound models endpoint requires an API key to fetch application specific models
+				models = await getUnboundModels(options.apiKey)
+				break
+			case "litellm":
+				// Type safety ensures apiKey and baseUrl are always provided for litellm
+				models = await getLiteLLMModels(options.apiKey, options.baseUrl)
+				break
+			default:
+				// Ensures router is exhaustively checked if RouterName is a strict union
+				const exhaustiveCheck: never = provider
+				throw new Error(`Unknown provider: ${exhaustiveCheck}`)
+		}
 
-	if (Object.keys(models).length > 0) {
-		// console.log(`[getModels] API fetch for ${router} -> ${Object.keys(models).length}`)
-		memoryCache.set(router, models)
+		// Cache the fetched models (even if empty, to signify a successful fetch with no models)
+		memoryCache.set(provider, models)
+		await writeModels(provider, models).catch((err) =>
+			console.error(`[getModels] Error writing ${provider} models to file cache:`, err),
+		)
 
 		try {
-			await writeModels(router, models)
-			// console.log(`[getModels] wrote ${router} models to file cache`)
+			models = await readModels(provider)
+			// console.log(`[getModels] read ${router} models from file cache`)
 		} catch (error) {
-			console.error(`[getModels] error writing ${router} models to file cache`, error)
+			console.error(`[getModels] error reading ${provider} models from file cache`, error)
 		}
-
-		return models
-	}
-
-	try {
-		models = await readModels(router)
-		// console.log(`[getModels] read ${router} models from file cache`)
+		return models || {}
 	} catch (error) {
-		console.error(`[getModels] error reading ${router} models from file cache`, error)
-	}
+		// Log the error and re-throw it so the caller can handle it (e.g., show a UI message).
+		console.error(`[getModels] Failed to fetch models in modelCache for ${provider}:`, error)
 
-	return models ?? {}
+		throw error // Re-throw the original error to be handled by the caller.
+	}
 }
 
 /**

src/api/providers/openrouter.ts

@@ -171,7 +171,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 
 	public async fetchModel() {
 		const [models, endpoints] = await Promise.all([
-			getModels("openrouter"),
+			getModels({ provider: "openrouter" }),
 			getModelEndpoints({
 				router: "openrouter",
 				modelId: this.options.openRouterModelId,

src/api/providers/requesty.ts

@@ -45,7 +45,7 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan
 	}
 
 	public async fetchModel() {
-		this.models = await getModels("requesty")
+		this.models = await getModels({ provider: "requesty" })
 		return this.getModel()
 	}
 

src/api/providers/router-provider.ts

@@ -44,7 +44,7 @@ export abstract class RouterProvider extends BaseProvider {
 	}
 
 	public async fetchModel() {
-		this.models = await getModels(this.name, this.client.apiKey, this.client.baseURL)
+		this.models = await getModels({ provider: this.name, apiKey: this.client.apiKey, baseUrl: this.client.baseURL })
 		return this.getModel()
 	}
 

src/core/webview/webviewMessageHandler.ts

@@ -6,7 +6,7 @@ import * as vscode from "vscode"
 import { ClineProvider } from "./ClineProvider"
 import { Language, ProviderSettings, GlobalState, Package } from "../../schemas"
 import { changeLanguage, t } from "../../i18n"
-import { RouterName, toRouterName } from "../../shared/api"
+import { RouterName, toRouterName, ModelRecord } from "../../shared/api"
 import { supportPrompt } from "../../shared/support-prompt"
 import { checkoutDiffPayloadSchema, checkoutRestorePayloadSchema, WebviewMessage } from "../../shared/WebviewMessage"
 import { checkExistKey } from "../../shared/checkExistApiConfig"
@@ -32,6 +32,7 @@ import { TelemetrySetting } from "../../shared/TelemetrySetting"
 import { getWorkspacePath } from "../../utils/path"
 import { Mode, defaultModeSlug } from "../../shared/modes"
 import { getModels, flushModels } from "../../api/providers/fetchers/modelCache"
+import { GetModelsOptions } from "../../shared/api"
 import { generateSystemPrompt } from "./generateSystemPrompt"
 import { getCommand } from "../../utils/commands"
 
@@ -170,10 +171,6 @@ export const webviewMessageHandler = async (provider: ClineProvider, message: We
 		case "askResponse":
 			provider.getCurrentCline()?.handleWebviewAskResponse(message.askResponse!, message.text, message.images)
 			break
-		case "autoCondenseContextPercent":
-			await updateGlobalState("autoCondenseContextPercent", message.value)
-			await provider.postStateToWebview()
-			break
 		case "terminalOperation":
 			if (message.terminalOperation) {
 				provider.getCurrentCline()?.handleTerminalOperation(message.terminalOperation)
@@ -282,29 +279,81 @@ export const webviewMessageHandler = async (provider: ClineProvider, message: We
 			await provider.resetState()
 			break
 		case "flushRouterModels":
-			const routerName: RouterName = toRouterName(message.text)
-			await flushModels(routerName)
+			const routerNameFlush: RouterName = toRouterName(message.text)
+			await flushModels(routerNameFlush)
 			break
 		case "requestRouterModels":
 			const { apiConfiguration } = await provider.getState()
 
-			const [openRouterModels, requestyModels, glamaModels, unboundModels, litellmModels] = await Promise.all([
-				getModels("openrouter", apiConfiguration.openRouterApiKey),
-				getModels("requesty", apiConfiguration.requestyApiKey),
-				getModels("glama", apiConfiguration.glamaApiKey),
-				getModels("unbound", apiConfiguration.unboundApiKey),
-				getModels("litellm", apiConfiguration.litellmApiKey, apiConfiguration.litellmBaseUrl),
-			])
+			const routerModels: Partial<Record<RouterName, ModelRecord>> = {
+				openrouter: {},
+				requesty: {},
+				glama: {},
+				unbound: {},
+				litellm: {},
+			}
+
+			const safeGetModels = async (options: GetModelsOptions): Promise<ModelRecord> => {
+				try {
+					return await getModels(options)
+				} catch (error) {
+					console.error(
+						`Failed to fetch models in webviewMessageHandler requestRouterModels for ${options.provider}:`,
+						error,
+					)
+					throw error // Re-throw to be caught by Promise.allSettled
+				}
+			}
+
+			const modelFetchPromises: Array<{ key: RouterName; options: GetModelsOptions }> = [
+				{ key: "openrouter", options: { provider: "openrouter" } },
+				{ key: "requesty", options: { provider: "requesty", apiKey: apiConfiguration.requestyApiKey } },
+				{ key: "glama", options: { provider: "glama" } },
+				{ key: "unbound", options: { provider: "unbound", apiKey: apiConfiguration.unboundApiKey } },
+			]
+
+			const litellmApiKey = apiConfiguration.litellmApiKey || message?.values?.litellmApiKey
+			const litellmBaseUrl = apiConfiguration.litellmBaseUrl || message?.values?.litellmBaseUrl
+			if (litellmApiKey && litellmBaseUrl) {
+				modelFetchPromises.push({
+					key: "litellm",
+					options: { provider: "litellm", apiKey: litellmApiKey, baseUrl: litellmBaseUrl },
+				})
+			}
+
+			const results = await Promise.allSettled(
+				modelFetchPromises.map(async ({ key, options }) => {
+					const models = await safeGetModels(options)
+					return { key, models } // key is RouterName here
+				}),
+			)
+
+			const fetchedRouterModels: Partial<Record<RouterName, ModelRecord>> = { ...routerModels }
+
+			results.forEach((result, index) => {
+				const routerName = modelFetchPromises[index].key // Get RouterName using index
+
+				if (result.status === "fulfilled") {
+					fetchedRouterModels[routerName] = result.value.models
+				} else {
+					// Handle rejection: Post a specific error message for this provider
+					const errorMessage = result.reason instanceof Error ? result.reason.message : String(result.reason)
+					console.error(`Error fetching models for ${routerName}:`, result.reason)
+
+					fetchedRouterModels[routerName] = {} // Ensure it's an empty object in the main routerModels message
+
+					provider.postMessageToWebview({
+						type: "singleRouterModelFetchResponse",
+						success: false,
+						error: errorMessage,
+						values: { provider: routerName },
+					})
+				}
+			})
 
 			provider.postMessageToWebview({
 				type: "routerModels",
-				routerModels: {
-					openrouter: openRouterModels,
-					requesty: requestyModels,
-					glama: glamaModels,
-					unbound: unboundModels,
-					litellm: litellmModels,
-				},
+				routerModels: fetchedRouterModels as Record<RouterName, ModelRecord>,
 			})
 			break
 		case "requestOpenAiModels":

src/shared/ExtensionMessage.ts

@@ -70,6 +70,7 @@ export interface ExtensionMessage {
 		| "commandExecutionStatus"
 		| "vsCodeSetting"
 		| "condenseTaskContextResponse"
+		| "singleRouterModelFetchResponse"
 	text?: string
 	action?:
 		| "chatButtonClicked"

src/shared/api.ts

@@ -1166,7 +1166,7 @@ export const unboundDefaultModelInfo: ModelInfo = {
 
 // LiteLLM
 // https://docs.litellm.ai/
-export const litellmDefaultModelId = "anthropic/claude-3-7-sonnet-20250219"
+export const litellmDefaultModelId = "claude-3-7-sonnet-20250219"
 export const litellmDefaultModelInfo: ModelInfo = {
 	maxTokens: 8192,
 	contextWindow: 200_000,
@@ -1832,3 +1832,15 @@ export function toRouterName(value?: string): RouterName {
 export type ModelRecord = Record<string, ModelInfo>
 
 export type RouterModels = Record<RouterName, ModelRecord>
+
+/**
+ * Options for fetching models from different providers.
+ * This is a discriminated union type where the provider property determines
+ * which other properties are required.
+ */
+export type GetModelsOptions =
+	| { provider: "openrouter" }
+	| { provider: "glama" }
+	| { provider: "requesty"; apiKey?: string }
+	| { provider: "unbound"; apiKey?: string }
+	| { provider: "litellm"; apiKey: string; baseUrl: string }

webview-ui/src/components/settings/ApiOptions.tsx

@@ -391,11 +391,7 @@ const ApiOptions = ({
 			)}
 
 			{selectedProvider === "litellm" && (
-				<LiteLLM
-					apiConfiguration={apiConfiguration}
-					setApiConfigurationField={setApiConfigurationField}
-					routerModels={routerModels}
-				/>
+				<LiteLLM apiConfiguration={apiConfiguration} setApiConfigurationField={setApiConfigurationField} />
 			)}
 
 			{selectedProvider === "human-relay" && (