Add caching layer for OpenRouter provider

Author: cte

package-lock.json

@@ -407,6 +407,7 @@
 		"puppeteer-chromium-resolver": "^23.0.0",
 		"puppeteer-core": "^23.4.0",
 		"reconnecting-eventsource": "^1.6.4",
+		"sanitize-filename": "^1.6.3",
 		"say": "^0.16.0",
 		"serialize-error": "^11.0.3",
 		"simple-git": "^3.27.0",

package.json

@@ -47,6 +47,7 @@ export const getModels = async (
 	baseUrl: string | undefined = undefined,
 ): Promise<ModelRecord> => {
 	let models = memoryCache.get<ModelRecord>(router)
+
 	if (models) {
 		// console.log(`[getModels] NodeCache hit for ${router} -> ${Object.keys(models).length}`)
 		return models
@@ -82,15 +83,19 @@ export const getModels = async (
 		try {
 			await writeModels(router, models)
 			// console.log(`[getModels] wrote ${router} models to file cache`)
-		} catch (error) {}
+		} catch (error) {
+			console.error(`[getModels] error writing ${router} models to file cache`, error)
+		}
 
 		return models
 	}
 
 	try {
 		models = await readModels(router)
 		// console.log(`[getModels] read ${router} models from file cache`)
-	} catch (error) {}
+	} catch (error) {
+		console.error(`[getModels] error reading ${router} models from file cache`, error)
+	}
 
 	return models ?? {}
 }

src/api/providers/fetchers/cache.ts renamed to src/api/providers/fetchers/modelCache.ts

@@ -0,0 +1,74 @@
+import * as path from "path"
+import fs from "fs/promises"
+
+import NodeCache from "node-cache"
+import sanitize from "sanitize-filename"
+
+import { ContextProxy } from "../../../core/config/ContextProxy"
+import { getCacheDirectoryPath } from "../../../shared/storagePathManager"
+import { RouterName, ModelRecord } from "../../../shared/api"
+import { fileExistsAtPath } from "../../../utils/fs"
+
+import { getOpenRouterModelEndpoints } from "./openrouter"
+
+const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
+
+const getCacheKey = (router: RouterName, modelId: string) => sanitize(`${router}_${modelId}`)
+
+async function writeModelEndpoints(key: string, data: ModelRecord) {
+	const filename = `${key}_endpoints.json`
+	const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath)
+	await fs.writeFile(path.join(cacheDir, filename), JSON.stringify(data, null, 2))
+}
+
+async function readModelEndpoints(key: string): Promise<ModelRecord | undefined> {
+	const filename = `${key}_endpoints.json`
+	const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath)
+	const filePath = path.join(cacheDir, filename)
+	const exists = await fileExistsAtPath(filePath)
+	return exists ? JSON.parse(await fs.readFile(filePath, "utf8")) : undefined
+}
+
+export const getModelEndpoints = async (router: RouterName, modelId?: string): Promise<ModelRecord> => {
+	// OpenRouter is the only provider that supports model endpoints, but you
+	// can see how we'd extend this to other providers in the future.
+	if (router !== "openrouter" || !modelId) {
+		return {}
+	}
+
+	const key = getCacheKey(router, modelId)
+	let modelProviders = memoryCache.get<ModelRecord>(key)
+
+	if (modelProviders) {
+		// console.log(`[getModelProviders] NodeCache hit for ${key} -> ${Object.keys(modelProviders).length}`)
+		return modelProviders
+	}
+
+	modelProviders = await getOpenRouterModelEndpoints(modelId)
+
+	if (Object.keys(modelProviders).length > 0) {
+		// console.log(`[getModelProviders] API fetch for ${key} -> ${Object.keys(modelProviders).length}`)
+		memoryCache.set(key, modelProviders)
+
+		try {
+			await writeModelEndpoints(key, modelProviders)
+			// console.log(`[getModelProviders] wrote ${key} endpoints to file cache`)
+		} catch (error) {
+			console.error(`[getModelProviders] error writing ${key} endpoints to file cache`, error)
+		}
+
+		return modelProviders
+	}
+
+	try {
+		modelProviders = await readModelEndpoints(router)
+		// console.log(`[getModelProviders] read ${key} endpoints from file cache`)
+	} catch (error) {
+		console.error(`[getModelProviders] error reading ${key} endpoints from file cache`, error)
+	}
+
+	return modelProviders ?? {}
+}
+
+export const flushModelProviders = async (router: RouterName, modelId: string) =>
+	memoryCache.del(getCacheKey(router, modelId))

src/api/providers/fetchers/modelEndpointCache.ts

@@ -20,7 +20,8 @@ import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/c
 import { getModelParams, SingleCompletionHandler } from "../index"
 import { DEFAULT_HEADERS, DEEP_SEEK_DEFAULT_TEMPERATURE } from "./constants"
 import { BaseProvider } from "./base-provider"
-import { getModels } from "./fetchers/cache"
+import { getModels } from "./fetchers/modelCache"
+import { getModelEndpoints } from "./fetchers/modelEndpointCache"
 
 const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]"
 
@@ -57,6 +58,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 	protected options: ApiHandlerOptions
 	private client: OpenAI
 	protected models: ModelRecord = {}
+	protected endpoints: ModelRecord = {}
 
 	constructor(options: ApiHandlerOptions) {
 		super()
@@ -168,13 +170,26 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 	}
 
 	public async fetchModel() {
-		this.models = await getModels("openrouter")
+		const [models, endpoints] = await Promise.all([
+			getModels("openrouter"),
+			getModelEndpoints("openrouter", this.options.openRouterModelId),
+		])
+
+		this.models = models
+		this.endpoints = endpoints
+
 		return this.getModel()
 	}
 
 	override getModel() {
 		const id = this.options.openRouterModelId ?? openRouterDefaultModelId
-		const info = this.models[id] ?? openRouterDefaultModelInfo
+		let info = this.models[id] ?? openRouterDefaultModelInfo
+
+		// If a specific provider is requested, use the endpoint for that provider.
+		if (this.options.openRouterSpecificProvider && this.endpoints[this.options.openRouterSpecificProvider]) {
+			info = this.endpoints[this.options.openRouterSpecificProvider]
+			console.log(`[OpenRouterHandler] Using specific provider: ${this.options.openRouterSpecificProvider}`, info)
+		}
 
 		const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || id === "perplexity/sonar-reasoning"
 

src/api/providers/openrouter.ts

@@ -12,7 +12,7 @@ import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { SingleCompletionHandler } from "../"
 import { BaseProvider } from "./base-provider"
 import { DEFAULT_HEADERS } from "./constants"
-import { getModels } from "./fetchers/cache"
+import { getModels } from "./fetchers/modelCache"
 import OpenAI from "openai"
 
 // Requesty usage includes an extra field for Anthropic use cases.

src/api/providers/requesty.ts

@@ -2,7 +2,7 @@ import OpenAI from "openai"
 
 import { ApiHandlerOptions, RouterName, ModelRecord, ModelInfo } from "../../shared/api"
 import { BaseProvider } from "./base-provider"
-import { getModels } from "./fetchers/cache"
+import { getModels } from "./fetchers/modelCache"
 
 type RouterProviderOptions = {
 	name: RouterName

src/api/providers/router-provider.ts

@@ -34,7 +34,7 @@ import { TelemetrySetting } from "../../shared/TelemetrySetting"
 import { getWorkspacePath } from "../../utils/path"
 import { Mode, defaultModeSlug } from "../../shared/modes"
 import { GlobalState } from "../../schemas"
-import { getModels, flushModels } from "../../api/providers/fetchers/cache"
+import { getModels, flushModels } from "../../api/providers/fetchers/modelCache"
 import { generateSystemPrompt } from "./generateSystemPrompt"
 
 const ALLOWED_VSCODE_SETTINGS = new Set(["terminal.integrated.inheritEnv"])