Skip to content

Commit 97beb13

Browse files
daniel-lxsdaniel-lxs
committed
Remove coalescing, background refresh, and cache logging
Address review feedback by removing out-of-scope optimizations: 1. Remove in-flight coalescing infrastructure - Delete inFlightModelFetches and inFlightEndpointFetches maps - Eliminate promise sharing across concurrent requests 2. Remove background refresh on file cache hit - Simplify to synchronous flow: memory → file → network - No more fire-and-forget background updates 3. Remove cache performance logging - Delete console.log statements for cache_hit, file_hit, bg_refresh - Clean up debugging artifacts from development 4. Fix active-provider scoping gap - Include ollama/lmstudio/huggingface in requestRouterModels when active - Prevents empty response that breaks chat flows for local providers Result: Simpler, more maintainable code focused on core goal of reducing unnecessary network requests by scoping to active provider.
1 parent 3f4edff commit 97beb13

File tree

3 files changed

+107
-229
lines changed

3 files changed

+107
-229
lines changed

‎src/api/providers/fetchers/modelCache.ts‎

Lines changed: 55 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,6 @@ import { getRooModels } from "./roo"
2626

2727
const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
2828

29-
// Coalesce concurrent fetches per provider within this extension host
30-
const inFlightModelFetches = new Map<RouterName, Promise<ModelRecord>>()
31-
3229
async function writeModels(router: RouterName, data: ModelRecord) {
3330
const filename = `${router}_models.json`
3431
const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath)
@@ -63,160 +60,75 @@ export const getModels = async (options: GetModelsOptions): Promise<ModelRecord>
6360
return cached
6461
}
6562

66-
// 2) Try file cache snapshot (Option A), then kick off background refresh
63+
// 2) Try file cache snapshot
6764
try {
6865
const file = await readModels(provider)
6966
if (file && Object.keys(file).length > 0) {
7067
memoryCache.set(provider, file)
71-
72-
// Start background refresh if not already in-flight (do not await)
73-
if (!inFlightModelFetches.has(provider)) {
74-
const signal = AbortSignal.timeout(30_000)
75-
const bgPromise = (async (): Promise<ModelRecord> => {
76-
let models: ModelRecord = {}
77-
switch (provider) {
78-
case "openrouter":
79-
models = await getOpenRouterModels(undefined, signal)
80-
break
81-
case "requesty":
82-
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
83-
break
84-
case "glama":
85-
models = await getGlamaModels(signal)
86-
break
87-
case "unbound":
88-
models = await getUnboundModels(options.apiKey, signal)
89-
break
90-
case "litellm":
91-
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
92-
break
93-
case "ollama":
94-
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
95-
break
96-
case "lmstudio":
97-
models = await getLMStudioModels(options.baseUrl, signal)
98-
break
99-
case "deepinfra":
100-
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
101-
break
102-
case "io-intelligence":
103-
models = await getIOIntelligenceModels(options.apiKey, signal)
104-
break
105-
case "vercel-ai-gateway":
106-
models = await getVercelAiGatewayModels(undefined, signal)
107-
break
108-
case "huggingface":
109-
models = await getHuggingFaceModels(signal)
110-
break
111-
case "roo": {
112-
const rooBaseUrl =
113-
options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
114-
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
115-
break
116-
}
117-
default:
118-
throw new Error(`Unknown provider: ${provider}`)
119-
}
120-
121-
memoryCache.set(provider, models)
122-
await writeModels(provider, models).catch((err) => {
123-
console.error(
124-
`[modelCache] Error writing ${provider} to file cache during background refresh:`,
125-
err instanceof Error ? err.message : String(err),
126-
)
127-
})
128-
return models || {}
129-
})()
130-
131-
inFlightModelFetches.set(provider, bgPromise)
132-
Promise.resolve(bgPromise)
133-
.catch((err) => {
134-
console.error(
135-
`[modelCache] Background refresh failed for ${provider}:`,
136-
err instanceof Error ? err.message : String(err),
137-
)
138-
})
139-
.finally(() => inFlightModelFetches.delete(provider))
140-
}
141-
14268
return file
14369
}
14470
} catch {
145-
// ignore file read errors; fall through to network/coalesce path
146-
}
147-
148-
// 3) Coalesce concurrent fetches
149-
const existing = inFlightModelFetches.get(provider)
150-
if (existing) {
151-
return existing
71+
// ignore file read errors; fall through to network fetch
15272
}
15373

154-
// 4) Network fetch wrapped as a single in-flight promise for this provider
74+
// 3) Network fetch
15575
const signal = AbortSignal.timeout(30_000)
156-
const fetchPromise = (async (): Promise<ModelRecord> => {
157-
let models: ModelRecord = {}
158-
switch (provider) {
159-
case "openrouter":
160-
models = await getOpenRouterModels(undefined, signal)
161-
break
162-
case "requesty":
163-
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
164-
break
165-
case "glama":
166-
models = await getGlamaModels(signal)
167-
break
168-
case "unbound":
169-
models = await getUnboundModels(options.apiKey, signal)
170-
break
171-
case "litellm":
172-
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
173-
break
174-
case "ollama":
175-
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
176-
break
177-
case "lmstudio":
178-
models = await getLMStudioModels(options.baseUrl, signal)
179-
break
180-
case "deepinfra":
181-
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
182-
break
183-
case "io-intelligence":
184-
models = await getIOIntelligenceModels(options.apiKey, signal)
185-
break
186-
case "vercel-ai-gateway":
187-
models = await getVercelAiGatewayModels(undefined, signal)
188-
break
189-
case "huggingface":
190-
models = await getHuggingFaceModels(signal)
191-
break
192-
case "roo": {
193-
const rooBaseUrl =
194-
options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
195-
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
196-
break
197-
}
198-
default: {
199-
throw new Error(`Unknown provider: ${provider}`)
200-
}
76+
let models: ModelRecord = {}
77+
78+
switch (provider) {
79+
case "openrouter":
80+
models = await getOpenRouterModels(undefined, signal)
81+
break
82+
case "requesty":
83+
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
84+
break
85+
case "glama":
86+
models = await getGlamaModels(signal)
87+
break
88+
case "unbound":
89+
models = await getUnboundModels(options.apiKey, signal)
90+
break
91+
case "litellm":
92+
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
93+
break
94+
case "ollama":
95+
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
96+
break
97+
case "lmstudio":
98+
models = await getLMStudioModels(options.baseUrl, signal)
99+
break
100+
case "deepinfra":
101+
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
102+
break
103+
case "io-intelligence":
104+
models = await getIOIntelligenceModels(options.apiKey, signal)
105+
break
106+
case "vercel-ai-gateway":
107+
models = await getVercelAiGatewayModels(undefined, signal)
108+
break
109+
case "huggingface":
110+
models = await getHuggingFaceModels(signal)
111+
break
112+
case "roo": {
113+
const rooBaseUrl = options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
114+
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
115+
break
201116
}
202-
memoryCache.set(provider, models)
117+
default: {
118+
throw new Error(`Unknown provider: ${provider}`)
119+
}
120+
}
203121

204-
await writeModels(provider, models).catch((err) => {
205-
console.error(
206-
`[modelCache] Error writing ${provider} to file cache after network fetch:`,
207-
err instanceof Error ? err.message : String(err),
208-
)
209-
})
122+
memoryCache.set(provider, models)
210123

211-
return models || {}
212-
})()
124+
await writeModels(provider, models).catch((err) => {
125+
console.error(
126+
`[modelCache] Error writing ${provider} to file cache after network fetch:`,
127+
err instanceof Error ? err.message : String(err),
128+
)
129+
})
213130

214-
inFlightModelFetches.set(provider, fetchPromise)
215-
try {
216-
return await fetchPromise
217-
} finally {
218-
inFlightModelFetches.delete(provider)
219-
}
131+
return models || {}
220132
}
221133

222134
/**

‎src/api/providers/fetchers/modelEndpointCache.ts‎

Lines changed: 23 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@ import { getOpenRouterModelEndpoints } from "./openrouter"
1414

1515
const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
1616

17-
// Coalesce concurrent endpoint fetches per (router,modelId)
18-
const inFlightEndpointFetches = new Map<string, Promise<ModelRecord>>()
19-
2017
const getCacheKey = (router: RouterName, modelId: string) => sanitize(`${router}_${modelId}`)
2118

2219
async function writeModelEndpoints(key: string, data: ModelRecord) {
@@ -53,107 +50,47 @@ export const getModelEndpoints = async ({
5350
// 1) Try memory cache
5451
const cached = memoryCache.get<ModelRecord>(key)
5552
if (cached) {
56-
// Using console.log for cache layer logging (no provider access in utility functions)
57-
console.log(`[endpointCache] cache_hit: ${key} (${Object.keys(cached).length} endpoints)`)
5853
return cached
5954
}
6055

61-
// 2) Try file cache snapshot (Option A), then kick off background refresh
56+
// 2) Try file cache snapshot
6257
try {
6358
const file = await readModelEndpoints(key)
6459
if (file && Object.keys(file).length > 0) {
65-
// Using console.log for cache layer logging (no provider access in utility functions)
66-
console.log(`[endpointCache] file_hit: ${key} (${Object.keys(file).length} endpoints, bg_refresh queued)`)
67-
// Populate memory cache immediately
6860
memoryCache.set(key, file)
69-
70-
// Start background refresh if not already in-flight (do not await)
71-
if (!inFlightEndpointFetches.has(key)) {
72-
const signal = AbortSignal.timeout(30_000)
73-
const bgPromise = (async (): Promise<ModelRecord> => {
74-
const modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
75-
if (Object.keys(modelProviders).length > 0) {
76-
console.log(
77-
`[endpointCache] bg_refresh_done: ${key} (${Object.keys(modelProviders).length} endpoints)`,
78-
)
79-
memoryCache.set(key, modelProviders)
80-
try {
81-
await writeModelEndpoints(key, modelProviders)
82-
} catch (error) {
83-
console.error(
84-
`[endpointCache] Error writing ${key} to file cache during background refresh:`,
85-
error instanceof Error ? error.message : String(error),
86-
)
87-
}
88-
return modelProviders
89-
}
90-
return {}
91-
})()
92-
93-
inFlightEndpointFetches.set(key, bgPromise)
94-
Promise.resolve(bgPromise)
95-
.catch((err) => {
96-
// Log background refresh failures for monitoring
97-
console.error(
98-
`[endpointCache] Background refresh failed for ${key}:`,
99-
err instanceof Error ? err.message : String(err),
100-
)
101-
})
102-
.finally(() => inFlightEndpointFetches.delete(key))
103-
}
104-
10561
return file
10662
}
10763
} catch {
108-
// ignore file read errors; fall through
64+
// ignore file read errors; fall through to network fetch
10965
}
11066

111-
// 3) Coalesce concurrent fetches
112-
const inFlight = inFlightEndpointFetches.get(key)
113-
if (inFlight) {
114-
// Using console.log for cache layer logging (no provider access in utility functions)
115-
console.log(`[endpointCache] coalesced_wait: ${key}`)
116-
return inFlight
117-
}
118-
119-
// 4) Single network fetch for this key
67+
// 3) Network fetch
12068
const signal = AbortSignal.timeout(30_000)
121-
const fetchPromise = (async (): Promise<ModelRecord> => {
122-
let modelProviders: ModelRecord = {}
123-
modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
124-
125-
if (Object.keys(modelProviders).length > 0) {
126-
console.log(`[endpointCache] network_fetch_done: ${key} (${Object.keys(modelProviders).length} endpoints)`)
127-
// Update memory cache first
128-
memoryCache.set(key, modelProviders)
129-
130-
// Best-effort persist
131-
try {
132-
await writeModelEndpoints(key, modelProviders)
133-
} catch (error) {
134-
console.error(
135-
`[endpointCache] Error writing ${key} to file cache after network fetch:`,
136-
error instanceof Error ? error.message : String(error),
137-
)
138-
}
139-
140-
return modelProviders
141-
}
69+
let modelProviders: ModelRecord = {}
70+
71+
modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
72+
73+
if (Object.keys(modelProviders).length > 0) {
74+
memoryCache.set(key, modelProviders)
14275

143-
// Fallback to file cache if network returned empty (rare)
14476
try {
145-
const file = await readModelEndpoints(key)
146-
return file ?? {}
147-
} catch {
148-
return {}
77+
await writeModelEndpoints(key, modelProviders)
78+
} catch (error) {
79+
console.error(
80+
`[endpointCache] Error writing ${key} to file cache after network fetch:`,
81+
error instanceof Error ? error.message : String(error),
82+
)
14983
}
150-
})()
15184

152-
inFlightEndpointFetches.set(key, fetchPromise)
85+
return modelProviders
86+
}
87+
88+
// Fallback to file cache if network returned empty (rare)
15389
try {
154-
return await fetchPromise
155-
} finally {
156-
inFlightEndpointFetches.delete(key)
90+
const file = await readModelEndpoints(key)
91+
return file ?? {}
92+
} catch {
93+
return {}
15794
}
15895
}
15996

0 commit comments

Comments
 (0)