Skip to content

Commit b9c9d21

Browse files
roomoteroomote
committed
fix: improve Ollama embeddings error handling and add retry logic
- Add detailed error logging to capture actual Ollama API responses - Implement retry logic with exponential backoff for transient failures - Add better error detection for model not found errors - Improve error messages to be more descriptive - Add validation for embeddings response format Fixes #6526
1 parent 5c05762 commit b9c9d21

File tree

1 file changed

+144
-62
lines changed

1 file changed

+144
-62
lines changed

‎src/services/code-index/embedders/ollama.ts‎

Lines changed: 144 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ import { TelemetryEventName } from "@roo-code/types"
1111
const OLLAMA_EMBEDDING_TIMEOUT_MS = 60000 // 60 seconds for embedding requests
1212
const OLLAMA_VALIDATION_TIMEOUT_MS = 30000 // 30 seconds for validation requests
1313

14+
// Retry configuration
15+
const MAX_RETRIES = 3
16+
const INITIAL_RETRY_DELAY_MS = 1000 // 1 second
17+
const MAX_RETRY_DELAY_MS = 10000 // 10 seconds
18+
1419
/**
1520
* Implements the IEmbedder interface using a local Ollama instance.
1621
*/
@@ -64,77 +69,154 @@ export class CodeIndexOllamaEmbedder implements IEmbedder {
6469
})
6570
: texts
6671

67-
try {
68-
// Note: Standard Ollama API uses 'prompt' for single text, not 'input' for array.
69-
// Implementing based on user's specific request structure.
72+
let lastError: Error | null = null
73+
let retryDelay = INITIAL_RETRY_DELAY_MS
7074

71-
// Add timeout to prevent indefinite hanging
72-
const controller = new AbortController()
73-
const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS)
75+
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
76+
try {
77+
// Note: Standard Ollama API uses 'prompt' for single text, not 'input' for array.
78+
// Implementing based on user's specific request structure.
7479

75-
const response = await fetch(url, {
76-
method: "POST",
77-
headers: {
78-
"Content-Type": "application/json",
79-
},
80-
body: JSON.stringify({
81-
model: modelToUse,
82-
input: processedTexts, // Using 'input' as requested
83-
}),
84-
signal: controller.signal,
85-
})
86-
clearTimeout(timeoutId)
87-
88-
if (!response.ok) {
89-
let errorBody = t("embeddings:ollama.couldNotReadErrorBody")
90-
try {
91-
errorBody = await response.text()
92-
} catch (e) {
93-
// Ignore error reading body
94-
}
95-
throw new Error(
96-
t("embeddings:ollama.requestFailed", {
97-
status: response.status,
98-
statusText: response.statusText,
99-
errorBody,
80+
// Add timeout to prevent indefinite hanging
81+
const controller = new AbortController()
82+
const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS)
83+
84+
const response = await fetch(url, {
85+
method: "POST",
86+
headers: {
87+
"Content-Type": "application/json",
88+
},
89+
body: JSON.stringify({
90+
model: modelToUse,
91+
input: processedTexts, // Using 'input' as requested
10092
}),
101-
)
102-
}
93+
signal: controller.signal,
94+
})
95+
clearTimeout(timeoutId)
10396

104-
const data = await response.json()
97+
if (!response.ok) {
98+
let errorBody = t("embeddings:ollama.couldNotReadErrorBody")
99+
let errorDetails: any = {}
100+
try {
101+
errorBody = await response.text()
102+
// Try to parse as JSON to get more details
103+
try {
104+
errorDetails = JSON.parse(errorBody)
105+
} catch {
106+
// Not JSON, use as is
107+
}
108+
} catch (e) {
109+
// Ignore error reading body
110+
}
105111

106-
// Extract embeddings using 'embeddings' key as requested
107-
const embeddings = data.embeddings
108-
if (!embeddings || !Array.isArray(embeddings)) {
109-
throw new Error(t("embeddings:ollama.invalidResponseStructure"))
110-
}
112+
// Check if it's a model not found error
113+
if (
114+
response.status === 404 ||
115+
errorDetails.error?.includes("model") ||
116+
errorDetails.error?.includes("not found")
117+
) {
118+
throw new Error(t("embeddings:ollama.modelNotFound", { modelId: modelToUse }))
119+
}
111120

112-
return {
113-
embeddings: embeddings,
114-
}
115-
} catch (error: any) {
116-
// Capture telemetry before reformatting the error
117-
TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, {
118-
error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
119-
stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined,
120-
location: "OllamaEmbedder:createEmbeddings",
121-
})
122-
123-
// Log the original error for debugging purposes
124-
console.error("Ollama embedding failed:", error)
125-
126-
// Handle specific error types with better messages
127-
if (error.name === "AbortError") {
128-
throw new Error(t("embeddings:validation.connectionFailed"))
129-
} else if (error.message?.includes("fetch failed") || error.code === "ECONNREFUSED") {
130-
throw new Error(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl }))
131-
} else if (error.code === "ENOTFOUND") {
132-
throw new Error(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl }))
121+
throw new Error(
122+
t("embeddings:ollama.requestFailed", {
123+
status: response.status,
124+
statusText: response.statusText,
125+
errorBody,
126+
}),
127+
)
128+
}
129+
130+
const data = await response.json()
131+
132+
// Log the response structure for debugging
133+
if (!data || typeof data !== "object") {
134+
console.error("Ollama API response is not an object:", data)
135+
throw new Error(t("embeddings:ollama.invalidResponseStructure"))
136+
}
137+
138+
// Extract embeddings using 'embeddings' key as requested
139+
const embeddings = data.embeddings
140+
if (!embeddings || !Array.isArray(embeddings)) {
141+
console.error("Ollama API response structure:", JSON.stringify(data, null, 2))
142+
throw new Error(t("embeddings:ollama.invalidResponseStructure"))
143+
}
144+
145+
// Validate that embeddings is an array of arrays (2D array)
146+
if (embeddings.length > 0 && !Array.isArray(embeddings[0])) {
147+
console.error(
148+
"Ollama embeddings format invalid - expected array of arrays, got:",
149+
typeof embeddings[0],
150+
)
151+
throw new Error(t("embeddings:ollama.invalidResponseStructure"))
152+
}
153+
154+
return {
155+
embeddings: embeddings,
156+
}
157+
} catch (error: any) {
158+
lastError = error
159+
160+
// Don't retry for certain errors
161+
if (
162+
error.message?.includes(t("embeddings:ollama.modelNotFound", { modelId: "" }).split(":")[0]) ||
163+
error.message?.includes(t("embeddings:ollama.invalidResponseStructure"))
164+
) {
165+
break
166+
}
167+
168+
// Check if we should retry
169+
if (attempt < MAX_RETRIES) {
170+
// Check if it's a transient error that we should retry
171+
if (
172+
error.name === "AbortError" ||
173+
error.message?.includes("fetch failed") ||
174+
error.code === "ECONNREFUSED" ||
175+
error.code === "ENOTFOUND" ||
176+
error.code === "ETIMEDOUT" ||
177+
error.code === "ECONNRESET"
178+
) {
179+
console.log(`Ollama embedding attempt ${attempt} failed, retrying in ${retryDelay}ms...`)
180+
await new Promise((resolve) => setTimeout(resolve, retryDelay))
181+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY_MS) // Exponential backoff
182+
continue
183+
}
184+
}
185+
186+
// If we're here, we're not retrying
187+
break
133188
}
189+
}
190+
191+
// If we get here, all retries failed
192+
if (!lastError) {
193+
lastError = new Error("Unknown error in Ollama embedder")
194+
}
195+
// Capture telemetry before reformatting the error
196+
TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, {
197+
error: sanitizeErrorMessage(lastError instanceof Error ? lastError.message : String(lastError)),
198+
stack: lastError instanceof Error ? sanitizeErrorMessage(lastError.stack || "") : undefined,
199+
location: "OllamaEmbedder:createEmbeddings",
200+
})
201+
202+
// Log the original error for debugging purposes
203+
console.error("Ollama embedding failed after all retries:", lastError)
134204

135-
// Re-throw a more specific error for the caller
136-
throw new Error(t("embeddings:ollama.embeddingFailed", { message: error.message }))
205+
// Handle specific error types with better messages
206+
if (lastError.name === "AbortError") {
207+
throw new Error(t("embeddings:validation.connectionFailed"))
208+
} else if (lastError.message?.includes("fetch failed") || (lastError as any).code === "ECONNREFUSED") {
209+
throw new Error(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl }))
210+
} else if ((lastError as any).code === "ENOTFOUND") {
211+
throw new Error(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl }))
212+
} else if (lastError.message?.includes(t("embeddings:ollama.modelNotFound", { modelId: "" }).split(":")[0])) {
213+
throw lastError // Re-throw model not found as is
214+
} else if (lastError.message?.includes(t("embeddings:ollama.invalidResponseStructure"))) {
215+
throw lastError // Re-throw invalid response structure as is
137216
}
217+
218+
// Re-throw a more specific error for the caller
219+
throw new Error(t("embeddings:ollama.embeddingFailed", { message: lastError.message }))
138220
}
139221

140222
/**

0 commit comments

Comments
 (0)