From 87cb0050d1719f4876f016793f23b373f1cccdd6 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 13 Feb 2026 16:49:46 -0400 Subject: [PATCH 1/5] fix: add retry logic for OpenAI embedding API failures Fixes #149 When importing or uploading skills, the OpenAI embedding API call could fail with transient errors (rate limits, timeouts, network issues), causing the entire import to fail with a generic "Server Error". This adds retry logic with exponential backoff (1s, 2s, 4s delays): - Retries on 429 (rate limit) and 5xx server errors - Retries on network/fetch errors - Logs warnings for debugging - Max 3 retries before failing with clear error message Co-Authored-By: Claude Opus 4.6 --- convex/lib/embeddings.ts | 71 ++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/convex/lib/embeddings.ts b/convex/lib/embeddings.ts index 972e1c4a86..feaa6cbe93 100644 --- a/convex/lib/embeddings.ts +++ b/convex/lib/embeddings.ts @@ -12,27 +12,56 @@ export async function generateEmbedding(text: string) { return emptyEmbedding() } - const response = await fetch('https://api.openai.com/v1/embeddings', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model: EMBEDDING_MODEL, - input: text, - }), - }) - - if (!response.ok) { - const message = await response.text() - throw new Error(`Embedding failed: ${message}`) - } + const maxRetries = 3 + const baseDelay = 1000 + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const response = await fetch('https://api.openai.com/v1/embeddings', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: EMBEDDING_MODEL, + input: text, + }), + }) + + if (!response.ok) { + const message = await response.text() + const isRateLimit = response.status === 429 + const isServerError = response.status >= 500 - const payload = (await response.json()) as { - data?: Array<{ embedding: number[] }> + if ((isRateLimit || isServerError) && attempt < maxRetries) { + const delay = baseDelay * Math.pow(2, attempt) + console.warn( + `OpenAI API error (${response.status}), retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`, + ) + await new Promise((resolve) => setTimeout(resolve, delay)) + continue + } + + throw new Error(`Embedding failed: ${message}`) + } + + const payload = (await response.json()) as { + data?: Array<{ embedding: number[] }> + } + const embedding = payload.data?.[0]?.embedding + if (!embedding) throw new Error('Embedding missing from response') + return embedding + } catch (error) { + if (attempt < maxRetries && error instanceof Error && error.message.includes('fetch')) { + const delay = baseDelay * Math.pow(2, attempt) + console.warn(`Network error, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`) + await new Promise((resolve) => setTimeout(resolve, delay)) + continue + } + throw error + } } - const embedding = payload.data?.[0]?.embedding - if (!embedding) throw new Error('Embedding missing from response') - return embedding + + throw new Error('Embedding failed after retries') } From c89360fd44b4336c34ea2731327c0aaa70a86295 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 13 Feb 2026 22:40:17 -0400 Subject: [PATCH 2/5] fix: correct retry count and broaden network error catch Co-Authored-By: Claude Opus 4.6 --- convex/lib/embeddings.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convex/lib/embeddings.ts b/convex/lib/embeddings.ts index feaa6cbe93..2871b14b5e 100644 --- a/convex/lib/embeddings.ts +++ b/convex/lib/embeddings.ts @@ -15,7 +15,7 @@ export async function generateEmbedding(text: string) { const maxRetries = 3 const baseDelay = 1000 - for (let attempt = 0; attempt <= maxRetries; attempt++) { + for (let attempt = 0; attempt < maxRetries; attempt++) { try { const response = await fetch('https://api.openai.com/v1/embeddings', { method: 'POST', @@ -53,7 +53,7 @@ export async function generateEmbedding(text: string) { if (!embedding) throw new Error('Embedding missing from response') return embedding } catch (error) { - if (attempt < maxRetries && error instanceof Error && error.message.includes('fetch')) { + if (attempt < maxRetries && error instanceof Error) { const delay = baseDelay * Math.pow(2, attempt) console.warn(`Network error, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`) await new Promise((resolve) => setTimeout(resolve, delay)) From 4348d2a663434ccc9a4ef9d7a0f026ff10995e22 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 13 Feb 2026 23:18:43 -0400 Subject: [PATCH 3/5] fix: address retry loop off-by-one, broaden error catch, preserve original error Co-Authored-By: Claude Opus 4.6 --- convex/lib/embeddings.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/convex/lib/embeddings.ts b/convex/lib/embeddings.ts index 2871b14b5e..47a69e82a2 100644 --- a/convex/lib/embeddings.ts +++ b/convex/lib/embeddings.ts @@ -14,6 +14,7 @@ export async function generateEmbedding(text: string) { const maxRetries = 3 const baseDelay = 1000 + let lastError: unknown for (let attempt = 0; attempt < maxRetries; attempt++) { try { @@ -34,7 +35,7 @@ export async function generateEmbedding(text: string) { const isRateLimit = response.status === 429 const isServerError = response.status >= 500 - if ((isRateLimit || isServerError) && attempt < maxRetries) { + if ((isRateLimit || isServerError) && attempt < maxRetries - 1) { const delay = baseDelay * Math.pow(2, attempt) console.warn( `OpenAI API error (${response.status}), retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`, @@ -53,7 +54,8 @@ export async function generateEmbedding(text: string) { if (!embedding) throw new Error('Embedding missing from response') return embedding } catch (error) { - if (attempt < maxRetries && error instanceof Error) { + lastError = error + if (attempt < maxRetries - 1 && error instanceof Error) { const delay = baseDelay * Math.pow(2, attempt) console.warn(`Network error, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`) await new Promise((resolve) => setTimeout(resolve, delay)) @@ -63,5 +65,5 @@ export async function generateEmbedding(text: string) { } } - throw new Error('Embedding failed after retries') + throw lastError ?? new Error('Embedding failed after retries') } From a50679765e266d89dc644e72afe480d4fdf967ae Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 14 Feb 2026 04:52:50 +0100 Subject: [PATCH 4/5] fix: harden embeddings retry semantics --- convex/lib/embeddings.test.ts | 93 +++++++++++++++++++++++++++ convex/lib/embeddings.ts | 118 ++++++++++++++++++++++++++++------ 2 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 convex/lib/embeddings.test.ts diff --git a/convex/lib/embeddings.test.ts b/convex/lib/embeddings.test.ts new file mode 100644 index 0000000000..95b0c3edaa --- /dev/null +++ b/convex/lib/embeddings.test.ts @@ -0,0 +1,93 @@ +/* @vitest-environment node */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { EMBEDDING_DIMENSIONS, generateEmbedding } from './embeddings' + +const fetchMock = vi.fn() +const consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + +const originalFetch = globalThis.fetch +const originalApiKey = process.env.OPENAI_API_KEY + +function jsonResponse(payload: unknown, init?: ResponseInit) { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { + 'content-type': 'application/json', + }, + ...init, + }) +} + +beforeEach(() => { + fetchMock.mockReset() + globalThis.fetch = fetchMock as typeof fetch + process.env.OPENAI_API_KEY = 'test-key' + consoleWarnSpy.mockClear() +}) + +afterEach(() => { + globalThis.fetch = originalFetch + + if (originalApiKey === undefined) { + delete process.env.OPENAI_API_KEY + } else { + process.env.OPENAI_API_KEY = originalApiKey + } + + vi.useRealTimers() +}) + +describe('generateEmbedding', () => { + it('returns zero embedding when OPENAI_API_KEY is missing', async () => { + delete process.env.OPENAI_API_KEY + const result = await generateEmbedding('hello world') + + expect(result).toHaveLength(EMBEDDING_DIMENSIONS) + expect(result.every((value) => value === 0)).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('retries on 429 responses and then succeeds', async () => { + vi.useFakeTimers() + fetchMock.mockResolvedValueOnce(new Response('rate limited', { status: 429 })) + fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [0.25, 0.75] }] })) + + const promise = generateEmbedding('retry me') + await vi.runAllTimersAsync() + + await expect(promise).resolves.toEqual([0.25, 0.75]) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('does not retry non-retryable 4xx responses', async () => { + fetchMock.mockResolvedValueOnce(new Response('bad request', { status: 400 })) + + await expect(generateEmbedding('bad')).rejects.toThrow('Embedding failed: bad request') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('retries on network failures and then succeeds', async () => { + vi.useFakeTimers() + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')) + fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [1, 2, 3] }] })) + + const promise = generateEmbedding('network retry') + await vi.runAllTimersAsync() + + await expect(promise).resolves.toEqual([1, 2, 3]) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('retries timeouts up to max attempts and preserves timeout error', async () => { + vi.useFakeTimers() + fetchMock.mockRejectedValue(new DOMException('aborted', 'AbortError')) + + const promise = generateEmbedding('always timeout') + const rejection = expect(promise).rejects.toThrow('OpenAI API request timed out after 10 seconds') + await vi.runAllTimersAsync() + + await rejection + expect(fetchMock).toHaveBeenCalledTimes(3) + }) +}) diff --git a/convex/lib/embeddings.ts b/convex/lib/embeddings.ts index 47a69e82a2..082aca6db2 100644 --- a/convex/lib/embeddings.ts +++ b/convex/lib/embeddings.ts @@ -1,10 +1,67 @@ export const EMBEDDING_MODEL = 'text-embedding-3-small' export const EMBEDDING_DIMENSIONS = 1536 +const EMBEDDING_ENDPOINT = 'https://api.openai.com/v1/embeddings' +const REQUEST_TIMEOUT_MS = 10_000 +const MAX_ATTEMPTS = 3 +const BASE_RETRY_DELAY_MS = 1_000 + +class RetryableEmbeddingError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options) + this.name = 'RetryableEmbeddingError' + } +} + function emptyEmbedding() { return Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0) } +function parseRetryAfterMs(retryAfterHeader: string | null) { + if (!retryAfterHeader) return null + + const seconds = Number(retryAfterHeader) + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.round(seconds * 1000) + } + + const dateMs = Date.parse(retryAfterHeader) + if (Number.isFinite(dateMs)) { + return Math.max(0, dateMs - Date.now()) + } + + return null +} + +function getRetryDelayMs(attempt: number, retryAfterMs: number | null) { + const exponentialDelayMs = BASE_RETRY_DELAY_MS * 2 ** attempt + if (retryAfterMs == null) return exponentialDelayMs + return Math.max(exponentialDelayMs, retryAfterMs) +} + +function normalizeRetryableNetworkError(error: unknown) { + if (!(error instanceof Error)) return null + + if (error.name === 'AbortError') { + return new RetryableEmbeddingError( + `OpenAI API request timed out after ${Math.floor(REQUEST_TIMEOUT_MS / 1000)} seconds`, + { cause: error }, + ) + } + + if (error instanceof TypeError) { + return new RetryableEmbeddingError(`Embedding request failed: ${error.message}`, { cause: error }) + } + + return null +} + +function sleep(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms) + }) +} + export async function generateEmbedding(text: string) { const apiKey = process.env.OPENAI_API_KEY if (!apiKey) { @@ -12,13 +69,14 @@ export async function generateEmbedding(text: string) { return emptyEmbedding() } - const maxRetries = 3 - const baseDelay = 1000 - let lastError: unknown + let lastRetryableError: RetryableEmbeddingError | null = null + + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) - for (let attempt = 0; attempt < maxRetries; attempt++) { try { - const response = await fetch('https://api.openai.com/v1/embeddings', { + const response = await fetch(EMBEDDING_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -28,20 +86,29 @@ export async function generateEmbedding(text: string) { model: EMBEDDING_MODEL, input: text, }), + signal: controller.signal, }) if (!response.ok) { const message = await response.text() - const isRateLimit = response.status === 429 - const isServerError = response.status >= 500 - - if ((isRateLimit || isServerError) && attempt < maxRetries - 1) { - const delay = baseDelay * Math.pow(2, attempt) - console.warn( - `OpenAI API error (${response.status}), retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`, + const isRetryableStatus = response.status === 429 || response.status >= 500 + if (isRetryableStatus) { + const retryableError = new RetryableEmbeddingError( + `Embedding failed (${response.status}): ${message}`, ) - await new Promise((resolve) => setTimeout(resolve, delay)) - continue + lastRetryableError = retryableError + + if (attempt < MAX_ATTEMPTS - 1) { + const retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after')) + const delayMs = getRetryDelayMs(attempt, retryAfterMs) + console.warn( + `OpenAI embeddings retry in ${delayMs}ms (attempt ${attempt + 1}/${MAX_ATTEMPTS})`, + ) + await sleep(delayMs) + continue + } + + throw retryableError } throw new Error(`Embedding failed: ${message}`) @@ -54,16 +121,25 @@ export async function generateEmbedding(text: string) { if (!embedding) throw new Error('Embedding missing from response') return embedding } catch (error) { - lastError = error - if (attempt < maxRetries - 1 && error instanceof Error) { - const delay = baseDelay * Math.pow(2, attempt) - console.warn(`Network error, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`) - await new Promise((resolve) => setTimeout(resolve, delay)) - continue + const retryableNetworkError = normalizeRetryableNetworkError(error) + if (retryableNetworkError) { + lastRetryableError = retryableNetworkError + if (attempt < MAX_ATTEMPTS - 1) { + const delayMs = getRetryDelayMs(attempt, null) + console.warn( + `OpenAI embeddings network retry in ${delayMs}ms (attempt ${attempt + 1}/${MAX_ATTEMPTS})`, + ) + await sleep(delayMs) + continue + } + throw retryableNetworkError } + throw error + } finally { + clearTimeout(timeoutId) } } - throw lastError ?? new Error('Embedding failed after retries') + throw lastRetryableError ?? new Error('Embedding failed after retries') } From b610b7c3aafa6cba74cdfed254e132be300a0155 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 14 Feb 2026 04:53:36 +0100 Subject: [PATCH 5/5] style: format embeddings retry changes --- convex/lib/embeddings.test.ts | 4 +++- convex/lib/embeddings.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/convex/lib/embeddings.test.ts b/convex/lib/embeddings.test.ts index 95b0c3edaa..c36cc1d463 100644 --- a/convex/lib/embeddings.test.ts +++ b/convex/lib/embeddings.test.ts @@ -84,7 +84,9 @@ describe('generateEmbedding', () => { fetchMock.mockRejectedValue(new DOMException('aborted', 'AbortError')) const promise = generateEmbedding('always timeout') - const rejection = expect(promise).rejects.toThrow('OpenAI API request timed out after 10 seconds') + const rejection = expect(promise).rejects.toThrow( + 'OpenAI API request timed out after 10 seconds', + ) await vi.runAllTimersAsync() await rejection diff --git a/convex/lib/embeddings.ts b/convex/lib/embeddings.ts index 082aca6db2..69215692ef 100644 --- a/convex/lib/embeddings.ts +++ b/convex/lib/embeddings.ts @@ -50,7 +50,9 @@ function normalizeRetryableNetworkError(error: unknown) { } if (error instanceof TypeError) { - return new RetryableEmbeddingError(`Embedding request failed: ${error.message}`, { cause: error }) + return new RetryableEmbeddingError(`Embedding request failed: ${error.message}`, { + cause: error, + }) } return null