From 852a1265267e940d23467c2452db7344f37c0c2d Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Sun, 8 Jun 2025 18:59:58 +0000 Subject: [PATCH 01/10] Manually specify openai-compat format and parse it --- .../code-index/embedders/openai-compatible.ts | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 421cb7262c..fd1004af45 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -111,10 +111,37 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const response = await this.embeddingsClient.embeddings.create({ input: batchTexts, model: model, + encoding_format: "base64", // Use base64 to protect embedding dimensions from openai sabotage }) + // Convert base64 embeddings to float32 arrays + const processedEmbeddings = response.data.map((item: any) => { + if (typeof item.embedding === "string") { + const buffer = Buffer.from(item.embedding, "base64") + const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + return { + ...item, + embedding: Array.from(float32Array), + } + } + return item + }) + + // Replace the original data with processed embeddings + response.data = processedEmbeddings + + const embeddings = response.data.map((item) => item.embedding) + + console.log(`[OpenAI-Compatible Embedder] After mapping - embedding length: ${embeddings[0]?.length}`) + if (embeddings[0]) { + console.log( + `[OpenAI-Compatible Embedder] First 10 values after mapping:`, + embeddings[0].slice(0, 5), + ) + } + return { - embeddings: response.data.map((item) => item.embedding), + embeddings, usage: { promptTokens: response.usage?.prompt_tokens || 0, totalTokens: response.usage?.total_tokens || 0, From 6385a4b744e048250b875875297dc676bec5a2f3 Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Sun, 8 Jun 2025 20:22:58 +0000 Subject: [PATCH 02/10] fixup! Manually specify openai-compat format and parse it --- src/services/code-index/embedders/openai-compatible.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index fd1004af45..6c2b1222b7 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -141,7 +141,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } return { - embeddings, + embeddings: embeddings, usage: { promptTokens: response.usage?.prompt_tokens || 0, totalTokens: response.usage?.total_tokens || 0, From 681d8d24009252272922ba52f95f66535f499bdd Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Sun, 8 Jun 2025 20:41:34 +0000 Subject: [PATCH 03/10] Expect base64 in embedding test arguments --- .../code-index/embedders/__tests__/openai-compatible.spec.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index e1e5c64cd5..9b55c7243d 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -110,6 +110,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ input: testTexts, model: testModelId, + encoding_format: "base64", }) expect(result).toEqual({ embeddings: [[0.1, 0.2, 0.3]], @@ -130,6 +131,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ input: testTexts, model: testModelId, + encoding_format: "base64", }) expect(result).toEqual({ embeddings: [ @@ -154,6 +156,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ input: testTexts, model: customModel, + encoding_format: "base64", }) }) From 670cbe3e64cd181fd88c9f3f025ef3f7b1ab34ad Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Wed, 11 Jun 2025 03:20:51 +0000 Subject: [PATCH 04/10] fixup! Manually specify openai-compat format and parse it Remove debug logs --- src/services/code-index/embedders/openai-compatible.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 6c2b1222b7..c8acb977bf 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -132,14 +132,6 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const embeddings = response.data.map((item) => item.embedding) - console.log(`[OpenAI-Compatible Embedder] After mapping - embedding length: ${embeddings[0]?.length}`) - if (embeddings[0]) { - console.log( - `[OpenAI-Compatible Embedder] First 10 values after mapping:`, - embeddings[0].slice(0, 5), - ) - } - return { embeddings: embeddings, usage: { From 8b6bbcee59af104839ef900d90637fbab70c783b Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Wed, 11 Jun 2025 03:40:47 +0000 Subject: [PATCH 05/10] fixup! Manually specify openai-compat format and parse it Improve comment --- src/services/code-index/embedders/openai-compatible.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index c8acb977bf..cc7be40844 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -111,7 +111,12 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const response = await this.embeddingsClient.embeddings.create({ input: batchTexts, model: model, - encoding_format: "base64", // Use base64 to protect embedding dimensions from openai sabotage + // The OpenAI package has custom parsing that truncates embedding dimension to 256, + // which destroys accuracy. + // If we pass `encoding_format: "base64"`, it does not perform any parsing, + // leaving parsing up to us. This is likely a bug in the OpenAI package, possibly + // addressed by https://github.com/openai/openai-node/pull/1448 (but maybe not) + encoding_format: "base64", }) // Convert base64 embeddings to float32 arrays From 51853371d1e5481cf521b3b708f92a712ff162e4 Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Thu, 12 Jun 2025 03:50:09 +0000 Subject: [PATCH 06/10] fixup! Manually specify openai-compat format and parse it --- src/services/code-index/embedders/openai-compatible.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index cc7be40844..2f0051879d 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -114,8 +114,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { // The OpenAI package has custom parsing that truncates embedding dimension to 256, // which destroys accuracy. // If we pass `encoding_format: "base64"`, it does not perform any parsing, - // leaving parsing up to us. This is likely a bug in the OpenAI package, possibly - // addressed by https://github.com/openai/openai-node/pull/1448 (but maybe not) + // leaving parsing up to us. encoding_format: "base64", }) From b20841e8ba63746eee4671b4ffdd02bf74c24a66 Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Thu, 12 Jun 2025 03:08:31 +0000 Subject: [PATCH 07/10] Add tests to exercise base64 decode of embeddings --- .../__tests__/openai-compatible.spec.ts | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index 9b55c7243d..508505a027 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -176,6 +176,97 @@ describe("OpenAICompatibleEmbedder", () => { }) }) + /** + * Test base64 conversion logic + */ + describe("base64 conversion", () => { + it("should convert base64 encoded embeddings to float arrays", async () => { + const testTexts = ["Hello world"] + + // Create a Float32Array with test values that can be exactly represented in Float32 + const testEmbedding = new Float32Array([0.25, 0.5, 0.75, 1.0]) + + // Convert to base64 string (simulating what OpenAI API returns) + const buffer = Buffer.from(testEmbedding.buffer) + const base64String = buffer.toString("base64") + + const mockResponse = { + data: [{ embedding: base64String }], // Base64 string instead of array + usage: { prompt_tokens: 10, total_tokens: 15 }, + } + mockEmbeddingsCreate.mockResolvedValue(mockResponse) + + const result = await embedder.createEmbeddings(testTexts) + + expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ + input: testTexts, + model: testModelId, + encoding_format: "base64", + }) + + // Verify the base64 string was converted back to the original float array + expect(result).toEqual({ + embeddings: [[0.25, 0.5, 0.75, 1.0]], + usage: { promptTokens: 10, totalTokens: 15 }, + }) + }) + + it("should handle multiple base64 encoded embeddings", async () => { + const testTexts = ["Hello world", "Goodbye world"] + + // Create test embeddings with values that can be exactly represented in Float32 + const embedding1 = new Float32Array([0.25, 0.5, 0.75]) + const embedding2 = new Float32Array([1.0, 1.25, 1.5]) + + // Convert to base64 strings + const base64String1 = Buffer.from(embedding1.buffer).toString("base64") + const base64String2 = Buffer.from(embedding2.buffer).toString("base64") + + const mockResponse = { + data: [{ embedding: base64String1 }, { embedding: base64String2 }], + usage: { prompt_tokens: 20, total_tokens: 30 }, + } + mockEmbeddingsCreate.mockResolvedValue(mockResponse) + + const result = await embedder.createEmbeddings(testTexts) + + expect(result).toEqual({ + embeddings: [ + [0.25, 0.5, 0.75], + [1.0, 1.25, 1.5], + ], + usage: { promptTokens: 20, totalTokens: 30 }, + }) + }) + + it("should handle mixed base64 and array embeddings", async () => { + const testTexts = ["Hello world", "Goodbye world"] + + // Create one base64 embedding and one regular array (edge case) + const embedding1 = new Float32Array([0.25, 0.5, 0.75]) + const base64String1 = Buffer.from(embedding1.buffer).toString("base64") + + const mockResponse = { + data: [ + { embedding: base64String1 }, // Base64 string + { embedding: [1.0, 1.25, 1.5] }, // Regular array + ], + usage: { prompt_tokens: 20, total_tokens: 30 }, + } + mockEmbeddingsCreate.mockResolvedValue(mockResponse) + + const result = await embedder.createEmbeddings(testTexts) + + expect(result).toEqual({ + embeddings: [ + [0.25, 0.5, 0.75], + [1.0, 1.25, 1.5], + ], + usage: { promptTokens: 20, totalTokens: 30 }, + }) + }) + }) + /** * Test batching logic when texts exceed token limits */ @@ -252,11 +343,15 @@ describe("OpenAICompatibleEmbedder", () => { const testTexts = ["Hello world"] const rateLimitError = { status: 429, message: "Rate limit exceeded" } + // Create base64 encoded embedding for successful response + const testEmbedding = new Float32Array([0.25, 0.5, 0.75]) + const base64String = Buffer.from(testEmbedding.buffer).toString("base64") + mockEmbeddingsCreate .mockRejectedValueOnce(rateLimitError) .mockRejectedValueOnce(rateLimitError) .mockResolvedValueOnce({ - data: [{ embedding: [0.1, 0.2, 0.3] }], + data: [{ embedding: base64String }], usage: { prompt_tokens: 10, total_tokens: 15 }, }) @@ -271,7 +366,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(3) expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("Rate limit hit, retrying in")) expect(result).toEqual({ - embeddings: [[0.1, 0.2, 0.3]], + embeddings: [[0.25, 0.5, 0.75]], usage: { promptTokens: 10, totalTokens: 15 }, }) }) From c98a5df1c51a82935551261784b064e629d11370 Mon Sep 17 00:00:00 2001 From: Dixie Flatline Date: Thu, 12 Jun 2025 03:51:13 +0000 Subject: [PATCH 08/10] Add tests to verify openai base64 and brokenness behavior --- .../__tests__/openai-compatible.spec.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index 508505a027..352973a1f2 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -458,5 +458,84 @@ describe("OpenAICompatibleEmbedder", () => { await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow() }) }) + + /** + * Test to confirm OpenAI package bug with base64 encoding + * This test verifies that when we request encoding_format: "base64", + * the OpenAI package returns unparsed base64 strings as expected. + * This is the behavior we rely on in our workaround. + */ + describe("OpenAI package base64 behavior verification", () => { + it("should return unparsed base64 when encoding_format is base64", async () => { + const testTexts = ["Hello world"] + + // Create a real OpenAI instance to test the actual package behavior + const realOpenAI = new (jest.requireActual("openai").OpenAI)({ + baseURL: testBaseUrl, + apiKey: testApiKey, + }) + + // Create test embedding data as base64 using values that can be exactly represented in Float32 + const testEmbedding = new Float32Array([0.25, 0.5, 0.75, 1.0]) + const buffer = Buffer.from(testEmbedding.buffer) + const base64String = buffer.toString("base64") + + // Mock the raw API response that would come from OpenAI + const mockApiResponse = { + data: [ + { + object: "embedding", + embedding: base64String, // Raw base64 string from API + index: 0, + }, + ], + model: "text-embedding-3-small", + object: "list", + usage: { + prompt_tokens: 2, + total_tokens: 2, + }, + } + + // Mock the methodRequest method which is called by post() + const mockMethodRequest = jest.fn() + const mockAPIPromise = { + then: jest.fn().mockImplementation((callback) => { + return Promise.resolve(callback(mockApiResponse)) + }), + catch: jest.fn(), + finally: jest.fn(), + } + mockMethodRequest.mockReturnValue(mockAPIPromise) + + // Replace the methodRequest method on the client + ;(realOpenAI as any).post = jest.fn().mockImplementation((path, opts) => { + return mockMethodRequest("post", path, opts) + }) + + // Call the embeddings.create method with base64 encoding + const response = await realOpenAI.embeddings.create({ + input: testTexts, + model: "text-embedding-3-small", + encoding_format: "base64", + }) + + // Verify that the response contains the raw base64 string + // This confirms the OpenAI package doesn't parse base64 when explicitly requested + expect(response.data[0].embedding).toBe(base64String) + expect(typeof response.data[0].embedding).toBe("string") + + // Verify we can manually convert it back to the original float array + const returnedBuffer = Buffer.from(response.data[0].embedding as string, "base64") + const returnedFloat32Array = new Float32Array( + returnedBuffer.buffer, + returnedBuffer.byteOffset, + returnedBuffer.byteLength / 4, + ) + const returnedArray = Array.from(returnedFloat32Array) + + expect(returnedArray).toEqual([0.25, 0.5, 0.75, 1.0]) + }) + }) }) }) From 17401b2398767c5143aaad0a457a2310f7bc3c6c Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Thu, 12 Jun 2025 08:34:42 -0500 Subject: [PATCH 09/10] feat: improve typing --- .../code-index/embedders/openai-compatible.ts | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 2f0051879d..b7e4079569 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -8,6 +8,19 @@ import { } from "../constants" import { getDefaultModelId } from "../../../shared/embeddingModels" +interface EmbeddingItem { + embedding: string | number[] + [key: string]: any +} + +interface OpenAIEmbeddingResponse { + data: EmbeddingItem[] + usage?: { + prompt_tokens?: number + total_tokens?: number + } +} + /** * OpenAI Compatible implementation of the embedder interface with batching and rate limiting. * This embedder allows using any OpenAI-compatible API endpoint by specifying a custom baseURL. @@ -108,21 +121,23 @@ export class OpenAICompatibleEmbedder implements IEmbedder { ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { try { - const response = await this.embeddingsClient.embeddings.create({ + const response = (await this.embeddingsClient.embeddings.create({ input: batchTexts, model: model, - // The OpenAI package has custom parsing that truncates embedding dimension to 256, - // which destroys accuracy. - // If we pass `encoding_format: "base64"`, it does not perform any parsing, - // leaving parsing up to us. + // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 + // when processing numeric arrays, which breaks compatibility with models using larger dimensions. + // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. encoding_format: "base64", - }) + })) as OpenAIEmbeddingResponse // Convert base64 embeddings to float32 arrays - const processedEmbeddings = response.data.map((item: any) => { + const processedEmbeddings = response.data.map((item: EmbeddingItem) => { if (typeof item.embedding === "string") { const buffer = Buffer.from(item.embedding, "base64") + + // Create Float32Array view over the buffer const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + return { ...item, embedding: Array.from(float32Array), @@ -134,7 +149,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { // Replace the original data with processed embeddings response.data = processedEmbeddings - const embeddings = response.data.map((item) => item.embedding) + const embeddings = response.data.map((item) => item.embedding as number[]) return { embeddings: embeddings, From b029b40bc90d12a8c6e46c90dabf8c28f0c84f25 Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Thu, 12 Jun 2025 11:58:04 -0500 Subject: [PATCH 10/10] refactor: switch from jest to vitest for mocking in tests --- .../embedders/__tests__/openai-compatible.spec.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index 352973a1f2..5c7c44a634 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -1,4 +1,4 @@ -import { vitest, describe, it, expect, beforeEach, afterEach } from "vitest" +import { vitest, describe, it, expect, beforeEach, afterEach, vi } from "vitest" import type { MockedClass, MockedFunction } from "vitest" import { OpenAI } from "openai" import { OpenAICompatibleEmbedder } from "../openai-compatible" @@ -470,7 +470,7 @@ describe("OpenAICompatibleEmbedder", () => { const testTexts = ["Hello world"] // Create a real OpenAI instance to test the actual package behavior - const realOpenAI = new (jest.requireActual("openai").OpenAI)({ + const realOpenAI = new ((await vi.importActual("openai")) as any).OpenAI({ baseURL: testBaseUrl, apiKey: testApiKey, }) @@ -498,18 +498,18 @@ describe("OpenAICompatibleEmbedder", () => { } // Mock the methodRequest method which is called by post() - const mockMethodRequest = jest.fn() + const mockMethodRequest = vi.fn() const mockAPIPromise = { - then: jest.fn().mockImplementation((callback) => { + then: vi.fn().mockImplementation((callback) => { return Promise.resolve(callback(mockApiResponse)) }), - catch: jest.fn(), - finally: jest.fn(), + catch: vi.fn(), + finally: vi.fn(), } mockMethodRequest.mockReturnValue(mockAPIPromise) // Replace the methodRequest method on the client - ;(realOpenAI as any).post = jest.fn().mockImplementation((path, opts) => { + ;(realOpenAI as any).post = vi.fn().mockImplementation((path, opts) => { return mockMethodRequest("post", path, opts) })