diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 24d3e7dbba..d13e516a87 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -325,10 +325,24 @@ export class DirectoryScanner implements IDirectoryScanner { success = true } catch (error) { lastError = error as Error - console.error(`[DirectoryScanner] Error processing batch (attempt ${attempts}):`, error) + const errorMessage = error instanceof Error ? error.message : String(error) + const batchInfo = { + batchSize: batchBlocks.length, + attempt: attempts, + maxRetries: MAX_BATCH_RETRIES, + fileCount: batchFileInfos.length, + sampleFiles: batchFileInfos.slice(0, 3).map((info) => info.filePath), + } + + console.error(`[DirectoryScanner] Error processing batch (attempt ${attempts}/${MAX_BATCH_RETRIES}):`, { + error: errorMessage, + batchInfo, + originalError: error, + }) if (attempts < MAX_BATCH_RETRIES) { const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1) + console.log(`[DirectoryScanner] Retrying batch in ${delay}ms...`) await new Promise((resolve) => setTimeout(resolve, delay)) } } diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index ccd1b619ad..d2d886f993 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -385,10 +385,14 @@ describe("QdrantVectorStore", () => { describe("upsertPoints", () => { it("should correctly call qdrantClient.upsert with processed points", async () => { + // Create vectors with correct dimensions (1536) + const mockVector1 = new Array(1536).fill(0).map((_, i) => i / 1536) + const mockVector2 = new Array(1536).fill(0).map((_, i) => (i + 100) / 1536) + const mockPoints = [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector1, payload: { filePath: "src/components/Button.tsx", content: "export const Button = () => {}", @@ -398,7 +402,7 @@ describe("QdrantVectorStore", () => { }, { id: "test-id-2", - vector: [0.4, 0.5, 0.6], + vector: mockVector2, payload: { filePath: "src/utils/helpers.ts", content: "export function helper() {}", @@ -417,7 +421,7 @@ describe("QdrantVectorStore", () => { points: [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector1, payload: { filePath: "src/components/Button.tsx", content: "export const Button = () => {}", @@ -432,7 +436,7 @@ describe("QdrantVectorStore", () => { }, { id: "test-id-2", - vector: [0.4, 0.5, 0.6], + vector: mockVector2, payload: { filePath: "src/utils/helpers.ts", content: "export function helper() {}", @@ -451,10 +455,11 @@ describe("QdrantVectorStore", () => { }) it("should handle points without filePath in payload", async () => { + const mockVector = new Array(1536).fill(0).map((_, i) => i / 1536) const mockPoints = [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector, payload: { content: "some content without filePath", startLine: 1, @@ -471,7 +476,7 @@ describe("QdrantVectorStore", () => { points: [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector, payload: { content: "some content without filePath", startLine: 1, @@ -488,17 +493,16 @@ describe("QdrantVectorStore", () => { await vectorStore.upsertPoints([]) - expect(mockQdrantClientInstance.upsert).toHaveBeenCalledWith(expectedCollectionName, { - points: [], - wait: true, - }) + // Should not call upsert for empty arrays (early return) + expect(mockQdrantClientInstance.upsert).not.toHaveBeenCalled() }) it("should correctly process pathSegments for nested file paths", async () => { + const mockVector = new Array(1536).fill(0).map((_, i) => i / 1536) const mockPoints = [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector, payload: { filePath: "src/components/ui/forms/InputField.tsx", content: "export const InputField = () => {}", @@ -516,7 +520,7 @@ describe("QdrantVectorStore", () => { points: [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector, payload: { filePath: "src/components/ui/forms/InputField.tsx", content: "export const InputField = () => {}", @@ -537,10 +541,11 @@ describe("QdrantVectorStore", () => { }) it("should handle error scenarios when qdrantClient.upsert fails", async () => { + const mockVector = new Array(1536).fill(0).map((_, i) => i / 1536) const mockPoints = [ { id: "test-id-1", - vector: [0.1, 0.2, 0.3], + vector: mockVector, payload: { filePath: "src/test.ts", content: "test content", @@ -554,10 +559,19 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.upsert.mockRejectedValue(upsertError) vitest.spyOn(console, "error").mockImplementation(() => {}) - await expect(vectorStore.upsertPoints(mockPoints)).rejects.toThrow(upsertError) + await expect(vectorStore.upsertPoints(mockPoints)).rejects.toThrow( + "Failed to upsert 1 points to collection ws-a1b2c3d4e5f6g7h8: Upsert failed", + ) expect(mockQdrantClientInstance.upsert).toHaveBeenCalledTimes(1) - expect(console.error).toHaveBeenCalledWith("Failed to upsert points:", upsertError) + expect(console.error).toHaveBeenCalledWith( + "Failed to upsert points:", + expect.objectContaining({ + message: "Upsert failed", + pointsCount: 1, + collectionName: "ws-a1b2c3d4e5f6g7h8", + }), + ) ;(console.error as any).mockRestore() }) }) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index b6f1725b55..10f4007742 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -118,6 +118,32 @@ export class QdrantVectorStore implements IVectorStore { } } + /** + * Validates vector dimensions against the expected collection vector size + * @param vectors Array of vectors to validate + * @returns true if all vectors have correct dimensions + */ + private validateVectorDimensions(vectors: number[][]): void { + for (let i = 0; i < vectors.length; i++) { + const vector = vectors[i] + if (!Array.isArray(vector)) { + throw new Error(`Vector at index ${i} is not an array`) + } + if (vector.length !== this.vectorSize) { + throw new Error( + `Vector at index ${i} has incorrect dimensions: expected ${this.vectorSize}, got ${vector.length}`, + ) + } + // Check for invalid values (NaN, Infinity) + for (let j = 0; j < vector.length; j++) { + const value = vector[j] + if (typeof value !== "number" || !isFinite(value)) { + throw new Error(`Vector at index ${i} contains invalid value at position ${j}: ${value}`) + } + } + } + } + /** * Upserts points into the vector store * @param points Array of points to upsert @@ -130,6 +156,34 @@ export class QdrantVectorStore implements IVectorStore { }>, ): Promise { try { + // Validate input points + if (!Array.isArray(points)) { + throw new Error("Points must be an array") + } + + if (points.length === 0) { + return // Nothing to upsert + } + + // Validate each point structure and collect vectors for dimension validation + const vectors: number[][] = [] + for (let i = 0; i < points.length; i++) { + const point = points[i] + if (!point.id || typeof point.id !== "string") { + throw new Error(`Point at index ${i} must have a valid string id`) + } + if (!Array.isArray(point.vector) || point.vector.length === 0) { + throw new Error(`Point at index ${i} must have a valid vector array`) + } + if (!point.payload || typeof point.payload !== "object") { + throw new Error(`Point at index ${i} must have a valid payload object`) + } + vectors.push(point.vector) + } + + // Validate vector dimensions + this.validateVectorDimensions(vectors) + const processedPoints = points.map((point) => { if (point.payload?.filePath) { const segments = point.payload.filePath.split(path.sep).filter(Boolean) @@ -141,23 +195,50 @@ export class QdrantVectorStore implements IVectorStore { {}, ) return { - ...point, + id: point.id, + vector: point.vector, payload: { ...point.payload, pathSegments, }, } } - return point + return { + id: point.id, + vector: point.vector, + payload: point.payload, + } }) - await this.client.upsert(this.collectionName, { + // Use the batch upsert operation with proper error handling + const upsertRequest = { points: processedPoints, wait: true, - }) - } catch (error) { - console.error("Failed to upsert points:", error) - throw error + } + + await this.client.upsert(this.collectionName, upsertRequest) + } catch (error: any) { + // Enhanced error logging to help debug the "Bad Request" issue + const errorMessage = error?.message || error?.toString() || "Unknown error" + const errorDetails = { + message: errorMessage, + status: error?.status || error?.response?.status, + statusText: error?.statusText || error?.response?.statusText, + data: error?.data || error?.response?.data, + pointsCount: points.length, + collectionName: this.collectionName, + vectorSize: this.vectorSize, + sampleVectorLengths: points.slice(0, 3).map((p) => p.vector?.length || "undefined"), + } + + console.error("Failed to upsert points:", errorDetails) + + // Re-throw with more context + const enhancedError = new Error( + `Failed to upsert ${points.length} points to collection ${this.collectionName}: ${errorMessage}`, + ) + enhancedError.cause = error + throw enhancedError } }