diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index fbc4a24118..0029dce128 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -127,7 +127,14 @@ export class CodeIndexOrchestrator { const collectionCreated = await this.vectorStore.initialize() if (collectionCreated) { + console.log( + "[CodeIndexOrchestrator] New collection was created, clearing cache file to force full reindex", + ) await this.cacheManager.clearCacheFile() + } else { + console.log( + "[CodeIndexOrchestrator] Using existing collection, preserving cache for incremental updates", + ) } this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index 822832d17c..c14c59b0f6 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -32,6 +32,7 @@ vitest.mock("path", async () => { const mockQdrantClientInstance = { getCollection: vitest.fn(), + getCollections: vitest.fn(), // Add this for testConnection method createCollection: vitest.fn(), deleteCollection: vitest.fn(), createPayloadIndex: vitest.fn(), @@ -514,6 +515,8 @@ describe("QdrantVectorStore", () => { describe("initialize", () => { it("should create a new collection if none exists and return true", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) // Mock getCollection to throw a 404-like error mockQdrantClientInstance.getCollection.mockRejectedValue({ response: { status: 404 }, @@ -546,6 +549,8 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) }) it("should not create a new collection if one exists with matching vectorSize and return false", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) // Mock getCollection to return existing collection info with matching vector size mockQdrantClientInstance.getCollection.mockResolvedValue({ config: { @@ -577,6 +582,8 @@ describe("QdrantVectorStore", () => { }) it("should recreate collection if it exists but vectorSize mismatches and return true", async () => { const differentVectorSize = 768 + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) // Mock getCollection to return existing collection info with different vector size first, // then return 404 to confirm deletion mockQdrantClientInstance.getCollection @@ -597,6 +604,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) mockQdrantClientInstance.createPayloadIndex.mockResolvedValue({} as any) vitest.spyOn(console, "warn").mockImplementation(() => {}) // Suppress console.warn + vitest.spyOn(console, "log").mockImplementation(() => {}) // Suppress console.log const result = await vectorStore.initialize() @@ -622,11 +630,15 @@ describe("QdrantVectorStore", () => { } expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) ;(console.warn as any).mockRestore() // Restore console.warn + ;(console.log as any).mockRestore() // Restore console.log }) it("should log warning for non-404 errors but still create collection", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const genericError = new Error("Generic Qdrant Error") mockQdrantClientInstance.getCollection.mockRejectedValue(genericError) vitest.spyOn(console, "warn").mockImplementation(() => {}) // Suppress console.warn + vitest.spyOn(console, "log").mockImplementation(() => {}) // Suppress console.log const result = await vectorStore.initialize() @@ -635,11 +647,12 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled() expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Warning during getCollectionInfo for "${expectedCollectionName}"`), - genericError.message, + // Check that the collection creation was logged + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining(`Creating new collection "${expectedCollectionName}"`), ) ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should re-throw error from createCollection when no collection initially exists", async () => { mockQdrantClientInstance.getCollection.mockRejectedValue({ @@ -663,6 +676,8 @@ describe("QdrantVectorStore", () => { ;(console.error as any).mockRestore() }) it("should log but not fail if payload index creation errors occur", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) // Mock successful collection creation mockQdrantClientInstance.getCollection.mockRejectedValue({ response: { status: 404 }, @@ -674,6 +689,7 @@ describe("QdrantVectorStore", () => { const indexError = new Error("Index creation failed") mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(indexError) vitest.spyOn(console, "warn").mockImplementation(() => {}) // Suppress console.warn + vitest.spyOn(console, "log").mockImplementation(() => {}) // Suppress console.log const result = await vectorStore.initialize() @@ -685,18 +701,19 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) // Verify warnings were logged for each failed index - expect(console.warn).toHaveBeenCalledTimes(5) - for (let i = 0; i <= 4; i++) { - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Could not create payload index for pathSegments.${i}`), - indexError.message, - ) - } - + // Note: There might be additional warnings from the new logging + expect(console.warn).toHaveBeenCalled() + // Check that warnings were logged for failed indexes + const warnCalls = (console.warn as any).mock.calls + const indexWarnings = warnCalls.filter((call: any[]) => call[0]?.includes("Could not create payload index")) + expect(indexWarnings).toHaveLength(5) ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should throw vectorDimensionMismatch error when deleteCollection fails during recreation", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const differentVectorSize = 768 mockQdrantClientInstance.getCollection.mockResolvedValue({ config: { @@ -712,6 +729,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.deleteCollection.mockRejectedValue(deleteError) vitest.spyOn(console, "error").mockImplementation(() => {}) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) // The error should have a cause property set to the original error let caughtError: any @@ -734,9 +752,12 @@ describe("QdrantVectorStore", () => { expect(console.error).toHaveBeenCalledTimes(2) // One for the critical error, one for the outer catch ;(console.error as any).mockRestore() ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should throw vectorDimensionMismatch error when createCollection fails during recreation", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const differentVectorSize = 768 mockQdrantClientInstance.getCollection .mockResolvedValueOnce({ @@ -760,6 +781,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.createCollection.mockRejectedValue(createError) vitest.spyOn(console, "error").mockImplementation(() => {}) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) // Should throw an error with cause property set to the original error let caughtError: any @@ -777,14 +799,18 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.deleteCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createPayloadIndex).not.toHaveBeenCalled() - // Should log warning, critical error, and outer error - expect(console.warn).toHaveBeenCalledTimes(1) + // Should log warnings and errors + expect(console.warn).toHaveBeenCalled() + expect(console.log).toHaveBeenCalled() expect(console.error).toHaveBeenCalledTimes(2) ;(console.error as any).mockRestore() ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should verify collection deletion before proceeding with recreation", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const differentVectorSize = 768 mockQdrantClientInstance.getCollection .mockResolvedValueOnce({ @@ -806,6 +832,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) mockQdrantClientInstance.createPayloadIndex.mockResolvedValue({} as any) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) const result = await vectorStore.initialize() @@ -816,9 +843,12 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should throw error if collection still exists after deletion attempt", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const differentVectorSize = 768 mockQdrantClientInstance.getCollection .mockResolvedValueOnce({ @@ -844,6 +874,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.deleteCollection.mockResolvedValue(true as any) vitest.spyOn(console, "error").mockImplementation(() => {}) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) let caughtError: any try { @@ -863,9 +894,12 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).not.toHaveBeenCalled() ;(console.error as any).mockRestore() ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should handle dimension mismatch scenario from 2048 to 768 dimensions", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) // Simulate the exact scenario from the issue: switching from 2048 to 768 dimensions const oldVectorSize = 2048 const newVectorSize = 768 @@ -893,6 +927,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) mockQdrantClientInstance.createPayloadIndex.mockResolvedValue({} as any) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) const result = await newVectorStore.initialize() @@ -907,9 +942,12 @@ describe("QdrantVectorStore", () => { }) expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) it("should provide detailed error context for different failure scenarios", async () => { + // Mock getCollections for testConnection + mockQdrantClientInstance.getCollections.mockResolvedValue({ collections: [] }) const differentVectorSize = 768 mockQdrantClientInstance.getCollection.mockResolvedValue({ config: { @@ -926,6 +964,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.deleteCollection.mockRejectedValue(deleteError) vitest.spyOn(console, "error").mockImplementation(() => {}) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "log").mockImplementation(() => {}) let caughtError: any try { @@ -942,6 +981,7 @@ describe("QdrantVectorStore", () => { expect(caughtError.cause).toBe(deleteError) ;(console.error as any).mockRestore() ;(console.warn as any).mockRestore() + ;(console.log as any).mockRestore() }) }) @@ -976,16 +1016,22 @@ describe("QdrantVectorStore", () => { const genericError = new Error("Network error") mockQdrantClientInstance.getCollection.mockRejectedValue(genericError) vitest.spyOn(console, "warn").mockImplementation(() => {}) + vitest.spyOn(console, "error").mockImplementation(() => {}) const result = await vectorStore.collectionExists() expect(result).toBe(false) - expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(1) - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Warning during getCollectionInfo for "${expectedCollectionName}"`), + // 4 calls: 1 initial + 3 retries + expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(4) + // The warning is now an error after retries fail + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining( + `Failed to connect to Qdrant after 3 retries for collection "${expectedCollectionName}"`, + ), genericError.message, ) ;(console.warn as any).mockRestore() + ;(console.error as any).mockRestore() }) describe("deleteCollection", () => { it("should delete collection when it exists", async () => { diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 50f39666c4..ebf6dafa5b 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -125,21 +125,75 @@ export class QdrantVectorStore implements IVectorStore { } } - private async getCollectionInfo(): Promise { + private async getCollectionInfo(retryCount: number = 0): Promise { + const maxRetries = 3 + const retryDelay = 1000 * Math.pow(2, retryCount) // Exponential backoff: 1s, 2s, 4s + try { const collectionInfo = await this.client.getCollection(this.collectionName) return collectionInfo } catch (error: unknown) { - if (error instanceof Error) { + const errorMessage = error instanceof Error ? error.message : String(error) + const isConnectionError = this.isConnectionError(errorMessage) + + // If it's a connection error and we haven't exceeded retries, retry + if (isConnectionError && retryCount < maxRetries) { console.warn( - `[QdrantVectorStore] Warning during getCollectionInfo for "${this.collectionName}". Collection may not exist or another error occurred:`, - error.message, + `[QdrantVectorStore] Connection error during getCollectionInfo for "${this.collectionName}". Retrying in ${retryDelay}ms... (attempt ${retryCount + 1}/${maxRetries})`, + errorMessage, ) + await new Promise((resolve) => setTimeout(resolve, retryDelay)) + return this.getCollectionInfo(retryCount + 1) } + + // Log different messages based on error type + if (isConnectionError) { + console.error( + `[QdrantVectorStore] Failed to connect to Qdrant after ${maxRetries} retries for collection "${this.collectionName}":`, + errorMessage, + ) + } else if (this.isNotFoundError(errorMessage)) { + // Collection doesn't exist - this is expected for first-time setup + console.log(`[QdrantVectorStore] Collection "${this.collectionName}" does not exist. Will create it.`) + } else { + console.warn( + `[QdrantVectorStore] Unexpected error during getCollectionInfo for "${this.collectionName}":`, + errorMessage, + ) + } + return null } } + /** + * Checks if an error message indicates a connection failure + */ + private isConnectionError(errorMessage: string): boolean { + const connectionErrorPatterns = [ + "ECONNREFUSED", + "ETIMEDOUT", + "ENOTFOUND", + "ENETUNREACH", + "EHOSTUNREACH", + "ECONNRESET", + "fetch failed", + "network", + "connect", + ] + const lowerMessage = errorMessage.toLowerCase() + return connectionErrorPatterns.some((pattern) => lowerMessage.includes(pattern.toLowerCase())) + } + + /** + * Checks if an error message indicates the collection was not found + */ + private isNotFoundError(errorMessage: string): boolean { + const notFoundPatterns = ["404", "not found", "doesn't exist", "does not exist"] + const lowerMessage = errorMessage.toLowerCase() + return notFoundPatterns.some((pattern) => lowerMessage.includes(pattern.toLowerCase())) + } + /** * Initializes the vector store * @returns Promise resolving to boolean indicating if a new collection was created @@ -147,10 +201,17 @@ export class QdrantVectorStore implements IVectorStore { async initialize(): Promise { let created = false try { + // Add initial connection test with retry + await this.testConnection() + const collectionInfo = await this.getCollectionInfo() if (collectionInfo === null) { - // Collection info not retrieved (assume not found or inaccessible), create it + // Only create collection if we're sure it doesn't exist (not just a connection issue) + // The getCollectionInfo method now handles retries for connection issues + console.log( + `[QdrantVectorStore] Creating new collection "${this.collectionName}" with vector size ${this.vectorSize}`, + ) await this.client.createCollection(this.collectionName, { vectors: { size: this.vectorSize, @@ -158,7 +219,9 @@ export class QdrantVectorStore implements IVectorStore { }, }) created = true + console.log(`[QdrantVectorStore] Successfully created collection "${this.collectionName}"`) } else { + console.log(`[QdrantVectorStore] Collection "${this.collectionName}" already exists`) // Collection exists, check vector size const vectorsConfig = collectionInfo.config?.params?.vectors let existingVectorSize: number @@ -206,6 +269,41 @@ export class QdrantVectorStore implements IVectorStore { } } + /** + * Tests the connection to Qdrant with retries + */ + private async testConnection(): Promise { + const maxRetries = 3 + let lastError: Error | null = null + + for (let i = 0; i < maxRetries; i++) { + try { + // Try to get collections list as a connection test + await this.client.getCollections() + console.log(`[QdrantVectorStore] Successfully connected to Qdrant at ${this.qdrantUrl}`) + return + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + const retryDelay = 1000 * Math.pow(2, i) // Exponential backoff + console.warn( + `[QdrantVectorStore] Failed to connect to Qdrant (attempt ${i + 1}/${maxRetries}). Retrying in ${retryDelay}ms...`, + lastError.message, + ) + if (i < maxRetries - 1) { + await new Promise((resolve) => setTimeout(resolve, retryDelay)) + } + } + } + + // If we get here, all retries failed + throw new Error( + t("embeddings:vectorStore.qdrantConnectionFailed", { + qdrantUrl: this.qdrantUrl, + errorMessage: lastError?.message || "Connection failed after multiple retries", + }), + ) + } + /** * Recreates the collection with a new vector dimension, handling failures gracefully. * @param existingVectorSize The current vector size of the existing collection