Remove old embedding disk cache support (#1188)

With the new embeddings model, we should never use the old caches so there's no need to migrate them over

Author: mjbvz

src/platform/workspaceChunkSearch/node/workspaceChunkAndEmbeddingCache.ts

@@ -11,31 +11,14 @@ import { CancellationToken } from '../../../util/vs/base/common/cancellation';
 import { ResourceMap } from '../../../util/vs/base/common/map';
 import { Schemas } from '../../../util/vs/base/common/network';
 import { URI } from '../../../util/vs/base/common/uri';
-import { IRange, Range } from '../../../util/vs/editor/common/core/range';
+import { Range } from '../../../util/vs/editor/common/core/range';
 import { IInstantiationService, ServicesAccessor } from '../../../util/vs/platform/instantiation/common/instantiation';
 import { FileChunkWithEmbedding } from '../../chunking/common/chunk';
-import { Embedding, EmbeddingType, EmbeddingVector, getWellKnownEmbeddingTypeInfo } from '../../embeddings/common/embeddingsComputer';
+import { Embedding, EmbeddingType, getWellKnownEmbeddingTypeInfo } from '../../embeddings/common/embeddingsComputer';
 import { IFileSystemService } from '../../filesystem/common/fileSystemService';
 import { ILogService } from '../../log/common/logService';
 import { FileRepresentation, IWorkspaceFileIndex } from './workspaceFileIndex';
 
-interface PersistedCache {
-	readonly version: string;
-	readonly embeddingModel: string | undefined;
-	readonly entries: Record<string, PersistedCacheEntry>;
-}
-
-interface PersistedCacheEntry {
-	readonly contentVersionId: string | undefined;
-	readonly hash: string | undefined;
-	readonly entries: readonly {
-		readonly text: string;
-		readonly range: IRange;
-		readonly embedding: EmbeddingVector | /* base64*/ string;
-		readonly chunkHash: string | undefined;
-	}[];
-}
-
 type CacheEntry = {
 	readonly contentVersionId: string | undefined;
 	readonly fileHash: string | undefined;
@@ -80,49 +63,8 @@ export async function createWorkspaceChunkAndEmbeddingCache(
 }
 
 class OldDiskCache {
-	private static readonly version = '1.0.0';
 	private static cacheFileName = 'workspace-chunks.json';
 
-	public static decodeEmbedding(base64Str: string): EmbeddingVector {
-		const decoded = Buffer.from(base64Str, 'base64');
-		const float32Array = new Float32Array(decoded.buffer, decoded.byteOffset, decoded.byteLength / Float32Array.BYTES_PER_ELEMENT);
-		return Array.from(float32Array);
-	}
-
-	public static async readDiskCache(accessor: ServicesAccessor, embeddingType: EmbeddingType, cacheRoot: URI, logService: ILogService): Promise<Iterable<[string, PersistedCacheEntry]> | undefined> {
-		const fileSystem = accessor.get(IFileSystemService);
-
-		const cachePath = URI.joinPath(cacheRoot, OldDiskCache.cacheFileName);
-		try {
-			let file: Uint8Array | undefined;
-			try {
-				file = await fileSystem.readFile(cachePath, true);
-			} catch (e) {
-				// Expected, most likely file doesn't exist
-				return undefined;
-			}
-
-			const data: PersistedCache = JSON.parse(new TextDecoder().decode(file));
-			if (data.version !== OldDiskCache.version) {
-				logService.debug(`WorkspaceChunkAndEmbeddingCache: invalidating cache due to version mismatch. Expected ${OldDiskCache.version} but found ${data.version}`);
-				return undefined;
-			}
-
-			// Check mismatch in embedding models
-			// Older cached version don't store their embedding model but it's always text3small_512
-			if (
-				(data.embeddingModel === undefined && embeddingType !== EmbeddingType.text3small_512)
-				|| (data.embeddingModel !== undefined && data.embeddingModel !== embeddingType.id)
-			) {
-				logService.debug(`WorkspaceChunkAndEmbeddingCache: invalidating cache due to embeddings type mismatch. Expected ${embeddingType} but found ${data.embeddingModel}`);
-				return undefined;
-			}
-
-			return Object.entries(data.entries);
-		} catch {
-			return undefined;
-		}
-	}
 
 	static async deleteDiskCache(accessor: ServicesAccessor, cacheRoot: URI) {
 		const fileSystem = accessor.get(IFileSystemService);
@@ -221,48 +163,9 @@ class DbCache implements IWorkspaceChunkAndEmbeddingCache {
 		db.exec('DELETE FROM CacheMeta;');
 		db.prepare('INSERT INTO CacheMeta (version, embeddingModel) VALUES (?, ?)').run(this.version, embeddingType.id);
 
-		// Load existing disk db if it exists
-		const diskCache = cacheRoot !== ':memory:' ?
-			await instantiationService.invokeFunction(accessor => OldDiskCache.readDiskCache(
-				accessor,
-				embeddingType,
-				cacheRoot,
-				accessor.get(ILogService)
-			))
-			: undefined;
-		if (diskCache) {
-			try {
-				const insertFileStatement = db.prepare('INSERT OR REPLACE INTO Files (uri, contentVersionId) VALUES (?, ?)');
-				const insertChunkStatement = db.prepare(`INSERT INTO FileChunks (fileId, text, range_startLineNumber, range_startColumn, range_endLineNumber, range_endColumn, embedding, chunkHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
-
-				db.exec('BEGIN TRANSACTION');
-				for (const [uri, entry] of diskCache) {
-					const fileIdResult = insertFileStatement
-						.run(uri.toString(), entry.contentVersionId ?? '');
-
-					for (const chunk of entry.entries) {
-						insertChunkStatement.run(
-							fileIdResult.lastInsertRowid as number,
-							chunk.text,
-							chunk.range.startLineNumber,
-							chunk.range.startColumn,
-							chunk.range.endLineNumber,
-							chunk.range.endColumn,
-							packEmbedding({
-								type: embeddingType,
-								value: typeof chunk.embedding === 'string' ? OldDiskCache.decodeEmbedding(chunk.embedding) : chunk.embedding,
-							}),
-							chunk.chunkHash ?? ''
-						);
-					}
-				}
-			} finally {
-				db.exec('COMMIT');
-			}
-
-			if (cacheRoot !== ':memory:') {
-				void instantiationService.invokeFunction(accessor => OldDiskCache.deleteDiskCache(accessor, cacheRoot));
-			}
+		// Clean up old disk db if it exists
+		if (cacheRoot !== ':memory:') {
+			void instantiationService.invokeFunction(accessor => OldDiskCache.deleteDiskCache(accessor, cacheRoot));
 		}
 
 		// Validate all files in the database against the workspace index and remove any that are no longer present