add delete_index tool, CRLF normalization, non-UTF-8 detection, and UNC path validation

Adds a delete_index tool (gated by DISABLE_DESTRUCTIVE env var) to remove
directory indexes, normalizes CRLF line endings before chunking for consistent
results, skips non-UTF-8 files during indexing with a warning, and validates
Windows UNC path format in path validation.

Author: peteretelej

.gitignore

@@ -18,6 +18,8 @@ yarn-error.log*
 *.db
 *.sqlite
 *.sqlite3
+*.sqlite-wal
+*.sqlite-shm
 
 # Configuration (except examples)
 config.json

src/indexing.ts

@@ -14,6 +14,7 @@ import {
 import { loadGitignoreRules } from './gitignore.js';
 import { generateEmbedding } from './embedding.js';
 import { initializeStorage, FileRecord } from './storage.js';
+import { log } from './logger.js';
 
 export interface ScanOptions {
   ignorePatterns: string[];
@@ -251,7 +252,18 @@ export async function indexDirectories(paths: string[], config: Config): Promise
             await qdrant.deletePointsByFilePath(file.path);
           }
 
-          const content = await fs.readFile(file.path, 'utf-8');
+          const rawContent = await fs.readFile(file.path, 'utf-8');
+
+          // Skip non-UTF-8 files (Node.js inserts U+FFFD for invalid byte sequences)
+          if (rawContent.includes('\uFFFD')) {
+            log('warning', 'Skipping non-UTF-8 file', { path: file.path });
+            await sqlite.upsertFile(file, [], ['Skipped: file appears to be non-UTF-8 encoded']);
+            skipped++;
+            continue;
+          }
+
+          // Normalize CRLF to LF for consistent chunk boundaries
+          const content = rawContent.replace(/\r\n/g, '\n');
           const chunks = chunkText(content, config.indexing.chunkSize, config.indexing.chunkOverlap);
 
           // Store file metadata in SQLite

src/mcp-handlers.ts

@@ -74,6 +74,10 @@ interface GetChunkToolArgs {
   chunk_id: string;
 }
 
+interface DeleteIndexToolArgs {
+  directory_path: string;
+}
+
 // Type guard functions
 function isIndexToolArgs(args: unknown): args is IndexToolArgs {
   return typeof args === 'object' && args !== null &&
@@ -96,11 +100,16 @@ function isGetContentToolArgs(args: unknown): args is GetContentToolArgs {
 }
 
 function isGetChunkToolArgs(args: unknown): args is GetChunkToolArgs {
-  return typeof args === 'object' && args !== null && 
+  return typeof args === 'object' && args !== null &&
          typeof (args as GetChunkToolArgs).file_path === 'string' &&
          typeof (args as GetChunkToolArgs).chunk_id === 'string';
 }
 
+function isDeleteIndexToolArgs(args: unknown): args is DeleteIndexToolArgs {
+  return typeof args === 'object' && args !== null &&
+         typeof (args as DeleteIndexToolArgs).directory_path === 'string';
+}
+
 export async function handleIndexTool(args: unknown, config: Config): Promise<CallToolResult> {
   if (!isIndexToolArgs(args)) {
     throw new Error('directory_paths is required and must be an array');
@@ -319,6 +328,61 @@ export async function handleServerInfoTool(version: string): Promise<CallToolRes
   };
 }
 
+export async function handleDeleteIndexTool(args: unknown, config: Config): Promise<CallToolResult> {
+  if (!isDeleteIndexToolArgs(args)) {
+    throw new Error('directory_path is required');
+  }
+
+  const dirPath = args.directory_path.trim();
+  const { sqlite, qdrant } = await initializeStorage(config);
+
+  try {
+    // Check if the directory is actually indexed
+    const directory = await sqlite.getDirectory(dirPath);
+    if (!directory) {
+      throw new Error(
+        `Directory '${dirPath}' is not indexed. Use 'server_info' to see indexed directories.`
+      );
+    }
+
+    // Get files for this directory to clean up Qdrant points
+    const files = await sqlite.getFilesByDirectory(dirPath);
+    for (const file of files) {
+      try {
+        await qdrant.deletePointsByFilePath(file.path);
+      } catch (error) {
+        log('warning', 'Failed to delete Qdrant points for file', {
+          path: file.path,
+          error: error instanceof Error ? error.message : String(error)
+        });
+      }
+    }
+
+    // Delete file records and directory record from SQLite
+    const deletedFiles = sqlite.deleteFilesByDirectory(dirPath);
+    sqlite.deleteDirectory(dirPath);
+
+    // Refresh the indexed directories cache
+    refreshIndexedDirsCache(sqlite);
+
+    const chunksCount = files.reduce((sum, f) => sum + (f.chunks?.length || 0), 0);
+
+    log('info', 'Index deleted', { directory: dirPath, files: deletedFiles, chunks: chunksCount });
+    mcpServer?.sendLoggingMessage({ level: 'info', data: { event: 'index_deleted', directory: dirPath } });
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Deleted index for ${dirPath}: removed ${deletedFiles} files and ${chunksCount} chunks`
+        }
+      ]
+    };
+  } finally {
+    sqlite.close();
+  }
+}
+
 export function formatErrorResponse(error: unknown): CallToolResult {
   const errorMessage = error instanceof Error ? error.message : 'Unknown error';
   log('error', 'Tool error', { error: errorMessage });

src/mcp.ts

@@ -18,6 +18,7 @@ import {
   handleGetContentTool,
   handleGetChunkTool,
   handleServerInfoTool,
+  handleDeleteIndexTool,
   formatErrorResponse,
   setMcpServer
 } from './mcp-handlers.js';
@@ -28,7 +29,46 @@ const packageJsonPath = join(__dirname, '../package.json');
 const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
 const VERSION = packageJson.version;
 
-const MCP_TOOLS: Tool[] = [
+const DELETE_INDEX_TOOL: Tool = {
+  name: 'delete_index',
+  description: `Remove the index for a directory. Deletes all file records, vector embeddings, and the directory entry from the database.
+
+When to use this tool:
+- User wants to remove a directory from the search index
+- Cleaning up old or irrelevant indexed content
+- Freeing database space by removing unused indexes
+
+How it works:
+- Removes all file records for the specified directory from SQLite
+- Deletes corresponding vector embeddings from Qdrant
+- Removes the directory entry from the directories table
+- Does NOT delete the actual files on disk
+
+Examples:
+- Remove old project: directory_path="/home/user/old-project"
+- Clean up test data: directory_path="/home/user/test-files"
+
+Use server_info to see what directories are currently indexed before removing.`,
+  inputSchema: {
+    type: 'object',
+    properties: {
+      directory_path: {
+        type: 'string',
+        description: 'Absolute path of the directory whose index should be removed'
+      }
+    },
+    required: ['directory_path']
+  },
+  annotations: {
+    readOnlyHint: false,
+    destructiveHint: true,
+    idempotentHint: true,
+    openWorldHint: false
+  }
+};
+
+export function getMcpTools(): Tool[] {
+  const tools: Tool[] = [
   {
     name: 'index',
     description: `Index directories to make their files searchable. Processes files to create vector embeddings for semantic search.
@@ -296,7 +336,15 @@ Returns server version, indexing statistics, directory list, workspace informati
       openWorldHint: false
     }
   }
-];
+  ];
+
+  // Include delete_index tool unless DISABLE_DESTRUCTIVE is set
+  if (process.env.DISABLE_DESTRUCTIVE !== 'true') {
+    tools.push(DELETE_INDEX_TOOL);
+  }
+
+  return tools;
+}
 
 export async function startMcpServer(config: Config): Promise<void> {
   initLogLevel();
@@ -318,7 +366,7 @@ export async function startMcpServer(config: Config): Promise<void> {
 
   server.setRequestHandler(ListToolsRequestSchema, async () => {
     return {
-      tools: MCP_TOOLS
+      tools: getMcpTools()
     };
   });
 
@@ -344,7 +392,10 @@ export async function startMcpServer(config: Config): Promise<void> {
 
         case 'server_info':
           return await handleServerInfoTool(VERSION);
-        
+
+        case 'delete_index':
+          return await handleDeleteIndexTool(args, config);
+
         default:
           throw new Error(`Unknown tool: ${name}`);
       }

src/path-validation.ts

@@ -12,6 +12,14 @@ export function validatePathWithinIndexedDirs(filePath: string, indexedDirs: Set
     throw new Error('Access denied: path contains null bytes');
   }
 
+  // Validate Windows UNC path format if it starts with \\
+  if (filePath.startsWith('\\\\')) {
+    const uncParts = filePath.split('\\').filter(Boolean);
+    if (uncParts.length < 2) {
+      throw new Error('Invalid UNC path format: expected \\\\server\\share\\... pattern');
+    }
+  }
+
   // Resolve the path, following symlinks if the file exists
   let resolved: string;
   try {

src/storage.ts

@@ -457,6 +457,17 @@ export class SQLiteStorage {
     }
   }
 
+  deleteDirectory(path: string): void {
+    this.db.prepare('DELETE FROM directories WHERE path = ?').run(path);
+  }
+
+  deleteFilesByDirectory(directoryPath: string): number {
+    const result = this.db.prepare('DELETE FROM files WHERE path = ? OR path LIKE ?').run(
+      directoryPath, `${directoryPath}/%`
+    );
+    return result.changes;
+  }
+
   close(): void {
     openStorageInstances.delete(this);
     this.db.close();