feat(core,cli): implement LanceDB upsert and compact command

- Update LanceDBVectorStore to use mergeInsert API for upsert operations
- Add scalar index on 'id' column to speed up merge operations
- Implement optimize() method for vector store compaction
- Add new 'dev compact' CLI command for database optimization
- Prevents duplicate entries when re-indexing repository

This resolves the duplicate search results issue by using LanceDB's
merge_insert API instead of simple append operations. The compact
command allows users to optimize their vector store by merging
fragments and updating indices for better query performance.

Author: prosdev

packages/cli/src/cli.ts

@@ -3,6 +3,7 @@
 import chalk from 'chalk';
 import { Command } from 'commander';
 import { cleanCommand } from './commands/clean.js';
+import { compactCommand } from './commands/compact.js';
 import { exploreCommand } from './commands/explore.js';
 import { ghCommand } from './commands/gh.js';
 import { indexCommand } from './commands/index.js';
@@ -28,6 +29,7 @@ program.addCommand(planCommand);
 program.addCommand(ghCommand);
 program.addCommand(updateCommand);
 program.addCommand(statsCommand);
+program.addCommand(compactCommand);
 program.addCommand(cleanCommand);
 
 // Show help if no command provided

packages/cli/src/commands/compact.ts

@@ -0,0 +1,84 @@
+import { RepositoryIndexer } from '@lytics/dev-agent-core';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { loadConfig } from '../utils/config.js';
+import { logger } from '../utils/logger.js';
+
+export const compactCommand = new Command('compact')
+  .description('🗜️  Optimize and compact the vector store')
+  .option('-v, --verbose', 'Show detailed optimization information', false)
+  .action(async (options) => {
+    const spinner = ora('Loading configuration...').start();
+
+    try {
+      // Load config
+      const config = await loadConfig();
+      if (!config) {
+        spinner.fail('No config found');
+        logger.error('Run "dev init" first to initialize the repository');
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Initializing indexer...';
+      const indexer = new RepositoryIndexer(config);
+      await indexer.initialize();
+
+      // Get stats before optimization
+      const statsBefore = await indexer.getStats();
+      if (!statsBefore) {
+        spinner.fail('No index found');
+        logger.error('Run "dev index" first to index the repository');
+        await indexer.close();
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Optimizing vector store...';
+      const startTime = Date.now();
+
+      // Access the internal vector storage and call optimize
+      // We need to access the private vectorStorage property
+      // @ts-expect-error - accessing private property for optimization
+      await indexer.vectorStorage.optimize();
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+      // Get stats after optimization
+      const statsAfter = await indexer.getStats();
+
+      await indexer.close();
+
+      spinner.succeed(chalk.green('Vector store optimized successfully!'));
+
+      // Show results
+      logger.log('');
+      logger.log(chalk.bold('Optimization Results:'));
+      logger.log(`  ${chalk.cyan('Duration:')}        ${duration}s`);
+      logger.log(`  ${chalk.cyan('Total documents:')} ${statsAfter?.vectorsStored || 0}`);
+
+      if (options.verbose) {
+        logger.log('');
+        logger.log(chalk.bold('Before Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsBefore.vectorsStored} vectors`);
+        logger.log('');
+        logger.log(chalk.bold('After Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsAfter?.vectorsStored || 0} vectors`);
+      }
+
+      logger.log('');
+      logger.log(
+        chalk.gray(
+          'Optimization merges small data fragments, updates indices, and improves query performance.'
+        )
+      );
+    } catch (error) {
+      spinner.fail('Failed to optimize vector store');
+      logger.error(error instanceof Error ? error.message : String(error));
+      if (options.verbose && error instanceof Error && error.stack) {
+        logger.debug(error.stack);
+      }
+      process.exit(1);
+    }
+  });

packages/core/src/vector/index.ts

@@ -132,6 +132,18 @@ export class VectorStorage {
     };
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   * Call this after bulk indexing operations for better performance
+   */
+  async optimize(): Promise<void> {
+    if (!this.initialized) {
+      throw new Error('VectorStorage not initialized. Call initialize() first.');
+    }
+
+    await this.store.optimize();
+  }
+
   /**
    * Close the storage
    */

packages/core/src/vector/store.ts

@@ -43,7 +43,7 @@ export class LanceDBVectorStore implements VectorStore {
   }
 
   /**
-   * Add documents to the store
+   * Add documents to the store using upsert (prevents duplicates)
    */
   async add(documents: EmbeddingDocument[], embeddings: number[][]): Promise<void> {
     if (!this.connection) {
@@ -70,9 +70,16 @@ export class LanceDBVectorStore implements VectorStore {
       if (!this.table) {
         // Create table on first add
         this.table = await this.connection.createTable(this.tableName, data);
+        // Create scalar index on 'id' column for fast upsert operations
+        await this.ensureIdIndex();
       } else {
-        // Add to existing table
-        await this.table.add(data);
+        // Use mergeInsert to prevent duplicates (upsert operation)
+        // This updates existing documents with the same ID or inserts new ones
+        await this.table
+          .mergeInsert('id')
+          .whenMatchedUpdateAll()
+          .whenNotMatchedInsertAll()
+          .execute(data);
       }
     } catch (error) {
       throw new Error(
@@ -192,6 +199,44 @@ export class LanceDBVectorStore implements VectorStore {
     }
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   */
+  async optimize(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      await this.table.optimize();
+    } catch (error) {
+      throw new Error(
+        `Failed to optimize: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Ensure scalar index exists on 'id' column for fast upsert operations
+   */
+  private async ensureIdIndex(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      // Create a scalar index on the 'id' column to speed up mergeInsert operations
+      // LanceDB will use an appropriate index type automatically
+      await this.table.createIndex('id');
+    } catch (error) {
+      // Index may already exist or not be supported - log but don't fail
+      // Some versions of LanceDB may not support this or it may already exist
+      console.warn(
+        `Could not create index on 'id' column: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
   /**
    * Close the store
    */

packages/core/src/vector/types.ts

@@ -90,6 +90,11 @@ export interface VectorStore {
    */
   count(): Promise<number>;
 
+  /**
+   * Optimize the store (compact fragments, update indices)
+   */
+  optimize(): Promise<void>;
+
   /**
    * Close the store
    */