feat(core,cli): implement LanceDB upsert and compact command

prosdev · prosdev · commit 6ae458822372 · 2025-11-24T21:34:23.000-08:00
- Update LanceDBVectorStore to use mergeInsert API for upsert operations
- Add scalar index on 'id' column to speed up merge operations
- Implement optimize() method for vector store compaction
- Add new 'dev compact' CLI command for database optimization
- Prevents duplicate entries when re-indexing repository

This resolves the duplicate search results issue by using LanceDB's
merge_insert API instead of simple append operations. The compact
command allows users to optimize their vector store by merging
fragments and updating indices for better query performance.
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -3,6 +3,7 @@
 import chalk from 'chalk';
 import { Command } from 'commander';
 import { cleanCommand } from './commands/clean.js';
+import { compactCommand } from './commands/compact.js';
 import { exploreCommand } from './commands/explore.js';
 import { ghCommand } from './commands/gh.js';
 import { indexCommand } from './commands/index.js';
@@ -28,6 +29,7 @@ program.addCommand(planCommand);
 program.addCommand(ghCommand);
 program.addCommand(updateCommand);
 program.addCommand(statsCommand);
+program.addCommand(compactCommand);
 program.addCommand(cleanCommand);
 
 // Show help if no command provided
diff --git a/packages/cli/src/commands/compact.ts b/packages/cli/src/commands/compact.ts
@@ -0,0 +1,84 @@
+import { RepositoryIndexer } from '@lytics/dev-agent-core';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { loadConfig } from '../utils/config.js';
+import { logger } from '../utils/logger.js';
+
+export const compactCommand = new Command('compact')
+  .description('🗜️  Optimize and compact the vector store')
+  .option('-v, --verbose', 'Show detailed optimization information', false)
+  .action(async (options) => {
+    const spinner = ora('Loading configuration...').start();
+
+    try {
+      // Load config
+      const config = await loadConfig();
+      if (!config) {
+        spinner.fail('No config found');
+        logger.error('Run "dev init" first to initialize the repository');
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Initializing indexer...';
+      const indexer = new RepositoryIndexer(config);
+      await indexer.initialize();
+
+      // Get stats before optimization
+      const statsBefore = await indexer.getStats();
+      if (!statsBefore) {
+        spinner.fail('No index found');
+        logger.error('Run "dev index" first to index the repository');
+        await indexer.close();
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Optimizing vector store...';
+      const startTime = Date.now();
+
+      // Access the internal vector storage and call optimize
+      // We need to access the private vectorStorage property
+      // @ts-expect-error - accessing private property for optimization
+      await indexer.vectorStorage.optimize();
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+      // Get stats after optimization
+      const statsAfter = await indexer.getStats();
+
+      await indexer.close();
+
+      spinner.succeed(chalk.green('Vector store optimized successfully!'));
+
+      // Show results
+      logger.log('');
+      logger.log(chalk.bold('Optimization Results:'));
+      logger.log(`  ${chalk.cyan('Duration:')}        ${duration}s`);
+      logger.log(`  ${chalk.cyan('Total documents:')} ${statsAfter?.vectorsStored || 0}`);
+
+      if (options.verbose) {
+        logger.log('');
+        logger.log(chalk.bold('Before Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsBefore.vectorsStored} vectors`);
+        logger.log('');
+        logger.log(chalk.bold('After Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsAfter?.vectorsStored || 0} vectors`);
+      }
+
+      logger.log('');
+      logger.log(
+        chalk.gray(
+          'Optimization merges small data fragments, updates indices, and improves query performance.'
+        )
+      );
+    } catch (error) {
+      spinner.fail('Failed to optimize vector store');
+      logger.error(error instanceof Error ? error.message : String(error));
+      if (options.verbose && error instanceof Error && error.stack) {
+        logger.debug(error.stack);
+      }
+      process.exit(1);
+    }
+  });
diff --git a/packages/core/src/vector/index.ts b/packages/core/src/vector/index.ts
@@ -132,6 +132,18 @@ export class VectorStorage {
     };
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   * Call this after bulk indexing operations for better performance
+   */
+  async optimize(): Promise<void> {
+    if (!this.initialized) {
+      throw new Error('VectorStorage not initialized. Call initialize() first.');
+    }
+
+    await this.store.optimize();
+  }
+
   /**
    * Close the storage
    */
diff --git a/packages/core/src/vector/store.ts b/packages/core/src/vector/store.ts
@@ -43,7 +43,7 @@ export class LanceDBVectorStore implements VectorStore {
   }
 
   /**
-   * Add documents to the store
+   * Add documents to the store using upsert (prevents duplicates)
    */
   async add(documents: EmbeddingDocument[], embeddings: number[][]): Promise<void> {
     if (!this.connection) {
@@ -70,9 +70,16 @@ export class LanceDBVectorStore implements VectorStore {
       if (!this.table) {
         // Create table on first add
         this.table = await this.connection.createTable(this.tableName, data);
+        // Create scalar index on 'id' column for fast upsert operations
+        await this.ensureIdIndex();
       } else {
-        // Add to existing table
-        await this.table.add(data);
+        // Use mergeInsert to prevent duplicates (upsert operation)
+        // This updates existing documents with the same ID or inserts new ones
+        await this.table
+          .mergeInsert('id')
+          .whenMatchedUpdateAll()
+          .whenNotMatchedInsertAll()
+          .execute(data);
       }
     } catch (error) {
       throw new Error(
@@ -192,6 +199,44 @@ export class LanceDBVectorStore implements VectorStore {
     }
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   */
+  async optimize(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      await this.table.optimize();
+    } catch (error) {
+      throw new Error(
+        `Failed to optimize: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Ensure scalar index exists on 'id' column for fast upsert operations
+   */
+  private async ensureIdIndex(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      // Create a scalar index on the 'id' column to speed up mergeInsert operations
+      // LanceDB will use an appropriate index type automatically
+      await this.table.createIndex('id');
+    } catch (error) {
+      // Index may already exist or not be supported - log but don't fail
+      // Some versions of LanceDB may not support this or it may already exist
+      console.warn(
+        `Could not create index on 'id' column: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
   /**
    * Close the store
    */
diff --git a/packages/core/src/vector/types.ts b/packages/core/src/vector/types.ts
@@ -90,6 +90,11 @@ export interface VectorStore {
    */
   count(): Promise<number>;
 
+  /**
+   * Optimize the store (compact fragments, update indices)
+   */
+  optimize(): Promise<void>;
+
   /**
    * Close the store
    */
diff --git a/packages/mcp-server/bin/dev-agent-mcp.ts b/packages/mcp-server/bin/dev-agent-mcp.ts
@@ -5,7 +5,6 @@
  */
 
 import { RepositoryIndexer } from '@lytics/dev-agent-core';
-import { GitHubIndexer } from '@lytics/dev-agent-subagents';
 import {
   ExploreAdapter,
   GitHubAdapter,
@@ -31,16 +30,6 @@ async function main() {
 
     await indexer.initialize();
 
-    // Initialize GitHub indexer
-    const githubIndexer = new GitHubIndexer({
-      vectorStorePath: `${repositoryPath}/.dev-agent/github-vectors.lance`,
-      statePath: `${repositoryPath}/.dev-agent/github-state.json`,
-      autoUpdate: false, // Don't auto-update on server start
-    });
-
-    // Initialize GitHub indexer (lazy - will be ready when first used)
-    await githubIndexer.initialize();
-
     // Create and register adapters
     const searchAdapter = new SearchAdapter({
       repositoryIndexer: indexer,
@@ -72,7 +61,9 @@ async function main() {
 
     const githubAdapter = new GitHubAdapter({
       repositoryPath,
-      githubIndexer,
+      // GitHubIndexer will be lazily initialized on first use
+      vectorStorePath: `${repositoryPath}/.dev-agent/github-vectors.lance`,
+      statePath: `${repositoryPath}/.dev-agent/github-state.json`,
       defaultLimit: 10,
       defaultFormat: 'compact',
     });
@@ -95,7 +86,10 @@ async function main() {
     const shutdown = async () => {
       await server.stop();
       await indexer.close();
-      await githubIndexer.close();
+      // Close GitHub adapter if initialized
+      if (githubAdapter.githubIndexer) {
+        await githubAdapter.githubIndexer.close();
+      }
       process.exit(0);
     };
 
diff --git a/packages/mcp-server/src/adapters/built-in/github-adapter.ts b/packages/mcp-server/src/adapters/built-in/github-adapter.ts
@@ -14,7 +14,10 @@ import type { AdapterContext, ToolDefinition, ToolExecutionContext, ToolResult }
 
 export interface GitHubAdapterConfig {
   repositoryPath: string;
-  githubIndexer: GitHubIndexer;
+  // Either pass an initialized indexer OR paths for lazy initialization
+  githubIndexer?: GitHubIndexer;
+  vectorStorePath?: string;
+  statePath?: string;
   defaultLimit?: number;
   defaultFormat?: 'compact' | 'verbose';
 }
@@ -33,26 +36,65 @@ export class GitHubAdapter extends ToolAdapter {
   };
 
   private repositoryPath: string;
-  private githubIndexer: GitHubIndexer;
+  public githubIndexer?: GitHubIndexer; // Public for cleanup in shutdown
+  private vectorStorePath?: string;
+  private statePath?: string;
   private defaultLimit: number;
   private defaultFormat: 'compact' | 'verbose';
 
   constructor(config: GitHubAdapterConfig) {
     super();
     this.repositoryPath = config.repositoryPath;
     this.githubIndexer = config.githubIndexer;
+    this.vectorStorePath = config.vectorStorePath;
+    this.statePath = config.statePath;
     this.defaultLimit = config.defaultLimit ?? 10;
     this.defaultFormat = config.defaultFormat ?? 'compact';
+
+    // Validate: either githubIndexer OR both paths must be provided
+    if (!this.githubIndexer && (!this.vectorStorePath || !this.statePath)) {
+      throw new Error(
+        'GitHubAdapter requires either githubIndexer or both vectorStorePath and statePath'
+      );
+    }
   }
 
   async initialize(context: AdapterContext): Promise<void> {
     context.logger.info('GitHubAdapter initialized', {
       repositoryPath: this.repositoryPath,
       defaultLimit: this.defaultLimit,
       defaultFormat: this.defaultFormat,
+      lazyInit: !this.githubIndexer,
     });
   }
 
+  /**
+   * Lazy initialization of GitHubIndexer
+   * Only creates the indexer when first needed
+   */
+  private async ensureGitHubIndexer(): Promise<GitHubIndexer> {
+    if (this.githubIndexer) {
+      return this.githubIndexer;
+    }
+
+    // Validate paths are available
+    if (!this.vectorStorePath || !this.statePath) {
+      throw new Error('GitHubAdapter not configured for lazy initialization');
+    }
+
+    // Lazy initialization
+    const { GitHubIndexer: GitHubIndexerClass } = await import('@lytics/dev-agent-subagents');
+
+    this.githubIndexer = new GitHubIndexerClass({
+      vectorStorePath: this.vectorStorePath,
+      statePath: this.statePath,
+      autoUpdate: false,
+    });
+
+    await this.githubIndexer.initialize();
+    return this.githubIndexer;
+  }
+
   getToolDefinition(): ToolDefinition {
     return {
       name: 'dev_gh',
@@ -260,7 +302,8 @@ export class GitHubAdapter extends ToolAdapter {
     options: GitHubSearchOptions,
     format: string
   ): Promise<string> {
-    const results = await this.githubIndexer.search(query, options);
+    const indexer = await this.ensureGitHubIndexer();
+    const results = await indexer.search(query, options);
 
     if (results.length === 0) {
       return '## GitHub Search Results\n\nNo matching issues or PRs found. Try:\n- Using different keywords\n- Removing filters (type, state, labels)\n- Re-indexing GitHub data with "dev gh index"';
@@ -278,7 +321,8 @@ export class GitHubAdapter extends ToolAdapter {
    */
   private async getContext(number: number, format: string): Promise<string> {
     // Search for the specific issue/PR
-    const results = await this.githubIndexer.search(`#${number}`, { limit: 1 });
+    const indexer = await this.ensureGitHubIndexer();
+    const results = await indexer.search(`#${number}`, { limit: 1 });
 
     if (results.length === 0) {
       throw new Error(`Issue/PR #${number} not found`);
@@ -298,7 +342,8 @@ export class GitHubAdapter extends ToolAdapter {
    */
   private async getRelated(number: number, limit: number, format: string): Promise<string> {
     // First get the main issue/PR
-    const mainResults = await this.githubIndexer.search(`#${number}`, { limit: 1 });
+    const indexer = await this.ensureGitHubIndexer();
+    const mainResults = await indexer.search(`#${number}`, { limit: 1 });
 
     if (mainResults.length === 0) {
       throw new Error(`Issue/PR #${number} not found`);
@@ -307,7 +352,7 @@ export class GitHubAdapter extends ToolAdapter {
     const mainDoc = mainResults[0].document;
 
     // Search for related items using the title
-    const relatedResults = await this.githubIndexer.search(mainDoc.title, { limit: limit + 1 });
+    const relatedResults = await indexer.search(mainDoc.title, { limit: limit + 1 });
 
     // Filter out the main item itself
     const related = relatedResults.filter((r) => r.document.number !== number).slice(0, limit);
