feat(mcp): add ExploreAdapter and GitHubAdapter

packages/cli/src/cli.ts

@@ -3,6 +3,7 @@
 import chalk from 'chalk';
 import { Command } from 'commander';
 import { cleanCommand } from './commands/clean.js';
+import { compactCommand } from './commands/compact.js';
 import { exploreCommand } from './commands/explore.js';
 import { ghCommand } from './commands/gh.js';
 import { indexCommand } from './commands/index.js';
@@ -28,6 +29,7 @@ program.addCommand(planCommand);
 program.addCommand(ghCommand);
 program.addCommand(updateCommand);
 program.addCommand(statsCommand);
+program.addCommand(compactCommand);
 program.addCommand(cleanCommand);
 
 // Show help if no command provided

packages/cli/src/commands/compact.ts

@@ -0,0 +1,84 @@
+import { RepositoryIndexer } from '@lytics/dev-agent-core';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+import { loadConfig } from '../utils/config.js';
+import { logger } from '../utils/logger.js';
+
+export const compactCommand = new Command('compact')
+  .description('🗜️  Optimize and compact the vector store')
+  .option('-v, --verbose', 'Show detailed optimization information', false)
+  .action(async (options) => {
+    const spinner = ora('Loading configuration...').start();
+
+    try {
+      // Load config
+      const config = await loadConfig();
+      if (!config) {
+        spinner.fail('No config found');
+        logger.error('Run "dev init" first to initialize the repository');
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Initializing indexer...';
+      const indexer = new RepositoryIndexer(config);
+      await indexer.initialize();
+
+      // Get stats before optimization
+      const statsBefore = await indexer.getStats();
+      if (!statsBefore) {
+        spinner.fail('No index found');
+        logger.error('Run "dev index" first to index the repository');
+        await indexer.close();
+        process.exit(1);
+        return;
+      }
+
+      spinner.text = 'Optimizing vector store...';
+      const startTime = Date.now();
+
+      // Access the internal vector storage and call optimize
+      // We need to access the private vectorStorage property
+      // @ts-expect-error - accessing private property for optimization
+      await indexer.vectorStorage.optimize();
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+      // Get stats after optimization
+      const statsAfter = await indexer.getStats();
+
+      await indexer.close();
+
+      spinner.succeed(chalk.green('Vector store optimized successfully!'));
+
+      // Show results
+      logger.log('');
+      logger.log(chalk.bold('Optimization Results:'));
+      logger.log(`  ${chalk.cyan('Duration:')}        ${duration}s`);
+      logger.log(`  ${chalk.cyan('Total documents:')} ${statsAfter?.vectorsStored || 0}`);
+
+      if (options.verbose) {
+        logger.log('');
+        logger.log(chalk.bold('Before Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsBefore.vectorsStored} vectors`);
+        logger.log('');
+        logger.log(chalk.bold('After Optimization:'));
+        logger.log(`  ${chalk.cyan('Storage size:')} ${statsAfter?.vectorsStored || 0} vectors`);
+      }
+
+      logger.log('');
+      logger.log(
+        chalk.gray(
+          'Optimization merges small data fragments, updates indices, and improves query performance.'
+        )
+      );
+    } catch (error) {
+      spinner.fail('Failed to optimize vector store');
+      logger.error(error instanceof Error ? error.message : String(error));
+      if (options.verbose && error instanceof Error && error.stack) {
+        logger.debug(error.stack);
+      }
+      process.exit(1);
+    }
+  });

packages/cli/src/commands/stats.ts

@@ -1,4 +1,7 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
 import { RepositoryIndexer } from '@lytics/dev-agent-core';
+import { GitHubIndexer } from '@lytics/dev-agent-subagents';
 import chalk from 'chalk';
 import { Command } from 'commander';
 import ora from 'ora';
@@ -25,6 +28,36 @@ export const statsCommand = new Command('stats')
       await indexer.initialize();
 
       const stats = await indexer.getStats();
+
+      // Try to load GitHub stats
+      let githubStats = null;
+      try {
+        // Try to load repository from state file
+        let repository: string | undefined;
+        const statePath = path.join(config.repositoryPath, '.dev-agent/github-state.json');
+        try {
+          const stateContent = await fs.readFile(statePath, 'utf-8');
+          const state = JSON.parse(stateContent);
+          repository = state.repository;
+        } catch {
+          // State file doesn't exist
+        }
+
+        const githubIndexer = new GitHubIndexer(
+          {
+            vectorStorePath: `${config.vectorStorePath}-github`,
+            statePath,
+            autoUpdate: false,
+          },
+          repository
+        );
+        await githubIndexer.initialize();
+        githubStats = githubIndexer.getStats();
+        await githubIndexer.close();
+      } catch {
+        // GitHub not indexed, ignore
+      }
+
       await indexer.close();
 
       spinner.stop();
@@ -66,6 +99,36 @@ export const statsCommand = new Command('stats')
         logger.warn(`${stats.errors.length} error(s) during last indexing`);
       }
 
+      // Display GitHub stats if available
+      if (githubStats) {
+        logger.log('');
+        logger.log(chalk.bold.cyan('🔗 GitHub Integration'));
+        logger.log('');
+        logger.log(`${chalk.cyan('Repository:')}        ${githubStats.repository}`);
+        logger.log(`${chalk.cyan('Total Documents:')}   ${githubStats.totalDocuments}`);
+        logger.log(`${chalk.cyan('Issues:')}            ${githubStats.byType.issue || 0}`);
+        logger.log(`${chalk.cyan('Pull Requests:')}     ${githubStats.byType.pull_request || 0}`);
+        logger.log('');
+        logger.log(`${chalk.cyan('Open:')}              ${githubStats.byState.open || 0}`);
+        logger.log(`${chalk.cyan('Closed:')}            ${githubStats.byState.closed || 0}`);
+        if (githubStats.byState.merged) {
+          logger.log(`${chalk.cyan('Merged:')}            ${githubStats.byState.merged}`);
+        }
+        logger.log('');
+        logger.log(
+          `${chalk.cyan('Last Synced:')}       ${new Date(githubStats.lastIndexed).toLocaleString()}`
+        );
+      } else {
+        logger.log('');
+        logger.log(chalk.bold.cyan('🔗 GitHub Integration'));
+        logger.log('');
+        logger.log(
+          chalk.gray('Not indexed. Run') +
+            chalk.yellow(' dev gh index ') +
+            chalk.gray('to sync GitHub data.')
+        );
+      }
+
       logger.log('');
     } catch (error) {
       spinner.fail('Failed to load statistics');

packages/core/src/vector/index.ts

@@ -132,6 +132,18 @@ export class VectorStorage {
     };
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   * Call this after bulk indexing operations for better performance
+   */
+  async optimize(): Promise<void> {
+    if (!this.initialized) {
+      throw new Error('VectorStorage not initialized. Call initialize() first.');
+    }
+
+    await this.store.optimize();
+  }
+
   /**
    * Close the storage
    */

packages/core/src/vector/store.ts

@@ -43,7 +43,7 @@ export class LanceDBVectorStore implements VectorStore {
   }
 
   /**
-   * Add documents to the store
+   * Add documents to the store using upsert (prevents duplicates)
    */
   async add(documents: EmbeddingDocument[], embeddings: number[][]): Promise<void> {
     if (!this.connection) {
@@ -70,9 +70,16 @@ export class LanceDBVectorStore implements VectorStore {
       if (!this.table) {
         // Create table on first add
         this.table = await this.connection.createTable(this.tableName, data);
+        // Create scalar index on 'id' column for fast upsert operations
+        await this.ensureIdIndex();
       } else {
-        // Add to existing table
-        await this.table.add(data);
+        // Use mergeInsert to prevent duplicates (upsert operation)
+        // This updates existing documents with the same ID or inserts new ones
+        await this.table
+          .mergeInsert('id')
+          .whenMatchedUpdateAll()
+          .whenNotMatchedInsertAll()
+          .execute(data);
       }
     } catch (error) {
       throw new Error(
@@ -192,6 +199,44 @@ export class LanceDBVectorStore implements VectorStore {
     }
   }
 
+  /**
+   * Optimize the vector store (compact fragments, update indices)
+   */
+  async optimize(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      await this.table.optimize();
+    } catch (error) {
+      throw new Error(
+        `Failed to optimize: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Ensure scalar index exists on 'id' column for fast upsert operations
+   */
+  private async ensureIdIndex(): Promise<void> {
+    if (!this.table) {
+      return;
+    }
+
+    try {
+      // Create a scalar index on the 'id' column to speed up mergeInsert operations
+      // LanceDB will use an appropriate index type automatically
+      await this.table.createIndex('id');
+    } catch (error) {
+      // Index may already exist or not be supported - log but don't fail
+      // Some versions of LanceDB may not support this or it may already exist
+      console.warn(
+        `Could not create index on 'id' column: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
   /**
    * Close the store
    */

packages/core/src/vector/types.ts

@@ -90,6 +90,11 @@ export interface VectorStore {
    */
   count(): Promise<number>;
 
+  /**
+   * Optimize the store (compact fragments, update indices)
+   */
+  optimize(): Promise<void>;
+
   /**
    * Close the store
    */

packages/mcp-server/bin/dev-agent-mcp.ts

@@ -5,7 +5,13 @@
  */
 
 import { RepositoryIndexer } from '@lytics/dev-agent-core';
-import { PlanAdapter, SearchAdapter, StatusAdapter } from '../src/adapters/built-in';
+import {
+  ExploreAdapter,
+  GitHubAdapter,
+  PlanAdapter,
+  SearchAdapter,
+  StatusAdapter,
+} from '../src/adapters/built-in';
 import { MCPServer } from '../src/server/mcp-server';
 
 // Get config from environment
@@ -45,6 +51,23 @@ async function main() {
       timeout: 60000, // 60 seconds
     });
 
+    const exploreAdapter = new ExploreAdapter({
+      repositoryPath,
+      repositoryIndexer: indexer,
+      defaultLimit: 10,
+      defaultThreshold: 0.7,
+      defaultFormat: 'compact',
+    });
+
+    const githubAdapter = new GitHubAdapter({
+      repositoryPath,
+      // GitHubIndexer will be lazily initialized on first use
+      vectorStorePath: `${vectorStorePath}-github`,
+      statePath: `${repositoryPath}/.dev-agent/github-state.json`,
+      defaultLimit: 10,
+      defaultFormat: 'compact',
+    });
+
     // Create MCP server
     const server = new MCPServer({
       serverInfo: {
@@ -56,13 +79,17 @@ async function main() {
         logLevel,
       },
       transport: 'stdio',
-      adapters: [searchAdapter, statusAdapter, planAdapter],
+      adapters: [searchAdapter, statusAdapter, planAdapter, exploreAdapter, githubAdapter],
     });
 
     // Handle graceful shutdown
     const shutdown = async () => {
       await server.stop();
       await indexer.close();
+      // Close GitHub adapter if initialized
+      if (githubAdapter.githubIndexer) {
+        await githubAdapter.githubIndexer.close();
+      }
       process.exit(0);
     };