feat(core): add QMD-inspired hybrid search pipeline (#40)

* feat(core): add QMD-inspired hybrid search pipeline

Implement a hybrid search system combining vector embeddings with keyword search:

- Vector embeddings using node-llama-cpp with GGUF models (nomic-embed-text)
- RRF (Reciprocal Rank Fusion) for combining multiple rankers
- Query expansion with LLM and synonym fallback
- Position-aware score blending (different weights for top3, top10, rest)
- SQLite storage with sqlite-vec extension (falls back to in-memory)
- Auto-download of local GGUF models (~2GB)

New search module:
- embeddings.ts: Vector embedding generation service
- vector-store.ts: SQLite-based vector storage
- rrf.ts: RRF fusion algorithm
- expansion.ts: Query expansion with caching
- hybrid.ts: Main hybrid search pipeline
- local-models.ts: GGUF model management

CLI integration:
- skillkit recommend --hybrid: Enable hybrid search
- skillkit recommend --expand: Enable query expansion
- skillkit recommend --rerank: Enable LLM re-ranking
- skillkit recommend --build-index: Build embedding index

Tests: 56 new tests for search module (all passing)

* fix(search): address CodeRabbit review findings

- Add missing beforeEach import in embeddings.test.ts
- Update optional dependencies to current versions
- Fix weights/variations length mismatch in expansion.ts
- Avoid mutating input arrays in rrf.ts mergeRankings
- Delete orphaned chunk vectors in vector-store.ts delete() and clear()
- Add comment for empty catch block in vector-store.ts
- Make hybridSearch helper use buildIndex for true hybrid search
- Handle graceful degradation when embeddings unavailable
- Add node-llama-cpp mock in hybrid.test.ts

* fix(search): address CodeRabbit review round 2

- Fix gemma GGUF URL to point to valid community repo (bartowski)
- Add SQL injection protection for table names in vector-store.ts
- Fix resource leak with try/finally in hybridSearch helper
- Add backpressure handling for large model downloads
- Clamp startLine to non-negative in chunk calculation
- Add runtime validation warning for --expand/--rerank without --hybrid
- Update node-llama-cpp API usage (createEmbeddingContext, createContext)
- Update pnpm-lock.yaml for new optional dependency versions

* fix(search): use @ts-ignore for optional node-llama-cpp imports

CI doesn't install optional dependencies, so @ts-ignore is needed
instead of @ts-expect-error to suppress module not found errors.

Author: rohitg00

packages/cli/src/commands/recommend.ts

@@ -52,6 +52,9 @@ export class RecommendCommand extends Command {
       ['Update skill index', '$0 recommend --update'],
       ['Search for skills by task', '$0 recommend --task "authentication"'],
       ['Search for skills (alias)', '$0 recommend --search "testing"'],
+      ['Hybrid search (vector + keyword)', '$0 recommend --search "auth" --hybrid'],
+      ['Hybrid search with query expansion', '$0 recommend --search "auth" --hybrid --expand'],
+      ['Build hybrid search index', '$0 recommend --build-index'],
     ],
   });
 
@@ -125,6 +128,26 @@ export class RecommendCommand extends Command {
     description: 'Show category path for each recommendation',
   });
 
+  // Hybrid search mode
+  hybrid = Option.Boolean('--hybrid,-H', false, {
+    description: 'Use hybrid search (vector + keyword)',
+  });
+
+  // Query expansion
+  expand = Option.Boolean('--expand,-x', false, {
+    description: 'Enable query expansion (requires --hybrid)',
+  });
+
+  // Reranking
+  rerank = Option.Boolean('--rerank', false, {
+    description: 'Enable LLM reranking (requires --hybrid)',
+  });
+
+  // Build index
+  buildIndex = Option.Boolean('--build-index', false, {
+    description: 'Build/rebuild the hybrid search embedding index',
+  });
+
   async execute(): Promise<number> {
     const targetPath = resolve(this.projectPath || process.cwd());
 
@@ -133,6 +156,16 @@ export class RecommendCommand extends Command {
       return await this.updateIndex();
     }
 
+    // Handle hybrid index building
+    if (this.buildIndex) {
+      return await this.buildHybridIndex();
+    }
+
+    // Validate hybrid-dependent options
+    if ((this.expand || this.rerank) && !this.hybrid) {
+      warn('--expand and --rerank require --hybrid flag. These options will be ignored.');
+    }
+
     if (!this.quiet && !this.json) {
       header('Skill Recommendations');
     }
@@ -168,6 +201,9 @@ export class RecommendCommand extends Command {
     // Handle search mode (--search or --task)
     const searchQuery = this.search || this.task;
     if (searchQuery) {
+      if (this.hybrid) {
+        return await this.handleHybridSearch(engine, searchQuery);
+      }
       return this.handleSearch(engine, searchQuery);
     }
 
@@ -446,6 +482,132 @@ export class RecommendCommand extends Command {
     console.log(colors.muted('More details: skillkit recommend --explain --verbose'));
   }
 
+  private async handleHybridSearch(engine: RecommendationEngine, query: string): Promise<number> {
+    if (!this.quiet && !this.json) {
+      header(`Hybrid Search: "${query}"`);
+    }
+
+    const s = !this.quiet && !this.json ? spinner() : null;
+    s?.start('Initializing hybrid search...');
+
+    try {
+      await engine.initHybridSearch();
+      s?.message('Searching...');
+
+      const results = await engine.hybridSearch({
+        query,
+        limit: this.limit ? parseInt(this.limit, 10) : 10,
+        hybrid: true,
+        enableExpansion: this.expand,
+        enableReranking: this.rerank,
+        filters: {
+          minScore: this.minScore ? parseInt(this.minScore, 10) : undefined,
+        },
+      });
+
+      s?.stop(`Found ${results.length} results`);
+
+      if (this.json) {
+        console.log(JSON.stringify(results, null, 2));
+        return 0;
+      }
+
+      if (results.length === 0) {
+        warn(`No skills found matching "${query}"`);
+        return 0;
+      }
+
+      console.log('');
+      console.log(colors.bold(`Hybrid search results for "${query}" (${results.length} found):`));
+      if (this.expand && results[0]?.expandedTerms?.length) {
+        console.log(colors.muted(`  Expanded: ${results[0].expandedTerms.join(', ')}`));
+      }
+      console.log('');
+
+      for (const result of results) {
+        let relevanceColor: (text: string) => string;
+        if (result.relevance >= 70) {
+          relevanceColor = colors.success;
+        } else if (result.relevance >= 50) {
+          relevanceColor = colors.warning;
+        } else {
+          relevanceColor = colors.muted;
+        }
+        const relevanceBar = progressBar(result.relevance, 100, 10);
+
+        console.log(`  ${relevanceColor(`${result.relevance}%`)} ${colors.dim(relevanceBar)} ${colors.bold(result.skill.name)}`);
+
+        if (result.skill.description) {
+          console.log(`      ${colors.muted(truncate(result.skill.description, 70))}`);
+        }
+
+        if (this.verbose) {
+          const scores: string[] = [];
+          if (typeof result.vectorSimilarity === 'number') {
+            scores.push(`vector: ${(result.vectorSimilarity * 100).toFixed(0)}%`);
+          }
+          if (typeof result.keywordScore === 'number') {
+            scores.push(`keyword: ${result.keywordScore.toFixed(0)}%`);
+          }
+          if (typeof result.rrfScore === 'number') {
+            scores.push(`rrf: ${result.rrfScore.toFixed(3)}`);
+          }
+          if (scores.length > 0) {
+            console.log(`      ${colors.dim('Scores:')} ${scores.join(' | ')}`);
+          }
+        }
+
+        if (result.matchedTerms.length > 0) {
+          console.log(`      ${colors.dim('Matched:')} ${result.matchedTerms.join(', ')}`);
+        }
+
+        console.log('');
+      }
+
+      return 0;
+    } catch (err) {
+      s?.stop(colors.error('Hybrid search failed'));
+      console.log(colors.muted(err instanceof Error ? err.message : String(err)));
+      console.log(colors.muted('Falling back to standard search...'));
+      return this.handleSearch(engine, query);
+    }
+  }
+
+  private async buildHybridIndex(): Promise<number> {
+    if (!this.quiet) {
+      header('Build Hybrid Search Index');
+    }
+
+    const index = this.loadIndex();
+    if (!index || index.skills.length === 0) {
+      warn('No skill index found. Run --update first.');
+      return 1;
+    }
+
+    const s = spinner();
+    s.start('Initializing...');
+
+    try {
+      const engine = new RecommendationEngine();
+      engine.loadIndex(index);
+
+      await engine.buildHybridIndex((progress) => {
+        const percentage = Math.round((progress.current / progress.total) * 100);
+        s.message(`${progress.phase}: ${progress.message || ''} (${percentage}%)`);
+      });
+
+      s.stop(colors.success(`${symbols.success} Built hybrid index for ${index.skills.length} skills`));
+      console.log(colors.muted('  Index stored in: ~/.skillkit/search.db'));
+      console.log(colors.muted('  Use --hybrid flag for vector+keyword search\n'));
+
+      return 0;
+    } catch (err) {
+      s.stop(colors.error('Failed to build hybrid index'));
+      console.log(colors.muted(err instanceof Error ? err.message : String(err)));
+      return 1;
+    }
+  }
+
   private handleSearch(engine: RecommendationEngine, query: string): number {
     if (!this.quiet && !this.json) {
       header(`Search: "${query}"`);

packages/core/package.json

@@ -37,6 +37,10 @@
     "./learning": {
       "import": "./dist/learning/index.js",
       "types": "./dist/learning/index.d.ts"
+    },
+    "./search": {
+      "import": "./dist/search/index.js",
+      "types": "./dist/search/index.d.ts"
     }
   },
   "files": [
@@ -53,7 +57,13 @@
     "yaml": "^2.6.1",
     "zod": "^3.24.1"
   },
+  "optionalDependencies": {
+    "node-llama-cpp": "^3.15.0",
+    "better-sqlite3": "^12.0.0",
+    "sqlite-vec": "^0.1.6"
+  },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
     "@types/node": "^22.10.5",
     "tsup": "^8.3.5",
     "typescript": "^5.7.2",

packages/core/src/index.ts

@@ -101,3 +101,6 @@ export * from './connectors/index.js';
 
 // Execution Flow (Step Tracking & Metrics - Phase 21)
 export * from './execution/index.js';
+
+// Hybrid Search (QMD-Inspired Vector + Keyword Search)
+export * from './search/index.js';

packages/core/src/recommend/engine.ts

@@ -11,25 +11,51 @@ import type {
   SearchOptions,
   SearchResult,
   FreshnessResult,
+  RecommendHybridSearchOptions,
+  RecommendHybridSearchResult,
 } from './types.js';
 import { DEFAULT_SCORING_WEIGHTS, TAG_TO_TECH, getTechTags } from './types.js';
+import type { HybridSearchPipeline } from '../search/hybrid.js';
 
 /**
  * Recommendation engine for matching skills to project profiles
  */
 export class RecommendationEngine {
   private weights: ScoringWeights;
   private index: SkillIndex | null = null;
+  private hybridPipeline: HybridSearchPipeline | null = null;
 
   constructor(weights?: Partial<ScoringWeights>) {
     this.weights = { ...DEFAULT_SCORING_WEIGHTS, ...weights };
   }
 
+  /**
+   * Initialize hybrid search pipeline for vector + keyword search
+   */
+  async initHybridSearch(): Promise<void> {
+    const { createHybridSearchPipeline } = await import('../search/hybrid.js');
+    this.hybridPipeline = createHybridSearchPipeline();
+    if (this.index) {
+      this.hybridPipeline.loadSkillsIndex(this.index);
+    }
+    await this.hybridPipeline.initialize();
+  }
+
+  /**
+   * Check if hybrid search is available
+   */
+  isHybridSearchAvailable(): boolean {
+    return this.hybridPipeline !== null && this.hybridPipeline.isInitialized();
+  }
+
   /**
    * Load skill index from cache or generate from local skills
    */
   loadIndex(index: SkillIndex): void {
     this.index = index;
+    if (this.hybridPipeline) {
+      this.hybridPipeline.loadSkillsIndex(index);
+    }
   }
 
   /**
@@ -603,6 +629,71 @@ export class RecommendationEngine {
     return { relevance, matchedTerms, snippet };
   }
 
+  /**
+   * Hybrid search combining vector embeddings and keyword matching
+   */
+  async hybridSearch(options: RecommendHybridSearchOptions): Promise<RecommendHybridSearchResult[]> {
+    const { query, limit = 10, hybrid = true, enableExpansion = false, enableReranking = false, filters } = options;
+
+    if (!hybrid || !this.hybridPipeline) {
+      const basicResults = this.search({ query, limit, semantic: true, filters });
+      return basicResults.map((r) => ({
+        ...r,
+        hybridScore: r.relevance / 100,
+      }));
+    }
+
+    const response = await this.hybridPipeline.search({
+      query,
+      limit,
+      enableExpansion,
+      enableReranking,
+    });
+
+    let results = response.results.map((r) => ({
+      skill: r.skill,
+      relevance: r.relevance,
+      matchedTerms: r.matchedTerms,
+      snippet: r.snippet,
+      hybridScore: r.hybridScore,
+      vectorSimilarity: r.vectorSimilarity,
+      keywordScore: r.keywordScore,
+      rrfScore: r.rrfScore,
+      expandedTerms: r.expandedTerms,
+    }));
+
+    if (filters?.tags && filters.tags.length > 0) {
+      results = results.filter((r) =>
+        r.skill.tags?.some((t) => filters.tags!.includes(t))
+      );
+    }
+    if (filters?.verified) {
+      results = results.filter((r) => r.skill.verified);
+    }
+    if (filters?.minScore) {
+      results = results.filter((r) => r.relevance >= filters.minScore!);
+    }
+
+    return results.slice(0, limit);
+  }
+
+  /**
+   * Build hybrid search index from skills
+   */
+  async buildHybridIndex(
+    onProgress?: (progress: { phase: string; current: number; total: number; message?: string }) => void
+  ): Promise<void> {
+    if (!this.index) {
+      throw new Error('No skill index loaded. Call loadIndex() first.');
+    }
+
+    if (!this.hybridPipeline) {
+      await this.initHybridSearch();
+    }
+
+    await this.hybridPipeline!.buildIndex(this.index.skills, onProgress);
+  }
+
   /**
    * Check freshness of installed skills against project dependencies
    *

packages/core/src/recommend/types.ts

@@ -326,3 +326,25 @@ export interface ReasoningRecommendationResult extends RecommendationResult {
     strategy: string;
   };
 }
+
+/**
+ * Hybrid search options for RecommendationEngine
+ */
+export interface RecommendHybridSearchOptions extends SearchOptions {
+  hybrid?: boolean;
+  enableExpansion?: boolean;
+  enableReranking?: boolean;
+  semanticWeight?: number;
+  keywordWeight?: number;
+}
+
+/**
+ * Hybrid search result with additional metadata for RecommendationEngine
+ */
+export interface RecommendHybridSearchResult extends SearchResult {
+  hybridScore?: number;
+  vectorSimilarity?: number;
+  keywordScore?: number;
+  rrfScore?: number;
+  expandedTerms?: string[];
+}