chore: iterate on evals

Author: ThomasRooney

mise.toml

@@ -43,7 +43,7 @@ run = "cd packages/playground && npx vite"
 
 [tasks."bench:realistic"]
 description = "Benchmark embedding providers against realistic test fixture (none, hash)"
-run = "node packages/eval/dist/bin.js benchmark --cases packages/eval/fixtures/realistic/cases.json --docs-dir packages/eval/fixtures/realistic --work-dir /tmp/bench-realistic --build-command packages/cli/dist/index.js --server-command packages/server/dist/bin.js --providers none,hash"
+run = "node packages/eval/dist/bin.js benchmark --cases packages/eval/fixtures/realistic/cases.json --docs-dir packages/eval/fixtures/realistic --work-dir /tmp/bench-realistic --build-command packages/cli/dist/index.js --server-command packages/server/dist/bin.js --embeddings none,hash"
 depends = ["build"]
 
 [tasks.nuke]

packages/cli/src/index.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
 import path from "node:path";
 import fg from "fast-glob";
@@ -11,17 +11,20 @@ const { version: CLI_VERSION } = require("../package.json") as { version: string
 import {
   buildLanceDbIndex,
   buildChunks,
+  computeChunkFingerprint,
   createEmbeddingProvider,
   embedChunksIncremental,
   loadCache,
+  loadChunksFromPreviousIndex,
   parseManifestJson,
   resolveFileConfig,
   saveCache,
   type Chunk,
   type EmbedProgressEvent,
   type EmbeddingMetadata,
   type IndexBuildStep,
-  type Manifest
+  type Manifest,
+  type PreviousIndexReader
 } from "@speakeasy-api/docs-mcp-core";
 import { buildHeuristicManifest } from "./fix.js";
 import { resolveSourceCommit } from "./git.js";
@@ -116,8 +119,63 @@ program
     const outDir = path.resolve(options.out);
     const files = await listMarkdownFiles(docsDir);
     const manifestCache = new Map<string, Manifest>();
+    const lanceDbPath = path.join(outDir, ".lancedb");
+    const lanceDbTmpPath = path.join(outDir, ".lancedb.tmp");
+    const lanceDbOldPath = path.join(outDir, ".lancedb.old");
+
+    // Clean up stale tmp/old dirs from interrupted builds
+    await rm(lanceDbTmpPath, { recursive: true, force: true });
+    await rm(lanceDbOldPath, { recursive: true, force: true });
+
+    // Load previous index for chunk caching (old .lancedb/ stays readable during build)
+    let previousIndex: PreviousIndexReader | null = options.rebuildCache
+      ? null
+      : await loadChunksFromPreviousIndex(lanceDbPath);
+
+    // Canary validation: re-chunk the first 10 fingerprint-matching files to
+    // detect chunking logic changes without maintaining a version number.
+    if (previousIndex) {
+      let validated = 0;
+      for (const file of files) {
+        if (validated >= 10) break;
+        const markdown = await readFile(file, "utf8");
+        const relative = toPosix(path.relative(docsDir, file));
+        const manifestContext = await loadNearestManifest(file, docsDir, manifestCache);
+        const resolved = resolveFileConfig({
+          relativeFilePath: relative,
+          markdown,
+          ...(manifestContext
+            ? {
+                manifest: manifestContext.manifest,
+                manifestBaseDir: manifestContext.manifestBaseDir
+              }
+            : {})
+        });
+
+        const fingerprint = computeChunkFingerprint(markdown, resolved.strategy, resolved.metadata);
+        if (previousIndex.fingerprints.get(relative) !== fingerprint) continue;
+
+        const freshChunks = buildChunks({
+          filepath: relative,
+          markdown,
+          strategy: resolved.strategy,
+          metadata: resolved.metadata
+        });
+        const cachedChunks = await previousIndex.getChunks(relative);
+
+        if (JSON.stringify(freshChunks) !== JSON.stringify(cachedChunks)) {
+          console.warn(`warn: chunk cache canary mismatch for ${relative}; discarding cache`);
+          previousIndex.close();
+          previousIndex = null;
+          break;
+        }
+        validated++;
+      }
+    }
 
     const chunks: Chunk[] = [];
+    const newFileFingerprints: Record<string, string> = {};
+    let chunkCacheHits = 0;
     for (let fi = 0; fi < files.length; fi++) {
       writeProgress(`Chunking [${fi + 1}/${files.length}]...`);
       const file = files[fi]!;
@@ -135,6 +193,16 @@ program
           : {})
       });
 
+      const fingerprint = computeChunkFingerprint(markdown, resolved.strategy, resolved.metadata);
+      newFileFingerprints[relative] = fingerprint;
+
+      if (previousIndex?.fingerprints.get(relative) === fingerprint) {
+        const cachedChunks = await previousIndex.getChunks(relative);
+        chunks.push(...cachedChunks);
+        chunkCacheHits++;
+        continue;
+      }
+
       const fileChunks = buildChunks({
         filepath: relative,
         markdown,
@@ -144,7 +212,8 @@ program
       chunks.push(...fileChunks);
     }
     clearProgress();
-    console.warn(`Chunked ${files.length} files into ${chunks.length.toLocaleString()} chunks`);
+    const cacheSuffix = chunkCacheHits > 0 ? ` (${chunkCacheHits} cached)` : "";
+    console.warn(`Chunked ${files.length} files into ${chunks.length.toLocaleString()} chunks${cacheSuffix}`);
 
     const providerInput: {
       provider: "none" | "hash" | "openai";
@@ -252,7 +321,9 @@ program
       sourceCommit
     );
     const metadataKeys = Object.keys(metadata.taxonomy);
-    const lanceDbPath = path.join(outDir, ".lancedb");
+
+    // Close previous index before writing the new one
+    previousIndex?.close();
 
     const indexStepLabels: Record<IndexBuildStep, string> = {
       "writing-table": "Building search index: writing table...",
@@ -265,11 +336,13 @@ program
       chunks: Chunk[];
       metadataKeys: string[];
       vectorsByChunkId?: Map<string, number[]>;
+      fileFingerprints?: Record<string, string>;
       onProgress?: (step: IndexBuildStep) => void;
     } = {
-      dbPath: lanceDbPath,
+      dbPath: lanceDbTmpPath,
       chunks,
       metadataKeys,
+      fileFingerprints: newFileFingerprints,
       onProgress: (step) => writeProgress(indexStepLabels[step]),
     };
     if (vectorsByChunkId) {
@@ -295,6 +368,12 @@ program
       )
     );
 
+    // Atomic swap: .lancedb.tmp → .lancedb
+    await rm(lanceDbOldPath, { recursive: true, force: true });
+    try { await rename(lanceDbPath, lanceDbOldPath); } catch {}
+    await rename(lanceDbTmpPath, lanceDbPath);
+    await rm(lanceDbOldPath, { recursive: true, force: true }).catch(() => {});
+
     console.log(`wrote ${chunks.length} chunks and .lancedb index to ${outDir}`);
   });
 

packages/core/src/chunk-cache.ts

@@ -0,0 +1,152 @@
+import { connect } from "@lancedb/lancedb";
+import { sha256hex } from "./embedding.js";
+import type { Chunk, ChunkingStrategy } from "./types.js";
+import type { ChunkRow } from "./lancedb.js";
+
+const DEFAULT_TABLE_NAME = "chunks";
+
+/**
+ * Compute a fingerprint for a file's chunking inputs.
+ * Changes when the markdown, strategy, or metadata change.
+ */
+export function computeChunkFingerprint(
+  markdown: string,
+  strategy: ChunkingStrategy,
+  metadata: Record<string, string>
+): string {
+  return sha256hex(
+    [JSON.stringify(strategy), JSON.stringify(metadata), markdown].join("\0")
+  );
+}
+
+export interface PreviousIndexReader {
+  fingerprints: Map<string, string>; // filepath → fingerprint
+  getChunks(filepath: string): Promise<Chunk[]>;
+  close(): void;
+}
+
+/**
+ * Open the previous `.lancedb/` index and extract per-file fingerprints.
+ * Returns null if the index doesn't exist or lacks the `file_fingerprint` column.
+ */
+export async function loadChunksFromPreviousIndex(
+  dbPath: string,
+  tableName?: string
+): Promise<PreviousIndexReader | null> {
+  const table = tableName ?? DEFAULT_TABLE_NAME;
+
+  let db;
+  try {
+    db = await connect(dbPath);
+  } catch {
+    return null;
+  }
+
+  try {
+    const tableNames = await db.tableNames();
+    if (!tableNames.includes(table)) {
+      db.close();
+      return null;
+    }
+
+    const tbl = await db.openTable(table);
+
+    // Probe for file_fingerprint column by fetching a single row
+    const probeRows = await tbl
+      .query()
+      .select(["filepath", "file_fingerprint"])
+      .limit(1)
+      .toArray();
+
+    if (probeRows.length === 0) {
+      // Empty table — nothing to cache from
+      tbl.close();
+      db.close();
+      return null;
+    }
+
+    const probeRow = probeRows[0]!;
+    if (
+      !("file_fingerprint" in probeRow) ||
+      typeof probeRow.file_fingerprint !== "string"
+    ) {
+      // Old-format index without fingerprints
+      tbl.close();
+      db.close();
+      return null;
+    }
+
+    // Load all fingerprints (lightweight: only two string columns)
+    const fpRows = await tbl
+      .query()
+      .select(["filepath", "file_fingerprint"])
+      .toArray();
+
+    const fingerprints = new Map<string, string>();
+    for (const row of fpRows) {
+      const filepath = row.filepath as string;
+      const fp = row.file_fingerprint as string;
+      if (fp) {
+        fingerprints.set(filepath, fp);
+      }
+    }
+
+    return {
+      fingerprints,
+      async getChunks(filepath: string): Promise<Chunk[]> {
+        const escaped = filepath.replace(/'/g, "''");
+        const rows = await tbl
+          .query()
+          .where(`filepath = '${escaped}'`)
+          .toArray();
+
+        return rows
+          .map((row) => row as ChunkRow)
+          .sort((a, b) => toNumber(a.chunk_index) - toNumber(b.chunk_index))
+          .map((row) => rowToChunk(row));
+      },
+      close() {
+        tbl.close();
+        db.close();
+      },
+    };
+  } catch {
+    db.close();
+    return null;
+  }
+}
+
+function toNumber(value: unknown): number {
+  if (typeof value === "number") return value;
+  if (typeof value === "bigint") return Number(value);
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function rowToChunk(row: ChunkRow): Chunk {
+  let metadata: Record<string, string> = {};
+  if (typeof row.metadata_json === "string" && row.metadata_json.trim()) {
+    try {
+      const parsed = JSON.parse(row.metadata_json) as Record<string, unknown>;
+      for (const [key, value] of Object.entries(parsed)) {
+        if (typeof value === "string") {
+          metadata[key] = value;
+        }
+      }
+    } catch {
+      metadata = {};
+    }
+  }
+
+  return {
+    chunk_id: String(row.chunk_id ?? ""),
+    filepath: String(row.filepath ?? ""),
+    heading: String(row.heading ?? ""),
+    heading_level: toNumber(row.heading_level),
+    content: String(row.content ?? ""),
+    content_text: String(row.content_text ?? ""),
+    breadcrumb: String(row.breadcrumb ?? ""),
+    chunk_index: toNumber(row.chunk_index),
+    metadata,
+  };
+}

packages/core/src/embedding.ts

@@ -37,6 +37,7 @@ export class NoopEmbeddingProvider implements EmbeddingProvider {
   readonly name = "none";
   readonly model = "none";
   readonly dimensions = 0;
+  readonly costPerMillionTokens = 0;
   readonly configFingerprint: string;
 
   constructor() {
@@ -56,6 +57,7 @@ export class HashEmbeddingProvider implements EmbeddingProvider {
   readonly name = "hash";
   readonly model: string;
   readonly dimensions: number;
+  readonly costPerMillionTokens = 0;
   readonly configFingerprint: string;
 
   constructor(options: { dimensions?: number; model?: string } = {}) {
@@ -80,10 +82,18 @@ export class HashEmbeddingProvider implements EmbeddingProvider {
  */
 const DEFAULT_MAX_INPUT_CHARS = 24_000;
 
+/** Known pricing in USD per 1M input tokens for OpenAI embedding models. */
+const OPENAI_COST_PER_M_TOKENS: Record<string, number> = {
+  "text-embedding-3-large": 0.13,
+  "text-embedding-3-small": 0.02,
+  "text-embedding-ada-002": 0.10,
+};
+
 export class OpenAIEmbeddingProvider implements EmbeddingProvider {
   readonly name = "openai";
   readonly model: string;
   readonly dimensions: number;
+  readonly costPerMillionTokens: number;
   readonly configFingerprint: string;
 
   private readonly apiKey: string;
@@ -101,6 +111,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
 
     this.apiKey = options.apiKey;
     this.model = options.model ?? "text-embedding-3-large";
+    this.costPerMillionTokens = OPENAI_COST_PER_M_TOKENS[this.model] ?? 0;
     this.dimensions = options.dimensions ?? 3072;
     this.batchSize = options.batchSize ?? 128;
     this.baseUrl = (options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "");

packages/core/src/index.ts

@@ -9,3 +9,4 @@ export * from "./search.js";
 export * from "./embedding.js";
 export * from "./embedding-cache.js";
 export * from "./lancedb.js";
+export * from "./chunk-cache.js";