fix(perf): optimize MongoDB storage and add ForkedRepo index (#178)

* perf: add ForkedRepo compound index for query optimization

Add idx_forkedRepo_repo compound index {forkedRepo: 1, repo: 1} to
improve lookup performance for the ForkedRepo collection.

Identified by MongoDB Performance Advisor:
- Problem: Queries scanning 3830 docs to return 1
- Expected: 151ms → <10ms (93% improvement)

Also manually dropped the redundant candidate.data_1 index from
CNRepos (was a prefix of compound index candidate.data_1_createdPulls...).

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: address Copilot review comments

- Add `unique: true` to ForkedRepo index to prevent duplicate upserts
- Add `if (import.meta.main)` guard to migration script
- Move db.close() to finally block for proper cleanup on errors
- Add validation for --limit flag to fail fast on invalid values
- Fix docs to match actual --limit=N format

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: snomiao

scripts/migrate-cnrepos-trim-data.ts

@@ -8,18 +8,27 @@
  * Expected reduction: ~539 MB → ~10 MB (98%)
  *
  * Usage:
- *   bun scripts/migrate-cnrepos-trim-data.ts [--dry-run] [--limit N]
+ *   bun scripts/migrate-cnrepos-trim-data.ts [--dry-run] [--limit=N]
  *
  * Options:
- *   --dry-run   Preview changes without modifying data
- *   --limit N   Process only N documents (for testing)
+ *   --dry-run     Preview changes without modifying data
+ *   --limit=N     Process only N documents (for testing)
  */
 
 import { db } from "@/src/db";
 
 const DRY_RUN = process.argv.includes("--dry-run");
 const LIMIT_ARG = process.argv.find((a) => a.startsWith("--limit="));
-const LIMIT = LIMIT_ARG ? parseInt(LIMIT_ARG.split("=")[1]) : 0;
+const LIMIT = (() => {
+  if (!LIMIT_ARG) return 0;
+  const value = LIMIT_ARG.split("=")[1];
+  const parsed = Number.parseInt(value, 10);
+  if (Number.isNaN(parsed)) {
+    console.error(`Invalid --limit value "${value}". Please provide an integer.`);
+    process.exit(1);
+  }
+  return parsed;
+})();
 
 // Fields to keep in info.data
 const INFO_FIELDS = [
@@ -256,9 +265,12 @@ async function migrate() {
     console.log(`  Data size: ${(statsAfter.size / 1024 / 1024).toFixed(2)} MB`);
     console.log(`  Avg doc size: ${statsAfter.avgObjSize.toLocaleString()} bytes`);
   }
-
-  await db.close();
 }
 
-// Run
-await migrate();
+if (import.meta.main) {
+  try {
+    await migrate();
+  } finally {
+    await db.close();
+  }
+}

scripts/setup-performance-indexes.ts

@@ -3,11 +3,12 @@
  * Performance Index Migration Script
  *
  * This script creates critical indexes identified by MongoDB Performance Advisor
- * to dramatically improve query performance for SlackMsgs and CNRepos collections.
+ * to dramatically improve query performance for SlackMsgs, CNRepos, and ForkedRepo collections.
  *
  * Expected Impact:
  * - SlackMsgs: 637ms → <50ms query time (92% improvement)
  * - CNRepos: 420ms → <100ms query time (76% improvement)
+ * - ForkedRepo: 151ms → <10ms query time (93% improvement)
  * - Disk I/O reduction: ~823.8 MB per query cycle
  * - Query targeting: 79,841:1 → ~1:1 ratio
  *
@@ -22,6 +23,11 @@ import { db } from "@/src/db";
 import { SlackMsgs } from "@/lib/slack/SlackMsgs";
 import { CNRepos } from "@/src/CNRepos";
 
+// ForkedRepo collection (defined inline in src/createGithubForkForRepo.ts)
+const ForkedRepo = db.collection<{ repo: string; forkedRepo: string; updatedAt: Date }>(
+  "ForkedRepo",
+);
+
 async function setupPerformanceIndexes() {
   console.log("🚀 Setting up performance-critical indexes...\n");
 
@@ -85,6 +91,34 @@ async function setupPerformanceIndexes() {
     }
   }
 
+  // ===================================================================
+  // ISSUE #3: ForkedRepo - Missing compound index for lookups
+  // ===================================================================
+  console.log("📊 ForkedRepo Collection");
+  console.log("  Problem: Queries scanning 3830 docs to return 1");
+  console.log("  Query Pattern: { repo: ..., forkedRepo: ... }");
+  console.log("  Creating: idx_forkedRepo_repo");
+
+  try {
+    await ForkedRepo.createIndex(
+      { forkedRepo: 1, repo: 1 },
+      {
+        name: "idx_forkedRepo_repo",
+        background: true,
+        unique: true,
+      },
+    );
+    console.log("  ✅ Created compound index: idx_forkedRepo_repo");
+    console.log("  Expected improvement: 151ms → <10ms (93% faster)\n");
+  } catch (error) {
+    if ((error as Error).message.includes("already exists")) {
+      console.log("  ℹ️  Index idx_forkedRepo_repo already exists\n");
+    } else {
+      console.error("  ❌ Error creating idx_forkedRepo_repo:", error);
+      throw error;
+    }
+  }
+
   // ===================================================================
   // Verification
   // ===================================================================
@@ -108,6 +142,15 @@ async function setupPerformanceIndexes() {
     console.log(`  - ${idx.name}: ${keys}${highlight}`);
   });
 
+  // List ForkedRepo indexes
+  console.log("\n📋 ForkedRepo indexes:");
+  const forkedRepoIndexes = await ForkedRepo.listIndexes().toArray();
+  forkedRepoIndexes.forEach((idx) => {
+    const keys = JSON.stringify(idx.key);
+    const highlight = idx.name === "idx_forkedRepo_repo" ? " ⭐" : "";
+    console.log(`  - ${idx.name}: ${keys}${highlight}`);
+  });
+
   console.log("\n✨ All performance indexes created successfully!");
   console.log("\n📈 Next Steps:");
   console.log("  1. Monitor Performance Advisor in MongoDB Atlas");