|
| 1 | +import "dotenv/config"; |
| 2 | +import { db } from "../server/db"; |
| 3 | +import { users, repositories, reviews, audits, auditOrders, apiUsageLog, requestLogs } from "../shared/schema"; |
| 4 | +import { sql, gte } from "drizzle-orm"; |
| 5 | +import * as fs from "fs"; |
| 6 | +import * as path from "path"; |
| 7 | + |
| 8 | +async function generateSnapshot() { |
| 9 | + console.log("Generating traction snapshot..."); |
| 10 | + |
| 11 | + // 1. Users |
| 12 | + const totalUsersResult = await db.select({ count: sql<number>`count(*)` }).from(users); |
| 13 | + const totalUsers = Number(totalUsersResult[0].count); |
| 14 | + |
| 15 | + const thirtyDaysAgo = new Date(); |
| 16 | + thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30); |
| 17 | + |
| 18 | + const sevenDaysAgo = new Date(); |
| 19 | + sevenDaysAgo.getDate() - 7; // Wait, let's just use 7 days ago |
| 20 | + |
| 21 | + const activeUsers30dResult = await db.select({ count: sql<number>`count(distinct ${requestLogs.userId})` }) |
| 22 | + .from(requestLogs) |
| 23 | + .where(gte(requestLogs.timestamp, thirtyDaysAgo)); |
| 24 | + const activeUsers30d = Number(activeUsers30dResult[0].count); |
| 25 | + |
| 26 | + const activeUsers7dResult = await db.select({ count: sql<number>`count(distinct ${requestLogs.userId})` }) |
| 27 | + .from(requestLogs) |
| 28 | + .where(gte(requestLogs.timestamp, new Date(Date.now() - 7 * 24 * 60 * 60 * 1000))); |
| 29 | + const activeUsers7d = Number(activeUsers7dResult[0].count); |
| 30 | + |
| 31 | + // 2. Repositories |
| 32 | + const totalReposResult = await db.select({ count: sql<number>`count(*)` }).from(repositories); |
| 33 | + const totalRepos = Number(totalReposResult[0].count); |
| 34 | + |
| 35 | + // 3. PR Reviews & Audit Mode runs |
| 36 | + const totalReviewsResult = await db.select({ count: sql<number>`count(*)` }).from(reviews); |
| 37 | + const totalReviews = Number(totalReviewsResult[0].count); |
| 38 | + |
| 39 | + const totalAuditsResult = await db.select({ count: sql<number>`count(*)` }).from(audits); |
| 40 | + const totalAudits = Number(totalAuditsResult[0].count); |
| 41 | + |
| 42 | + // 4. Audit Orders & Revenue |
| 43 | + const auditOrdersResult = await db.select({ |
| 44 | + status: auditOrders.status, |
| 45 | + count: sql<number>`count(*)`, |
| 46 | + revenue: sql<number>`sum(${auditOrders.priceUsd})` |
| 47 | + }).from(auditOrders).groupBy(auditOrders.status); |
| 48 | + |
| 49 | + let totalRevenue = 0; |
| 50 | + let ordersSummary = ""; |
| 51 | + let payingCustomers = 0; |
| 52 | + if (auditOrdersResult.length === 0) { |
| 53 | + ordersSummary = "0 audit orders"; |
| 54 | + } else { |
| 55 | + ordersSummary = auditOrdersResult.map(row => { |
| 56 | + if (row.status === 'marked_paid_manually' || row.status === 'paid') { |
| 57 | + totalRevenue += Number(row.revenue || 0); |
| 58 | + payingCustomers += Number(row.count); |
| 59 | + } |
| 60 | + return `${row.count} ${row.status} ($${row.revenue || 0})`; |
| 61 | + }).join(", "); |
| 62 | + } |
| 63 | + |
| 64 | + // 5. Eval harness |
| 65 | + let evalNumbers = "Eval results file not found"; |
| 66 | + try { |
| 67 | + const evalPath = path.join(process.cwd(), "eval/results/latest.md"); |
| 68 | + if (fs.existsSync(evalPath)) { |
| 69 | + const evalContent = fs.readFileSync(evalPath, "utf-8"); |
| 70 | + // Extract Precision, Recall, F1 |
| 71 | + const precisionMatch = evalContent.match(/Precision\*\*:\s*([\d.]+%)/); |
| 72 | + const recallMatch = evalContent.match(/Recall\*\*:\s*([\d.]+%)/); |
| 73 | + const f1Match = evalContent.match(/F1 Score\*\*:\s*([\d.]+%)/); |
| 74 | + const bypassMatch = evalContent.match(/Bypass Rate\*\*:\s*(.*)/); |
| 75 | + |
| 76 | + evalNumbers = `Precision: ${precisionMatch?.[1] || 'N/A'}, Recall: ${recallMatch?.[1] || 'N/A'}, F1: ${f1Match?.[1] || 'N/A'}\nSafety Bypass Rate: ${bypassMatch?.[1] || 'N/A'}`; |
| 77 | + } |
| 78 | + } catch(e) { |
| 79 | + console.error("Failed to read eval results", e); |
| 80 | + } |
| 81 | + |
| 82 | + // 6. Cost observability |
| 83 | + const costResult = await db.select({ |
| 84 | + totalCost: sql<number>`sum(CAST(${apiUsageLog.costUsd} AS numeric))` |
| 85 | + }).from(apiUsageLog); |
| 86 | + const totalCost = Number(costResult[0]?.totalCost || 0); |
| 87 | + |
| 88 | + const avgCostPerReview = totalReviews > 0 ? (totalCost / totalReviews).toFixed(4) : "0"; |
| 89 | + |
| 90 | + const today = new Date().toISOString().split('T')[0]; |
| 91 | + const snapshotPath = path.join(process.cwd(), `docs/TRACTION_SNAPSHOT_${today}.md`); |
| 92 | + |
| 93 | + const snapshotContent = `# Traction Snapshot - ${today} |
| 94 | +
|
| 95 | +## Users & Activity |
| 96 | +- Total registered users: ${totalUsers} |
| 97 | +- Active users (last 7 days): ${activeUsers7d} |
| 98 | +- Active users (last 30 days): ${activeUsers30d} |
| 99 | +${activeUsers7d === 0 ? '> Fastest path to non-zero: Convert one of the eval corpus source repos maintainers into a free pilot this week.' : ''} |
| 100 | +
|
| 101 | +## Repositories |
| 102 | +- Total repositories connected: ${totalRepos} |
| 103 | +${totalRepos === 0 ? '> Fastest path to non-zero: Connect the CodeGuard repo itself and 2 open-source projects owned by the founders.' : ''} |
| 104 | +
|
| 105 | +## Product Usage |
| 106 | +- Total PR reviews run: ${totalReviews} |
| 107 | +- Total Audit Mode runs completed: ${totalAudits} |
| 108 | +
|
| 109 | +## Revenue & Orders |
| 110 | +- Audit Orders: ${ordersSummary} |
| 111 | +- Total Revenue: $${totalRevenue} |
| 112 | +- Paying Customers: ${payingCustomers} |
| 113 | +${payingCustomers === 0 ? '> Fastest path to non-zero: Convert one pilot user via the manual-comp flow already built in the pricing mission, or manually invoice a design partner.' : ''} |
| 114 | +
|
| 115 | +## Evaluation Metrics (latest.md) |
| 116 | +${evalNumbers} |
| 117 | +
|
| 118 | +## Unit Economics |
| 119 | +- Average cost per review/audit: $${avgCostPerReview} (Total cost: $${totalCost.toFixed(4)}) |
| 120 | +
|
| 121 | +*Note: This snapshot is generated directly from live database metrics.* |
| 122 | +`; |
| 123 | + |
| 124 | + fs.mkdirSync(path.join(process.cwd(), "docs"), { recursive: true }); |
| 125 | + fs.writeFileSync(snapshotPath, snapshotContent); |
| 126 | + console.log(`Snapshot written to docs/TRACTION_SNAPSHOT_${today}.md`); |
| 127 | + process.exit(0); |
| 128 | +} |
| 129 | + |
| 130 | +generateSnapshot().catch(console.error); |
0 commit comments