Merge pull request #158 from laurentftech/fix/test-coverage-scenario-counting

fix(test-coverage): count only real scenarios, dedupe, fix the percentage

Author: clay-good

src/core/test-generator/coverage-analyzer.test.ts

@@ -162,6 +162,48 @@ describe("test 3") {}
     expect(report.byDomain['auth']?.hasDrift).toBe(true);
   });
 
+  it('ignores tags pointing at scenarios that do not exist in the parsed specs', async () => {
+    // Regression: example/fixture tags (e.g. a foreign "billing" domain) living
+    // in the test suite must NOT count as coverage, must not appear in `covered`,
+    // and must not inflate coveragePercent.
+    const testDir = join(tmpDir, 'spec-tests', 'auth');
+    await mkdir(testDir, { recursive: true });
+    await writeFile(
+      join(testDir, 'mixed.spec.ts'),
+      [
+        // real
+        '// openlore: {"domain":"auth","requirement":"UserLogin","scenario":"SuccessfulLogin"}',
+        // bogus — domain/requirement/scenario not in AUTH_SPEC
+        '// openlore: {"domain":"billing","requirement":"Invoice","scenario":"Paid"}',
+        '// openlore: {"domain":"auth","requirement":"UserLogin","scenario":"DoesNotExist"}',
+      ].join('\n')
+    );
+
+    const report = await analyzeTestCoverage({ rootPath: tmpDir, testDirs: ['spec-tests'] });
+
+    expect(report.coveredScenarios).toBe(1); // only the real one
+    expect(report.coveragePercent).toBe(33.3); // 1 / 3, not 3/3
+    expect(report.covered.map((c) => c.scenarioName)).toEqual(['SuccessfulLogin']);
+    expect(report.covered.some((c) => c.domain === 'billing')).toBe(false);
+    // invariant: covered + uncovered = total
+    expect(report.coveredScenarios + report.uncovered.length).toBe(report.totalScenarios);
+  });
+
+  it('dedupes a scenario tagged by multiple files', async () => {
+    const testDir = join(tmpDir, 'spec-tests', 'auth');
+    await mkdir(testDir, { recursive: true });
+    const tag = '// openlore: {"domain":"auth","requirement":"UserLogin","scenario":"SuccessfulLogin"}';
+    await writeFile(join(testDir, 'a.spec.ts'), tag + '\ndescribe("a") {}');
+    await writeFile(join(testDir, 'b.spec.ts'), tag + '\ndescribe("b") {}');
+
+    const report = await analyzeTestCoverage({ rootPath: tmpDir, testDirs: ['spec-tests'] });
+
+    expect(report.coveredScenarios).toBe(1);
+    expect(report.taggedScenarios).toBe(1);
+    expect(report.covered).toHaveLength(1);
+    expect(report.byDomain['auth'].covered).toBe(1);
+  });
+
   it('supports Python # openlore: tags', async () => {
     const testDir = join(tmpDir, 'spec-tests', 'auth');
     await mkdir(testDir, { recursive: true });

src/core/test-generator/coverage-analyzer.ts

@@ -279,14 +279,32 @@ export async function analyzeTestCoverage(opts: {
   }
 
   // ── 5. Build final covered / uncovered sets ──────────────────────────────
-  const allCovered: CoveredScenario[] = [...tagCovered];
-  for (const [, entry] of semanticCovered) {
-    allCovered.push(entry);
-  }
-
-  const allCoveredKeys = new Set(
-    allCovered.map((c) => `${c.domain}::${c.requirement}::${c.scenarioName}`)
+  // Coverage is counted ONLY against scenarios that actually exist in the parsed
+  // specs. Two guards:
+  //   - drop tags whose scenario isn't a real parsed scenario (e.g. example/
+  //     fixture tags living inside the test suite itself, like the auth specs in
+  //     this analyzer's own tests) — otherwise they inflate the count and make
+  //     `covered + uncovered ≠ total`.
+  //   - dedupe by scenario key so several files tagging the same scenario, or a
+  //     repeated tag, count once.
+  const scenarioKeys = new Set(
+    allScenarios.map((s) => `${s.domain}::${s.requirement}::${s.scenarioName}`)
   );
+  const keyOf = (c: CoveredScenario): string =>
+    `${c.domain}::${c.requirement}::${c.scenarioName}`;
+
+  // tag entries first, then semantic — first write wins on dedupe, so a tagged
+  // scenario keeps its tag attribution.
+  const rawCovered: CoveredScenario[] = [...tagCovered, ...semanticCovered.values()];
+  const allCovered: CoveredScenario[] = [];
+  const allCoveredKeys = new Set<string>();
+  for (const c of rawCovered) {
+    const k = keyOf(c);
+    if (!scenarioKeys.has(k)) continue; // not a real scenario — ignore
+    if (allCoveredKeys.has(k)) continue; // already counted
+    allCoveredKeys.add(k);
+    allCovered.push(c);
+  }
 
   const uncovered: UncoveredScenario[] = allScenarios
     .filter((s) => !allCoveredKeys.has(`${s.domain}::${s.requirement}::${s.scenarioName}`))
@@ -328,10 +346,13 @@ export async function analyzeTestCoverage(opts: {
   }
 
   // ── 8. Totals ────────────────────────────────────────────────────────────
+  // All derived from the deduped, real-scenario-only `allCovered`, so the
+  // invariants hold: coveredScenarios = taggedScenarios + discoveredScenarios,
+  // and coveredScenarios + uncovered.length = totalScenarios.
   const totalScenarios = allScenarios.length;
-  const taggedScenarios = tagCovered.length;
-  const discoveredScenarios = semanticCovered.size;
   const coveredScenarios = allCovered.length;
+  const taggedScenarios = allCovered.filter((c) => c.discoveredBy === 'tag').length;
+  const discoveredScenarios = allCovered.filter((c) => c.discoveredBy === 'semantic').length;
   const coveragePercent =
     totalScenarios === 0
       ? 0