fix: validate scope checks ran and report overall test status (#80)

* fix: validate scope checks ran and report overall test status

- Add FAILURE checks to scope scenarios when authorization flow doesn't
  complete (ScopeFromWwwAuthenticateScenario, ScopeFromScopesSupportedScenario,
  ScopeOmittedWhenUndefinedScenario)
- Report client timeout and exit errors in test results
- Add OVERALL: PASSED/FAILED summary at end of test output
- Exit with code 1 on any failure (check failures, client timeout, or exit error)

Previously, if the OAuth authorization flow didn't complete, the scope
checks were never triggered and the test would incorrectly report success.
Now these scenarios emit a FAILURE if the expected scope check wasn't recorded.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* Count warnings as failures in overall test result

Warnings now contribute to the overall failure status, causing the test
to exit with code 1 when any warnings are present. Also displays warning
checks in the output summary, similar to how failures are displayed.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* Count warnings as failures in suite runner

Update the suite summary to show ✗ for scenarios with warnings,
and exit with code 1 if any warnings are present across the suite.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

---------

Co-authored-by: Claude <[email protected]>

Author: pcarleton

src/index.ts

@@ -125,7 +125,7 @@ program
           totalFailed += failed;
           totalWarnings += warnings;
 
-          const status = failed === 0 ? '✓' : '✗';
+          const status = failed === 0 && warnings === 0 ? '✓' : '✗';
           const warningStr = warnings > 0 ? `, ${warnings} warnings` : '';
           console.log(
             `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}`
@@ -145,7 +145,7 @@ program
         console.log(
           `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings`
         );
-        process.exit(totalFailed > 0 ? 1 : 0);
+        process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0);
       }
 
       // Require either --scenario or --suite
@@ -173,8 +173,12 @@ program
         timeout
       );
 
-      const { failed } = printClientResults(result.checks, verbose);
-      process.exit(failed > 0 ? 1 : 0);
+      const { overallFailure } = printClientResults(
+        result.checks,
+        verbose,
+        result.clientOutput
+      );
+      process.exit(overallFailure ? 1 : 0);
     } catch (error) {
       if (error instanceof ZodError) {
         console.error('Validation error:');

src/runner/client.ts

@@ -150,15 +150,30 @@ export async function runConformanceTest(
 
 export function printClientResults(
   checks: ConformanceCheck[],
-  verbose: boolean = false
-): { passed: number; failed: number; denominator: number; warnings: number } {
+  verbose: boolean = false,
+  clientOutput?: ClientExecutionResult
+): {
+  passed: number;
+  failed: number;
+  denominator: number;
+  warnings: number;
+  overallFailure: boolean;
+} {
   const denominator = checks.filter(
     (c) => c.status === 'SUCCESS' || c.status === 'FAILURE'
   ).length;
   const passed = checks.filter((c) => c.status === 'SUCCESS').length;
   const failed = checks.filter((c) => c.status === 'FAILURE').length;
   const warnings = checks.filter((c) => c.status === 'WARNING').length;
 
+  // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure)
+  const clientTimedOut = clientOutput?.timedOut ?? false;
+  const clientExitedWithError = clientOutput
+    ? clientOutput.exitCode !== 0
+    : false;
+  const overallFailure =
+    failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError;
+
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless
     console.log(JSON.stringify(checks, null, 2));
@@ -173,6 +188,16 @@ export function printClientResults(
     `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings`
   );
 
+  if (clientTimedOut) {
+    console.error(`\n⚠️  CLIENT TIMED OUT - Test incomplete`);
+  }
+
+  if (clientExitedWithError && !clientTimedOut) {
+    console.error(
+      `\n⚠️  CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete`
+    );
+  }
+
   if (failed > 0) {
     console.error('\nFailed Checks:');
     checks
@@ -185,7 +210,25 @@ export function printClientResults(
       });
   }
 
-  return { passed, failed, denominator, warnings };
+  if (warnings > 0) {
+    console.error('\nWarning Checks:');
+    checks
+      .filter((c) => c.status === 'WARNING')
+      .forEach((c) => {
+        console.error(`  - ${c.name}: ${c.description}`);
+        if (c.errorMessage) {
+          console.error(`    Warning: ${c.errorMessage}`);
+        }
+      });
+  }
+
+  if (overallFailure) {
+    console.error('\n❌ OVERALL: FAILED');
+  } else {
+    console.error('\n✅ OVERALL: PASSED');
+  }
+
+  return { passed, failed, denominator, warnings, overallFailure };
 }
 
 export async function runInteractiveMode(

src/scenarios/client/auth/scope-handling.ts

@@ -73,6 +73,21 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-www-authenticate'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-www-authenticate',
+        name: 'Client scope selection from WWW-Authenticate header',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -153,6 +168,21 @@ export class ScopeFromScopesSupportedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-scopes-supported'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-scopes-supported',
+        name: 'Client scope selection from scopes_supported',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -221,6 +251,21 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-omitted-when-undefined'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-omitted-when-undefined',
+        name: 'Client scope omission when scopes_supported undefined',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }