✨ Add --fail-on-diff option and improve SDK resilience (#169)

## Summary

- Visual diffs no longer break tests by default - SDKs can now choose
whether to fail on diffs
- Server returns 200 for diffs with `status: 'diff'` instead of 422
error
- Added `--fail-on-diff` CLI flag for `vizzly tdd start` and `vizzly tdd
run`

## Changes

**Server:**
- Visual diffs return `200` with `status: 'diff'` instead of `422`
- Consistent response format with `status` field: `'match'`, `'diff'`,
`'new'`, or `'baseline-updated'`

**CLI:**
- `--fail-on-diff` flag on `vizzly tdd start` and `vizzly tdd run`
- `failOnDiff` stored in `server.json` for SDK discovery

**Ember SDK:**
- Read `failOnDiff` from server.json, env var, or per-snapshot option
- Gracefully skip snapshots when no server running (like Percy)
- `failOnDiff` option on `vizzlySnapshot()` for per-snapshot control

## Usage

```bash
# Default: diffs logged as warnings, tests pass
vizzly tdd start

# Fail tests on visual diffs
vizzly tdd start --fail-on-diff

# Or via environment variable
VIZZLY_FAIL_ON_DIFF=true npm test

# Or per-snapshot
await vizzlySnapshot('critical-ui', { failOnDiff: true });
```

## Test plan

- [x] Updated `tdd-handler.test.js` for new 200 response format
- [x] Added tests for diff response structure
- [x] Added tests for `getServerInfo()` reading `failOnDiff`
- [x] All CLI tests pass
- [x] All Ember SDK tests pass

Author: Robdel12

clients/ember/bin/vizzly-browser.js

@@ -16,6 +16,7 @@
 
 import { closeBrowser, launchBrowser } from '../src/launcher/browser.js';
 import {
+  getServerInfo,
   setPage,
   startSnapshotServer,
   stopSnapshotServer,
@@ -66,15 +67,28 @@ async function cleanup() {
  */
 async function main() {
   try {
-    // 1. Start snapshot server first
+    // 1. Start snapshot server first (this also discovers the TDD server and caches its config)
     snapshotServer = await startSnapshotServer();
     let snapshotUrl = `http://127.0.0.1:${snapshotServer.port}`;
 
-    // 2. Launch browser with Playwright
+    // 2. Determine failOnDiff: env var > server.json > default (false)
+    // getServerInfo() returns cached info from the TDD server discovery that happened above
+    let failOnDiff = false;
+    if (process.env.VIZZLY_FAIL_ON_DIFF === 'true' || process.env.VIZZLY_FAIL_ON_DIFF === '1') {
+      failOnDiff = true;
+    } else {
+      let serverInfo = getServerInfo();
+      if (serverInfo?.failOnDiff) {
+        failOnDiff = true;
+      }
+    }
+
+    // 3. Launch browser with Playwright
     // Note: We set the page reference in launchBrowser before navigation
     // to avoid a race condition where tests run before page is set
     browserInstance = await launchBrowser(browserType, testUrl, {
       snapshotUrl,
+      failOnDiff,
       onPageCreated: page => {
         // Set page reference immediately when page is created
         // This happens BEFORE navigation so tests can capture screenshots

clients/ember/src/launcher/browser.js

@@ -61,11 +61,12 @@ function getChromiumArgs() {
  * @param {Object} options - Launch options
  * @param {string} options.snapshotUrl - URL of the snapshot HTTP server
  * @param {boolean} [options.headless] - Run in headless mode (default: true in CI)
+ * @param {boolean} [options.failOnDiff] - Whether tests should fail on visual diffs
  * @param {Function} [options.onPageCreated] - Callback when page is created (before navigation)
  * @returns {Promise<Object>} Browser instance with page reference
  */
 export async function launchBrowser(browserType, testUrl, options = {}) {
-  let { snapshotUrl, headless, onPageCreated } = options;
+  let { snapshotUrl, headless, failOnDiff, onPageCreated } = options;
 
   // Default headless based on CI environment
   if (headless === undefined) {
@@ -93,11 +94,15 @@ export async function launchBrowser(browserType, testUrl, options = {}) {
   let context = await browser.newContext();
   let page = await context.newPage();
 
-  // Inject snapshot URL into page context BEFORE navigation
-  // This ensures window.__VIZZLY_SNAPSHOT_URL__ is available when tests run
-  await page.addInitScript(url => {
-    window.__VIZZLY_SNAPSHOT_URL__ = url;
-  }, snapshotUrl);
+  // Inject Vizzly config into page context BEFORE navigation
+  // This ensures window.__VIZZLY_* is available when tests run
+  await page.addInitScript(
+    ({ snapshotUrl, failOnDiff }) => {
+      window.__VIZZLY_SNAPSHOT_URL__ = snapshotUrl;
+      window.__VIZZLY_FAIL_ON_DIFF__ = failOnDiff;
+    },
+    { snapshotUrl, failOnDiff }
+  );
 
   // Call onPageCreated callback BEFORE navigation
   // This allows setting up the page reference before tests can run

clients/ember/src/launcher/snapshot-server.js

@@ -34,9 +34,14 @@ export function getPage() {
   return pageRef;
 }
 
+/**
+ * Cached server info from auto-discovery
+ */
+let cachedServerInfo = null;
+
 /**
  * Auto-discover the Vizzly TDD server by searching for .vizzly/server.json
- * @returns {string|null} Server URL or null if not found
+ * @returns {{url: string, failOnDiff: boolean}|null} Server info or null if not found
  */
 function autoDiscoverTddServer() {
   let currentDir = process.cwd();
@@ -49,7 +54,11 @@ function autoDiscoverTddServer() {
       try {
         let serverInfo = JSON.parse(readFileSync(serverJsonPath, 'utf8'));
         if (serverInfo.port) {
-          return `http://localhost:${serverInfo.port}`;
+          cachedServerInfo = {
+            url: `http://localhost:${serverInfo.port}`,
+            failOnDiff: serverInfo.failOnDiff || false,
+          };
+          return cachedServerInfo;
         }
       } catch {
         // Invalid JSON, continue searching
@@ -62,6 +71,26 @@ function autoDiscoverTddServer() {
   return null;
 }
 
+/**
+ * Get the cached server info (for reading failOnDiff setting)
+ * @returns {{url: string, failOnDiff: boolean}|null}
+ */
+export function getServerInfo() {
+  // If not cached yet, discover it now
+  if (!cachedServerInfo) {
+    autoDiscoverTddServer();
+  }
+  return cachedServerInfo;
+}
+
+/**
+ * Clear the cached server info (for testing purposes)
+ * @private
+ */
+export function clearServerInfoCache() {
+  cachedServerInfo = null;
+}
+
 /**
  * Forward screenshot to Vizzly TDD server
  * @param {string} name - Screenshot name
@@ -70,9 +99,9 @@ function autoDiscoverTddServer() {
  * @returns {Promise<Object>} Response from TDD server
  */
 async function forwardToVizzly(name, imageBuffer, properties = {}) {
-  let tddServerUrl = autoDiscoverTddServer();
+  let serverInfo = autoDiscoverTddServer();
 
-  if (!tddServerUrl) {
+  if (!serverInfo) {
     // Check for cloud mode via environment
     if (process.env.VIZZLY_TOKEN) {
       // In cloud mode, we'd queue for upload
@@ -85,6 +114,8 @@ async function forwardToVizzly(name, imageBuffer, properties = {}) {
     );
   }
 
+  let tddServerUrl = serverInfo.url;
+
   let payload = {
     name,
     image: imageBuffer.toString('base64'),

clients/ember/src/test-support/index.js

@@ -134,6 +134,16 @@ function prepareTestingContainer(width = 1280, height = 720, fullPage = false) {
   };
 }
 
+/**
+ * Check if tests should fail on visual diffs
+ * Reads from VIZZLY_FAIL_ON_DIFF environment variable (injected by launcher)
+ * @returns {boolean}
+ */
+function shouldFailOnDiff() {
+  return window.__VIZZLY_FAIL_ON_DIFF__ === true ||
+         window.__VIZZLY_FAIL_ON_DIFF__ === 'true';
+}
+
 /**
  * Capture a visual snapshot
  *
@@ -154,6 +164,7 @@ function prepareTestingContainer(width = 1280, height = 720, fullPage = false) {
  * @param {number} [options.height=720] - Viewport height for the snapshot
  * @param {string} [options.scope='app'] - What to capture: 'app' (default), 'container', or 'page'
  * @param {Object} [options.properties] - Additional metadata for the snapshot
+ * @param {boolean} [options.failOnDiff] - Fail the test if visual diff is detected (overrides env var)
  * @returns {Promise<Object>} Snapshot result from Vizzly server
  *
  * @example
@@ -169,8 +180,8 @@ function prepareTestingContainer(width = 1280, height = 720, fullPage = false) {
  * await vizzlySnapshot('login-form', { selector: '[data-test-login-form]' });
  *
  * @example
- * // Capture the entire page including QUnit UI (rare use case)
- * await vizzlySnapshot('test-runner', { scope: 'page' });
+ * // Fail test if this specific snapshot has a diff
+ * await vizzlySnapshot('critical-ui', { failOnDiff: true });
  */
 export async function vizzlySnapshot(name, options = {}) {
   let {
@@ -180,6 +191,7 @@ export async function vizzlySnapshot(name, options = {}) {
     height = 720,
     scope = 'app',
     properties = {},
+    failOnDiff = null, // null means use env var, true/false overrides
   } = options;
 
   // Get snapshot URL injected by the launcher
@@ -252,13 +264,31 @@ export async function vizzlySnapshot(name, options = {}) {
       // This allows tests to pass when Vizzly isn't running (like Percy behavior)
       if (errorText.includes('No Vizzly server found')) {
         console.warn('[vizzly] Vizzly server not running. Skipping visual snapshot.');
-        return { skipped: true, reason: 'no-server' };
+        return { status: 'skipped', reason: 'no-server' };
       }
 
       throw new Error(`Vizzly snapshot failed: ${errorText}`);
     }
 
-    return await response.json();
+    let result = await response.json();
+
+    // Handle visual diff - server returns 200 with status: 'diff'
+    if (result.status === 'diff') {
+      // Determine if we should fail based on option or env var
+      let shouldFail = failOnDiff !== null ? failOnDiff : shouldFailOnDiff();
+
+      if (shouldFail) {
+        throw new Error(
+          `Visual difference detected for '${name}' (${result.diffPercentage?.toFixed(2)}% diff). ` +
+          `View diff in Vizzly dashboard.`
+        );
+      }
+
+      // Log warning but don't fail
+      console.warn(`[vizzly] Visual difference detected for '${name}'. View diff in Vizzly dashboard.`);
+    }
+
+    return result;
   } finally {
     // Always restore original styles
     cleanup();

clients/ember/tests/unit/snapshot-server.test.js

@@ -1,7 +1,11 @@
 import assert from 'node:assert';
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { afterEach, beforeEach, describe, it } from 'node:test';
 import {
+  clearServerInfoCache,
   getPage,
+  getServerInfo,
   setPage,
   startSnapshotServer,
   stopSnapshotServer,
@@ -188,4 +192,106 @@ describe('snapshot-server', () => {
       assert.strictEqual(response.status, 404);
     });
   });
+
+  describe('getServerInfo()', () => {
+    let testDir = join(process.cwd(), '.vizzly-test-temp');
+
+    beforeEach(() => {
+      // Clear the cached server info before each test
+      clearServerInfoCache();
+
+      // Clean up any existing test directory
+      if (existsSync(testDir)) {
+        rmSync(testDir, { recursive: true });
+      }
+    });
+
+    afterEach(() => {
+      // Clear cache after tests
+      clearServerInfoCache();
+
+      // Clean up test directory
+      if (existsSync(testDir)) {
+        rmSync(testDir, { recursive: true });
+      }
+    });
+
+    it('returns null when no server.json exists in isolated directory', () => {
+      // Create an isolated test directory without server.json
+      mkdirSync(testDir, { recursive: true });
+
+      let originalCwd = process.cwd();
+      try {
+        process.chdir(testDir);
+        clearServerInfoCache();
+
+        let info = getServerInfo();
+
+        // In an isolated directory without .vizzly/server.json, should return null
+        // (unless there's a server.json in a parent directory)
+        if (info !== null) {
+          assert.ok(typeof info.url === 'string', 'should have url');
+          assert.ok(typeof info.failOnDiff === 'boolean', 'should have failOnDiff');
+        }
+      } finally {
+        process.chdir(originalCwd);
+      }
+    });
+
+    it('reads failOnDiff from server.json when present', () => {
+      // Create a temporary .vizzly directory with server.json
+      let vizzlyDir = join(testDir, '.vizzly');
+      mkdirSync(vizzlyDir, { recursive: true });
+
+      let serverJson = {
+        pid: 12345,
+        port: 47392,
+        startTime: Date.now(),
+        failOnDiff: true,
+      };
+      writeFileSync(join(vizzlyDir, 'server.json'), JSON.stringify(serverJson));
+
+      // Change to test directory to test discovery
+      let originalCwd = process.cwd();
+      try {
+        process.chdir(testDir);
+        clearServerInfoCache();
+
+        let info = getServerInfo();
+
+        assert.ok(info !== null, 'should find server.json');
+        assert.strictEqual(info.url, 'http://localhost:47392', 'should have correct url');
+        assert.strictEqual(info.failOnDiff, true, 'should read failOnDiff as true');
+      } finally {
+        process.chdir(originalCwd);
+      }
+    });
+
+    it('defaults failOnDiff to false when not specified in server.json', () => {
+      let vizzlyDir = join(testDir, '.vizzly');
+      mkdirSync(vizzlyDir, { recursive: true });
+
+      let serverJson = {
+        pid: 12345,
+        port: 47393,
+        startTime: Date.now(),
+        // failOnDiff not specified
+      };
+      writeFileSync(join(vizzlyDir, 'server.json'), JSON.stringify(serverJson));
+
+      let originalCwd = process.cwd();
+      try {
+        process.chdir(testDir);
+        clearServerInfoCache();
+
+        let info = getServerInfo();
+
+        assert.ok(info !== null, 'should find server.json');
+        assert.strictEqual(info.url, 'http://localhost:47393', 'should have correct url');
+        assert.strictEqual(info.failOnDiff, false, 'should default failOnDiff to false');
+      } finally {
+        process.chdir(originalCwd);
+      }
+    });
+  });
 });

src/cli.js

@@ -368,6 +368,7 @@ tddCmd
   .option('--environment <env>', 'Environment name', 'test')
   .option('--threshold <number>', 'Comparison threshold', parseFloat)
   .option('--timeout <ms>', 'Server timeout in milliseconds', '30000')
+  .option('--fail-on-diff', 'Fail tests when visual differences are detected')
   .option('--token <token>', 'API token override')
   .option('--daemon-child', 'Internal: run as daemon child process')
   .action(async options => {
@@ -416,6 +417,7 @@ tddCmd
     '--set-baseline',
     'Accept current screenshots as new baseline (overwrites existing)'
   )
+  .option('--fail-on-diff', 'Fail tests when visual differences are detected')
   .option('--no-open', 'Skip opening report in browser')
   .action(async (command, options) => {
     const globalOptions = program.opts();

src/commands/tdd-daemon.js

@@ -86,6 +86,7 @@ export async function tddStartCommand(options = {}, globalOptions = {}) {
           ? ['--threshold', options.threshold.toString()]
           : []),
         ...(options.timeout ? ['--timeout', options.timeout] : []),
+        ...(options.failOnDiff ? ['--fail-on-diff'] : []),
         ...(options.token ? ['--token', options.token] : []),
         ...(globalOptions.json ? ['--json'] : []),
         ...(globalOptions.verbose ? ['--verbose'] : []),
@@ -250,6 +251,7 @@ export async function runDaemonChild(options = {}, globalOptions = {}) {
       pid: process.pid,
       port: port,
       startTime: Date.now(),
+      failOnDiff: options.failOnDiff || false,
     };
     writeFileSync(
       join(vizzlyDir, 'server.json'),