Handle screenshot errors in unit tests

Recovers image diff view in test failures:
https://screenshot.googleplex.com/9dKe3HUNzbj6Zpz

Fixed: 408112810
Change-Id: Ieb588c8cfb18d63549b179620809a70a1010a753
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6426041
Reviewed-by: Philip Pfaffe <[email protected]>
Commit-Queue: Alex Rudenko <[email protected]>

Author: OrKoN

front_end/testing/DOMHelpers.ts

@@ -331,8 +331,8 @@ export async function assertScreenshot(filename: string) {
   }
   await raf();
   // @ts-expect-error see karma config.
-  const result = await window.assertScreenshot(`#${TEST_CONTAINER_ID}`, filename);
-  if (result) {
-    throw new Error(result);
+  const errorMessage = await window.assertScreenshot(`#${TEST_CONTAINER_ID}`, filename);
+  if (errorMessage) {
+    throw new Error(errorMessage);
   }
 }

test/conductor/karma-resultsdb-reporter.ts

@@ -5,6 +5,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import * as ResultsDb from './resultsdb.js';
+import {ScreenshotError} from './screenshot-error.js';
 
 const chalk = require('chalk');
 const diff = require('diff');
@@ -113,6 +114,19 @@ export const ResultsDBReporter = function(
     }
 
     const testResult: ResultsDb.TestResult = {testId, duration, status, expected, summaryHtml};
+
+    if (result.log?.[0]?.startsWith('Error: ScreenshotError')) {
+      const screenshotError = ScreenshotError.errors.shift();
+      if (screenshotError) {
+        // Assert that the screenshot error matches the log.
+        // If it does not, it means something is wrong
+        // with the order of assertions and tests.«
+        if (!result.log?.[0]?.includes(screenshotError.message)) {
+          throw new Error('Unexpected screenshot assertion error');
+        }
+        [testResult.artifacts, testResult.summaryHtml] = screenshotError.toMiloArtifacts();
+      }
+    }
     ResultsDb.sendTestResult(testResult);
   };
   this.specSuccess = specComplete;

test/conductor/screenshot-error.ts

@@ -17,6 +17,7 @@ export class ScreenshotError extends Error {
   // The max length of the summary is 4000, but we need to leave some room for
   // the rest of the HTML formatting (e.g. <pre> and </pre>).
   static readonly SUMMARY_LENGTH_CUTOFF = 3900;
+  static errors: ScreenshotError[] = [];
   readonly screenshots: ArtifactGroup;
 
   private constructor(screenshots: ArtifactGroup, message?: string, cause?: Error) {

test/unit/karma.conf.ts

@@ -10,6 +10,7 @@ import {formatAsPatch, resultAssertionsDiff, ResultsDBReporter} from '../../test
 import {CHECKOUT_ROOT, GEN_DIR, SOURCE_ROOT} from '../../test/conductor/paths.js';
 import * as ResultsDb from '../../test/conductor/resultsdb.js';
 import {loadTests, TestConfig} from '../../test/conductor/test_config.js';
+import {ScreenshotError} from '../conductor/screenshot-error.js';
 import {assertElementScreenshotUnchanged} from '../shared/screenshots.js';
 
 const puppeteer = require('puppeteer-core');
@@ -61,8 +62,12 @@ const CustomChrome = function(this: any, _baseBrowserDecorator: unknown, args: B
         await assertElementScreenshotUnchanged(element, filename, {
           captureBeyondViewport: false,
         });
-      } catch (err) {
-        return err.message;
+        return undefined;
+      } catch (error) {
+        if (error instanceof ScreenshotError) {
+          ScreenshotError.errors.push(error);
+        }
+        return `ScreenshotError: ${error.message}`;
       }
     });