[Feature] Add scene comparison report. (Resolves #43)

lib/src/goldens/golden_comparisons.dart

@@ -11,8 +11,8 @@ GoldenCollectionMismatches compareGoldenCollections(
   // For every golden, look for missing and mismatching screenshots.
   for (final id in goldens.ids) {
     if (!screenshots.hasId(id)) {
-      mismatches[id] = MissingGoldenMismatch(
-        golden: goldens[id],
+      mismatches[id] = MissingCandidateMismatch(
+        golden: goldens[id]!,
       );
       continue;
     }
@@ -44,7 +44,7 @@ GoldenCollectionMismatches compareGoldenCollections(
   for (final id in screenshots.ids) {
     if (!goldens.hasId(id)) {
       mismatches[id] = MissingGoldenMismatch(
-        screenshot: screenshots[id],
+        screenshot: screenshots[id]!,
       );
     }
   }
@@ -132,22 +132,29 @@ class WrongSizeGoldenMismatch extends GoldenMismatch {
   Map<String, dynamic> get describeStructured => throw UnimplementedError();
 }
 
-/// Attempted to compare a screenshot to a golden, but either the screenshot was never
-/// generated, or the screenshot was generated for a golden that doesn't exist.
+/// Attempted to compare a candidate to a golden, but the candidate was generated for a golden that doesn't exist.
 class MissingGoldenMismatch extends GoldenMismatch {
   MissingGoldenMismatch({
-    super.golden,
-    super.screenshot,
-  });
+    required GoldenImage screenshot,
+  }) : super(screenshot: screenshot);
 
-  GoldenImage get _existingGolden => golden ?? screenshot!;
+  @override
+  String get describe =>
+      "A new screenshot was generated with ID '${screenshot!.id}', but there's no existing golden image with that ID.";
 
   @override
-  String get describe => "A new screenshot was generated with ID '${_existingGolden.id}', $_missingMessage";
+  Map<String, dynamic> get describeStructured => throw UnimplementedError();
+}
+
+/// Attempted to compare a candidante to a golden, but the candidate was never generated.
+class MissingCandidateMismatch extends GoldenMismatch {
+  MissingCandidateMismatch({
+    required GoldenImage golden,
+  }) : super(golden: golden);
 
-  String get _missingMessage => golden != null //
-      ? "but no screenshot was generated with that ID."
-      : "but there's no existing golden image with that ID.";
+  @override
+  String get describe =>
+      "A new screenshot was generated with ID '${golden!.id}', but no screenshot was generated with that ID.";
 
   @override
   Map<String, dynamic> get describeStructured => throw UnimplementedError();

lib/src/scenes/failure_scene.dart

@@ -0,0 +1,83 @@
+import 'dart:math';
+import 'dart:ui' as ui;
+
+import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart';
+import 'package:image/image.dart';
+
+import 'package:flutter_test_goldens/flutter_test_goldens.dart';
+
+/// Given a [mismatch] between a golden and a screenshot, generates an image
+/// that shows the golden, the screenshot, and the differences between them.
+Future<Image> generateFailureScene(GoldenMismatch mismatch) async {
+  final goldenWidth = mismatch.golden!.image.width;
+  final goldenHeight = mismatch.golden!.image.height;
+
+  final screenshotWidth = mismatch.screenshot!.image.width;
+  final screenshotHeight = mismatch.screenshot!.image.height;
+
+  final maxWidth = max(goldenWidth, screenshotWidth);
+  final maxHeight = max(goldenHeight, screenshotHeight);
+
+  final failureImage = Image(
+    width: maxWidth * 2,
+    height: maxHeight * 2,
+  );
+
+  // Copy golden to top left corner.
+  for (int x = 0; x < goldenWidth; x += 1) {
+    for (int y = 0; y < goldenHeight; y += 1) {
+      final goldenPixel = mismatch.golden!.image.getPixel(x, y);
+      failureImage.setPixel(x, y, goldenPixel);
+    }
+  }
+
+  // Copy screenshot to top right corner.
+  for (int x = 0; x < screenshotWidth; x += 1) {
+    for (int y = 0; y < screenshotHeight; y += 1) {
+      final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y);
+      failureImage.setPixel(maxWidth + x, y, screenshotPixel);
+    }
+  }
+
+  // Paint mismatch images.
+  final absoluteDiffColor = ColorUint32.rgb(255, 255, 0);
+  for (int x = 0; x < maxWidth; x += 1) {
+    for (int y = 0; y < maxHeight; y += 1) {
+      if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) {
+        // This pixel doesn't exist in the golden, or it doesn't exist in the
+        // screenshot. Therefore, we have nothing to compare. Treat this pixel
+        // as a max severity difference.
+
+        // Paint this pixel in the absolute diff image.
+        failureImage.setPixel(x, maxHeight + y, absoluteDiffColor);
+
+        // Paint this pixel in the relative severity diff image.
+        failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor);
+
+        continue;
+      }
+
+      // Check if the screenshot matches the golden.
+      final goldenPixel = mismatch.golden!.image.getPixel(x, y);
+      final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y);
+      final pixelsMatch = goldenPixel == screenshotPixel;
+      if (pixelsMatch) {
+        continue;
+      }
+
+      // Paint this pixel in the absolute diff image.
+      failureImage.setPixel(x, maxHeight + y, absoluteDiffColor);
+
+      // Paint this pixel in the relative severity diff image.
+      final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel);
+      final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!;
+      failureImage.setPixel(
+        goldenWidth + x,
+        goldenHeight + y,
+        ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0),
+      );
+    }
+  }
+
+  return failureImage;
+}

lib/src/scenes/gallery.dart

@@ -1,6 +1,5 @@
 import 'dart:convert';
 import 'dart:io';
-import 'dart:math';
 import 'dart:ui' as ui;
 
 import 'package:flutter/foundation.dart';
@@ -13,11 +12,12 @@ import 'package:flutter_test_goldens/src/goldens/golden_collections.dart';
 import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart';
 import 'package:flutter_test_goldens/src/goldens/golden_rendering.dart';
 import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart';
-import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart';
 import 'package:flutter_test_goldens/src/logging.dart';
 import 'package:flutter_test_goldens/src/png/png_metadata.dart';
+import 'package:flutter_test_goldens/src/scenes/failure_scene.dart';
 import 'package:flutter_test_goldens/src/scenes/golden_files.dart';
 import 'package:flutter_test_goldens/src/scenes/golden_scene.dart';
+import 'package:flutter_test_goldens/src/scenes/golden_scene_report_printer.dart';
 import 'package:flutter_test_goldens/src/scenes/scene_layout.dart';
 import 'package:image/image.dart';
 import 'package:path/path.dart';
@@ -471,102 +471,98 @@ class Gallery {
     // Compare goldens in the scene.
     FtgLog.pipeline.fine("Comparing goldens and screenshots");
     final mismatches = compareGoldenCollections(goldenCollection, screenshotCollection);
-    if (mismatches.mismatches.isNotEmpty) {
-      FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):");
-      for (final mismatch in mismatches.mismatches.values) {
-        FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch");
+
+    final items = <GoldenReportItem>[];
+    final missingCandidates = <MissingCandidateMismatch>[];
+    final extraCandidates = <MissingGoldenMismatch>[];
+
+    int totalPassed = 0;
+    int totalFailed = 0;
+
+    FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):");
+    for (final mismatch in mismatches.mismatches.values) {
+      FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch");
+      switch (mismatch) {
+        case MissingCandidateMismatch():
+          // A golden candidate is missing.
+          missingCandidates.add(mismatch);
+          break;
+        case MissingGoldenMismatch():
+          // We have a golden candidate, but not the original golden.
+          extraCandidates.add(mismatch);
+          break;
       }
+    }
 
-      for (final mismatch in mismatches.mismatches.values) {
-        if (mismatch.golden == null || mismatch.screenshot == null) {
-          continue;
-        }
-
-        FtgLog.pipeline.fine("Painting a golden failure: $mismatch");
-        Directory(_goldenFailureDirectoryPath).createSync();
-
-        await tester.runAsync(() async {
-          final goldenWidth = mismatch.golden!.image.width;
-          final goldenHeight = mismatch.golden!.image.height;
-
-          final screenshotWidth = mismatch.screenshot!.image.width;
-          final screenshotHeight = mismatch.screenshot!.image.height;
-
-          final maxWidth = max(goldenWidth, screenshotWidth);
-          final maxHeight = max(goldenHeight, screenshotHeight);
-
-          final failureImage = Image(
-            width: maxWidth * 2,
-            height: maxHeight * 2,
-          );
-
-          // Copy golden to top left corner.
-          for (int x = 0; x < goldenWidth; x += 1) {
-            for (int y = 0; y < goldenHeight; y += 1) {
-              final goldenPixel = mismatch.golden!.image.getPixel(x, y);
-              failureImage.setPixel(x, y, goldenPixel);
-            }
-          }
-
-          // Copy screenshot to top right corner.
-          for (int x = 0; x < screenshotWidth; x += 1) {
-            for (int y = 0; y < screenshotHeight; y += 1) {
-              final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y);
-              failureImage.setPixel(maxWidth + x, y, screenshotPixel);
-            }
-          }
-
-          // Paint mismatch images.
-          final absoluteDiffColor = ColorUint32.rgb(255, 255, 0);
-          for (int x = 0; x < maxWidth; x += 1) {
-            for (int y = 0; y < maxHeight; y += 1) {
-              if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) {
-                // This pixel doesn't exist in the golden, or it doesn't exist in the
-                // screenshot. Therefore, we have nothing to compare. Treat this pixel
-                // as a max severity difference.
-
-                // Paint this pixel in the absolute diff image.
-                failureImage.setPixel(x, maxHeight + y, absoluteDiffColor);
-
-                // Paint this pixel in the relative severity diff image.
-                failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor);
-
-                continue;
-              }
-
-              // Check if the screenshot matches the golden.
-              final goldenPixel = mismatch.golden!.image.getPixel(x, y);
-              final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y);
-              final pixelsMatch = goldenPixel == screenshotPixel;
-              if (pixelsMatch) {
-                continue;
-              }
-
-              // Paint this pixel in the absolute diff image.
-              failureImage.setPixel(x, maxHeight + y, absoluteDiffColor);
-
-              // Paint this pixel in the relative severity diff image.
-              final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel);
-              final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!;
-              failureImage.setPixel(
-                goldenWidth + x,
-                goldenHeight + y,
-                ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0),
-              );
-            }
-          }
-
-          await encodePngFile(
-            "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png",
-            failureImage,
-          );
-        });
+    // For each candidate found in the scene, report whether it passed or failed.
+    for (final screenshotId in screenshotCollection.ids) {
+      if (!goldenCollection.hasId(screenshotId)) {
+        // This candidate is an extra candidate, i.e., it was found in the scene,
+        // but it doesn't have a golden counterpart. We already reported extra candidates
+        // above, so we can skip this candidate.
+        continue;
       }
 
-      throw Exception("Goldens failed with ${mismatches.mismatches.length} mismatch(es)");
-    } else {
+      final mismatch = mismatches.mismatches[screenshotId];
+      if (mismatch == null) {
+        // The golden check passed.
+        totalPassed += 1;
+        items.add(
+          GoldenReportItem.success(description: goldenCollection.imagesById[screenshotId]!.id),
+        );
+      } else {
+        // The golden check failed.
+        totalFailed += 1;
+        items.add(
+          GoldenReportItem.failure(
+            description: goldenCollection.imagesById[screenshotId]!.id,
+            details: [
+              GoldenCheckDetail(
+                status: GoldenTestStatus.failure,
+                description: mismatch.toString(),
+                mismatch: mismatch,
+              ),
+            ],
+          ),
+        );
+      }
+    }
+
+    if (mismatches.mismatches.isEmpty) {
       FtgLog.pipeline.info("No golden mismatches found");
     }
+
+    for (final mismatch in mismatches.mismatches.values) {
+      if (mismatch.golden == null || mismatch.screenshot == null) {
+        continue;
+      }
+
+      FtgLog.pipeline.fine("Painting a golden failure: $mismatch");
+      Directory(_goldenFailureDirectoryPath).createSync();
+
+      await tester.runAsync(() async {
+        final failureImage = await generateFailureScene(mismatch);
+
+        await encodePngFile(
+          "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png",
+          failureImage,
+        );
+      });
+    }
+
+    final report = GoldenSceneReport(
+      sceneDescription: _sceneDescription,
+      items: items,
+      missingCandidates: missingCandidates,
+      extraCandidates: extraCandidates,
+      totalPassed: totalPassed,
+      totalFailed: totalFailed,
+    );
+    _printReport(report);
+
+    if (mismatches.mismatches.isNotEmpty) {
+      fail("Goldens failed with ${mismatches.mismatches.length} mismatch(es)");
+    }
   }
 
   String get _testFileDirectory => (goldenFileComparator as LocalFileComparator).basedir.path;
@@ -580,6 +576,11 @@ class Gallery {
       "$_goldenDirectory$_fileName${includeExtension ? ".png" : ""}";
 
   String get _goldenFailureDirectoryPath => "${_goldenDirectory}failures";
+
+  /// Prints the report in an human readable format to the console.
+  void _printReport(GoldenSceneReport report) {
+    GoldenSceneReportPrinter().printReport(report);
+  }
 }
 
 /// Pumps a widget tree into the given [tester], wrapping its content within the given [decorator].