diff --git a/lib/src/goldens/golden_comparisons.dart b/lib/src/goldens/golden_comparisons.dart index b8faddb..9b25d14 100644 --- a/lib/src/goldens/golden_comparisons.dart +++ b/lib/src/goldens/golden_comparisons.dart @@ -11,8 +11,8 @@ GoldenCollectionMismatches compareGoldenCollections( // For every golden, look for missing and mismatching screenshots. for (final id in goldens.ids) { if (!screenshots.hasId(id)) { - mismatches[id] = MissingGoldenMismatch( - golden: goldens[id], + mismatches[id] = MissingCandidateMismatch( + golden: goldens[id]!, ); continue; } @@ -44,7 +44,7 @@ GoldenCollectionMismatches compareGoldenCollections( for (final id in screenshots.ids) { if (!goldens.hasId(id)) { mismatches[id] = MissingGoldenMismatch( - screenshot: screenshots[id], + screenshot: screenshots[id]!, ); } } @@ -132,22 +132,29 @@ class WrongSizeGoldenMismatch extends GoldenMismatch { Map get describeStructured => throw UnimplementedError(); } -/// Attempted to compare a screenshot to a golden, but either the screenshot was never -/// generated, or the screenshot was generated for a golden that doesn't exist. +/// Attempted to compare a candidate to a golden, but the candidate was generated for a golden that doesn't exist. class MissingGoldenMismatch extends GoldenMismatch { MissingGoldenMismatch({ - super.golden, - super.screenshot, - }); + required GoldenImage screenshot, + }) : super(screenshot: screenshot); - GoldenImage get _existingGolden => golden ?? screenshot!; + @override + String get describe => + "A new screenshot was generated with ID '${screenshot!.id}', but there's no existing golden image with that ID."; @override - String get describe => "A new screenshot was generated with ID '${_existingGolden.id}', $_missingMessage"; + Map get describeStructured => throw UnimplementedError(); +} + +/// Attempted to compare a candidante to a golden, but the candidate was never generated. +class MissingCandidateMismatch extends GoldenMismatch { + MissingCandidateMismatch({ + required GoldenImage golden, + }) : super(golden: golden); - String get _missingMessage => golden != null // - ? "but no screenshot was generated with that ID." - : "but there's no existing golden image with that ID."; + @override + String get describe => + "A new screenshot was generated with ID '${golden!.id}', but no screenshot was generated with that ID."; @override Map get describeStructured => throw UnimplementedError(); diff --git a/lib/src/goldens/golden_scenes.dart b/lib/src/goldens/golden_scenes.dart index abe3a4d..a25ac05 100644 --- a/lib/src/goldens/golden_scenes.dart +++ b/lib/src/goldens/golden_scenes.dart @@ -128,6 +128,7 @@ RenderRepaintBoundary? _findNearestRepaintBoundary(Finder bounds) { class GoldenSceneMetadata { static GoldenSceneMetadata fromJson(Map json) { return GoldenSceneMetadata( + description: json["description"] ?? "", images: [ for (final imageJson in (json["images"] as List)) // GoldenImageMetadata.fromJson(imageJson), @@ -136,13 +137,16 @@ class GoldenSceneMetadata { } const GoldenSceneMetadata({ + required this.description, required this.images, }); + final String description; final List images; Map toJson() { return { + "description": description, "images": images.map((image) => image.toJson()).toList(growable: false), }; } diff --git a/lib/src/scenes/failure_scene.dart b/lib/src/scenes/failure_scene.dart new file mode 100644 index 0000000..c8ce071 --- /dev/null +++ b/lib/src/scenes/failure_scene.dart @@ -0,0 +1,83 @@ +import 'dart:math'; +import 'dart:ui' as ui; + +import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart'; +import 'package:image/image.dart'; + +import 'package:flutter_test_goldens/flutter_test_goldens.dart'; + +/// Given a [mismatch] between a golden and a screenshot, generates an image +/// that shows the golden, the screenshot, and the differences between them. +Future paintGoldenMismatchImages(GoldenMismatch mismatch) async { + final goldenWidth = mismatch.golden!.image.width; + final goldenHeight = mismatch.golden!.image.height; + + final screenshotWidth = mismatch.screenshot!.image.width; + final screenshotHeight = mismatch.screenshot!.image.height; + + final maxWidth = max(goldenWidth, screenshotWidth); + final maxHeight = max(goldenHeight, screenshotHeight); + + final failureImage = Image( + width: maxWidth * 2, + height: maxHeight * 2, + ); + + // Copy golden to top left corner. + for (int x = 0; x < goldenWidth; x += 1) { + for (int y = 0; y < goldenHeight; y += 1) { + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + failureImage.setPixel(x, y, goldenPixel); + } + } + + // Copy screenshot to top right corner. + for (int x = 0; x < screenshotWidth; x += 1) { + for (int y = 0; y < screenshotHeight; y += 1) { + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + failureImage.setPixel(maxWidth + x, y, screenshotPixel); + } + } + + // Paint mismatch images. + final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); + for (int x = 0; x < maxWidth; x += 1) { + for (int y = 0; y < maxHeight; y += 1) { + if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { + // This pixel doesn't exist in the golden, or it doesn't exist in the + // screenshot. Therefore, we have nothing to compare. Treat this pixel + // as a max severity difference. + + // Paint this pixel in the absolute diff image. + failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + + // Paint this pixel in the relative severity diff image. + failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); + + continue; + } + + // Check if the screenshot matches the golden. + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + final pixelsMatch = goldenPixel == screenshotPixel; + if (pixelsMatch) { + continue; + } + + // Paint this pixel in the absolute diff image. + failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + + // Paint this pixel in the relative severity diff image. + final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); + final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; + failureImage.setPixel( + goldenWidth + x, + goldenHeight + y, + ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), + ); + } + } + + return failureImage; +} diff --git a/lib/src/scenes/film_strip.dart b/lib/src/scenes/film_strip.dart index 5515e3d..8d84045 100644 --- a/lib/src/scenes/film_strip.dart +++ b/lib/src/scenes/film_strip.dart @@ -277,6 +277,7 @@ class FilmStrip { // Lookup and return metadata for the position and size of each golden image // within the gallery. return GoldenSceneMetadata( + description: goldenName, images: [ for (final golden in renderablePhotos.keys) GoldenImageMetadata( diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index f79cd8e..716f4bd 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -1,6 +1,5 @@ import 'dart:convert'; import 'dart:io'; -import 'dart:math'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; @@ -13,11 +12,12 @@ import 'package:flutter_test_goldens/src/goldens/golden_collections.dart'; import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart'; import 'package:flutter_test_goldens/src/goldens/golden_rendering.dart'; import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; -import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart'; import 'package:flutter_test_goldens/src/logging.dart'; import 'package:flutter_test_goldens/src/png/png_metadata.dart'; +import 'package:flutter_test_goldens/src/scenes/failure_scene.dart'; import 'package:flutter_test_goldens/src/scenes/golden_files.dart'; import 'package:flutter_test_goldens/src/scenes/golden_scene.dart'; +import 'package:flutter_test_goldens/src/scenes/golden_scene_report_printer.dart'; import 'package:flutter_test_goldens/src/scenes/scene_layout.dart'; import 'package:image/image.dart'; import 'package:path/path.dart'; @@ -396,6 +396,7 @@ class Gallery { // Lookup and return metadata for the position and size of each golden image // within the gallery. return GoldenSceneMetadata( + description: _sceneDescription, images: [ for (final golden in renderablePhotos.keys) GoldenImageMetadata( @@ -461,6 +462,16 @@ class Gallery { } final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); + // Extract scene metadata from the existing golden file. + final scenePngBytes = goldenFile.readAsBytesSync(); + final pngText = scenePngBytes.readTextMetadata(); + final sceneJsonText = pngText["flutter_test_goldens"]; + if (sceneJsonText == null) { + throw Exception("Golden image is missing scene metadata: ${goldenFile.path}"); + } + final sceneJson = JsonDecoder().convert(sceneJsonText); + final metadata = GoldenSceneMetadata.fromJson(sceneJson); + // Extract scene metadata from the current widget tree. FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); late final GoldenCollection screenshotCollection; @@ -471,102 +482,88 @@ class Gallery { // Compare goldens in the scene. FtgLog.pipeline.fine("Comparing goldens and screenshots"); final mismatches = compareGoldenCollections(goldenCollection, screenshotCollection); - if (mismatches.mismatches.isNotEmpty) { - FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):"); - for (final mismatch in mismatches.mismatches.values) { - FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); + + final items = []; + final missingCandidates = []; + final extraCandidates = []; + + FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):"); + for (final mismatch in mismatches.mismatches.values) { + FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); + switch (mismatch) { + case MissingCandidateMismatch(): + // A golden candidate is missing. + missingCandidates.add(mismatch); + break; + case MissingGoldenMismatch(): + // We have a golden candidate, but not the original golden. + extraCandidates.add(mismatch); + break; } + } - for (final mismatch in mismatches.mismatches.values) { - if (mismatch.golden == null || mismatch.screenshot == null) { - continue; - } - - FtgLog.pipeline.fine("Painting a golden failure: $mismatch"); - Directory(_goldenFailureDirectoryPath).createSync(); - - await tester.runAsync(() async { - final goldenWidth = mismatch.golden!.image.width; - final goldenHeight = mismatch.golden!.image.height; - - final screenshotWidth = mismatch.screenshot!.image.width; - final screenshotHeight = mismatch.screenshot!.image.height; - - final maxWidth = max(goldenWidth, screenshotWidth); - final maxHeight = max(goldenHeight, screenshotHeight); - - final failureImage = Image( - width: maxWidth * 2, - height: maxHeight * 2, - ); - - // Copy golden to top left corner. - for (int x = 0; x < goldenWidth; x += 1) { - for (int y = 0; y < goldenHeight; y += 1) { - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - failureImage.setPixel(x, y, goldenPixel); - } - } - - // Copy screenshot to top right corner. - for (int x = 0; x < screenshotWidth; x += 1) { - for (int y = 0; y < screenshotHeight; y += 1) { - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - failureImage.setPixel(maxWidth + x, y, screenshotPixel); - } - } - - // Paint mismatch images. - final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); - for (int x = 0; x < maxWidth; x += 1) { - for (int y = 0; y < maxHeight; y += 1) { - if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { - // This pixel doesn't exist in the golden, or it doesn't exist in the - // screenshot. Therefore, we have nothing to compare. Treat this pixel - // as a max severity difference. - - // Paint this pixel in the absolute diff image. - failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); - - // Paint this pixel in the relative severity diff image. - failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); - - continue; - } - - // Check if the screenshot matches the golden. - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - final pixelsMatch = goldenPixel == screenshotPixel; - if (pixelsMatch) { - continue; - } - - // Paint this pixel in the absolute diff image. - failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); - - // Paint this pixel in the relative severity diff image. - final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); - final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; - failureImage.setPixel( - goldenWidth + x, - goldenHeight + y, - ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), - ); - } - } - - await encodePngFile( - "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", - failureImage, - ); - }); + // For each candidate found in the scene, report whether it passed or failed. + for (final screenshotId in screenshotCollection.ids) { + if (!goldenCollection.hasId(screenshotId)) { + // This candidate is an extra candidate, i.e., it was found in the scene, + // but it doesn't have a golden counterpart. We already reported extra candidates + // above, so we can skip this candidate. + continue; } - throw Exception("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); - } else { + // Find the golden metadata for this candidate. + final goldenMetadata = metadata.images.where((image) => image.id == screenshotId).first; + + final mismatch = mismatches.mismatches[screenshotId]; + if (mismatch == null) { + // The golden check passed. + items.add( + GoldenReport.success(goldenMetadata), + ); + } else { + // The golden check failed. + items.add( + GoldenReport.failure( + metadata: goldenMetadata, + mismatch: mismatch, + ), + ); + } + } + + if (mismatches.mismatches.isEmpty) { FtgLog.pipeline.info("No golden mismatches found"); } + + for (final mismatch in mismatches.mismatches.values) { + if (mismatch.golden == null || mismatch.screenshot == null) { + continue; + } + + FtgLog.pipeline.fine("Painting a golden failure: $mismatch"); + Directory(_goldenFailureDirectoryPath).createSync(); + + await tester.runAsync(() async { + final failureImage = await paintGoldenMismatchImages(mismatch); + + await encodePngFile( + "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", + failureImage, + ); + }); + } + + final report = GoldenSceneReport( + metadata: metadata, + items: items, + missingCandidates: missingCandidates, + extraCandidates: extraCandidates, + ); + _printReport(report); + + if (mismatches.mismatches.isNotEmpty) { + fail("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); + } } String get _testFileDirectory => (goldenFileComparator as LocalFileComparator).basedir.path; @@ -580,6 +577,11 @@ class Gallery { "$_goldenDirectory$_fileName${includeExtension ? ".png" : ""}"; String get _goldenFailureDirectoryPath => "${_goldenDirectory}failures"; + + /// Prints the report in an human readable format to the console. + void _printReport(GoldenSceneReport report) { + GoldenSceneReportPrinter().printReport(report); + } } /// Pumps a widget tree into the given [tester], wrapping its content within the given [decorator]. diff --git a/lib/src/scenes/golden_scene.dart b/lib/src/scenes/golden_scene.dart index be1fb5c..5903e32 100644 --- a/lib/src/scenes/golden_scene.dart +++ b/lib/src/scenes/golden_scene.dart @@ -5,6 +5,8 @@ import 'package:flutter/material.dart' show Colors; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test_goldens/src/goldens/golden_camera.dart'; +import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart'; +import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; class GoldenScene extends StatelessWidget { const GoldenScene({ @@ -110,3 +112,90 @@ typedef GoldenPumper = Future Function( ); typedef GoldenSetup = FutureOr Function(WidgetTester tester); + +/// A report of a golden scene test. +/// +/// Reports the success or failure of each individual golden in the scene, as well as +/// the missing candidates and candidates that have no corresponding golden. +class GoldenSceneReport { + GoldenSceneReport({ + required this.metadata, + required this.items, + required this.missingCandidates, + required this.extraCandidates, + }); + + /// The metadata of the scene, such as the golden images and their positions. + final GoldenSceneMetadata metadata; + + /// The items found in the scene. + /// + /// Each item might be a successful or a failed golden check. + final List items; + + /// The golden candidates that were expected to be present in the scene, but were not found. + final List missingCandidates; + + /// The golden candidates that were found in the scene, but were not expected to be present. + final List extraCandidates; + + /// The total number of successful [items] in the scene. + int get totalPassed => items.where((e) => e.status == GoldenTestStatus.success).length; + + /// The total number of failed [items] in the scene. + /// + /// Only candidates that have a corresponding golden image and failed the golden check + /// count as a failure. + /// + /// See [missingCandidates] for candidates that were expected but not found, + /// and [extraCandidates] for candidates that were found but not expected. + int get totalFailed => items.where((e) => e.status == GoldenTestStatus.failure).length; +} + +/// A report of success or failure for a single golden within a scene. +/// +/// A [GoldenReport] holds the test results for a candidate that has a corresponding golden. +class GoldenReport { + factory GoldenReport.success(GoldenImageMetadata metadata) { + return GoldenReport( + status: GoldenTestStatus.success, + metadata: metadata, + ); + } + + factory GoldenReport.failure({ + required GoldenImageMetadata metadata, + required GoldenMismatch mismatch, + }) { + return GoldenReport( + status: GoldenTestStatus.failure, + metadata: metadata, + mismatch: mismatch, + ); + } + + GoldenReport({ + required this.status, + required this.metadata, + this.mismatch, + }) : assert( + status == GoldenTestStatus.success || mismatch != null, + "A failure report must have a mismatch.", + ); + + /// Whether the gallery item passed or failed the golden check. + final GoldenTestStatus status; + + /// The metadata of the candidate image of this report. + final GoldenImageMetadata metadata; + + /// The failure details of the gallery item, if it failed the golden check. + /// + /// Non-`null` if [status] is [GoldenTestStatus.failure] and `null` otherwise. + final GoldenMismatch? mismatch; +} + +enum GoldenTestStatus { + success, + failure, +} diff --git a/lib/src/scenes/golden_scene_report_printer.dart b/lib/src/scenes/golden_scene_report_printer.dart new file mode 100644 index 0000000..92af41a --- /dev/null +++ b/lib/src/scenes/golden_scene_report_printer.dart @@ -0,0 +1,83 @@ +import 'package:flutter_test_goldens/flutter_test_goldens.dart'; + +class GoldenSceneReportPrinter { + void printReport(GoldenSceneReport report) { + if (report.totalFailed == 0 && // + report.missingCandidates.isEmpty && + report.extraCandidates.isEmpty) { + // All checks passed. Don't print anything. + return; + } + + final buffer = StringBuffer(); + + // Report the summary of passed/failed tests and missing/extra candidates. + buffer.write("Golden scene has failures: ${report.metadata.description} ("); + buffer.write("✅ ${report.totalPassed}/${report.items.length}, "); + buffer.write("❌ ${report.totalFailed}/${report.items.length}"); + if (report.missingCandidates.isNotEmpty || report.extraCandidates.isNotEmpty) { + buffer.write(", ❓"); + + if (report.missingCandidates.isNotEmpty) { + buffer.write(" -${report.missingCandidates.length}"); + } + + if (report.extraCandidates.isNotEmpty) { + if (report.missingCandidates.isNotEmpty) { + buffer.write(" /"); + } + buffer.write(" +${report.extraCandidates.length}"); + } + } + buffer.writeln(")"); + + if (report.totalFailed > 0) { + buffer.writeln(""); + for (final item in report.items) { + if (item.status == GoldenTestStatus.success) { + buffer.writeln("✅ ${item.metadata.id}"); + continue; + } + + // This item has a failed check. + final mismatch = item.mismatch; + switch (mismatch) { + case WrongSizeGoldenMismatch(): + buffer.writeln( + '"❌ ${item.metadata.id}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); + break; + case PixelGoldenMismatch(): + buffer.writeln( + '"❌ ${item.metadata.id}" has a ${(mismatch.percent * 100).toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); + break; + case MissingGoldenMismatch(): + case MissingCandidateMismatch(): + // Don't print anything, missing goldens are reported at the end. + break; + default: + buffer.writeln('"❌ ${item.metadata.id}": ${mismatch!.describe}'); + break; + } + } + } + + if (report.missingCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Missing goldens:"); + for (final mismatch in report.missingCandidates) { + buffer.writeln('❓ "${mismatch.golden!.id}"'); + } + } + + if (report.extraCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Extra (unexpected) candidates:"); + for (final mismatch in report.extraCandidates) { + buffer.writeln('❓ "${mismatch.screenshot!.id}"'); + } + } + + // ignore: avoid_print + print(buffer.toString()); + } +}