From f2435ff7582f05fc49a67d810be5a48791399456 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Thu, 19 Jun 2025 15:01:43 -0300 Subject: [PATCH 1/7] [Feature] Add scene comparison report (Resolves #43) --- lib/src/goldens/golden_comparisons.dart | 2 +- lib/src/scenes/gallery.dart | 369 +++++++++++++++++++----- 2 files changed, 297 insertions(+), 74 deletions(-) diff --git a/lib/src/goldens/golden_comparisons.dart b/lib/src/goldens/golden_comparisons.dart index b8faddb..bc6d95c 100644 --- a/lib/src/goldens/golden_comparisons.dart +++ b/lib/src/goldens/golden_comparisons.dart @@ -101,7 +101,7 @@ class PixelGoldenMismatch extends GoldenMismatch { int get totalPixelCount => golden.size.width.toInt() * golden.size.height.toInt(); - double get percent => mismatchPixelCount / totalPixelCount; + double get percent => mismatchPixelCount / totalPixelCount * 100; @override String get describe => diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index f79cd8e..53d9a0d 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -471,102 +471,164 @@ class Gallery { // Compare goldens in the scene. FtgLog.pipeline.fine("Comparing goldens and screenshots"); final mismatches = compareGoldenCollections(goldenCollection, screenshotCollection); - if (mismatches.mismatches.isNotEmpty) { - FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):"); - for (final mismatch in mismatches.mismatches.values) { - FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); + + final items = []; + final missingCandidates = []; + final extraCandidates = []; + + int totalPassed = 0; + int totalFailed = 0; + + FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):"); + for (final mismatch in mismatches.mismatches.values) { + FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); + switch (mismatch) { + case MissingGoldenMismatch(screenshot: null): + // A golden candidate is missing. + missingCandidates.add(mismatch); + break; + case MissingGoldenMismatch(golden: null): + // We have a golden candidate, but not the original golden. + extraCandidates.add(mismatch); + break; } + } - for (final mismatch in mismatches.mismatches.values) { - if (mismatch.golden == null || mismatch.screenshot == null) { - continue; - } + for (final screenshotId in screenshotCollection.ids) { + if (!goldenCollection.hasId(screenshotId)) { + continue; + } + + final mismatch = mismatches.mismatches[screenshotId]; - FtgLog.pipeline.fine("Painting a golden failure: $mismatch"); - Directory(_goldenFailureDirectoryPath).createSync(); + final status = mismatch != null // + ? GoldenTestStatus.failure + : GoldenTestStatus.success; + + if (status == GoldenTestStatus.success) { + totalPassed += 1; + } else { + totalFailed += 1; + } + + items.add( + GoldenReportItem( + status: status, + description: goldenCollection.imagesById[screenshotId]!.id, + details: [ + if (mismatch != null) + GoldenCheckDetail( + status: GoldenTestStatus.failure, + description: mismatch.toString(), + mismatch: mismatch, + ), + ], + ), + ); + } - await tester.runAsync(() async { - final goldenWidth = mismatch.golden!.image.width; - final goldenHeight = mismatch.golden!.image.height; + final report = GoldenSceneReport( + sceneDescription: _sceneDescription, + items: items, + missingCandidates: missingCandidates, + extraCandidates: extraCandidates, + totalPassed: totalPassed, + totalFailed: totalFailed, + ); + _printReport(report); - final screenshotWidth = mismatch.screenshot!.image.width; - final screenshotHeight = mismatch.screenshot!.image.height; + if (mismatches.mismatches.isEmpty) { + FtgLog.pipeline.info("No golden mismatches found"); + return; + } - final maxWidth = max(goldenWidth, screenshotWidth); - final maxHeight = max(goldenHeight, screenshotHeight); + for (final mismatch in mismatches.mismatches.values) { + if (mismatch.golden == null || mismatch.screenshot == null) { + continue; + } - final failureImage = Image( - width: maxWidth * 2, - height: maxHeight * 2, - ); + FtgLog.pipeline.fine("Painting a golden failure: $mismatch"); + Directory(_goldenFailureDirectoryPath).createSync(); - // Copy golden to top left corner. - for (int x = 0; x < goldenWidth; x += 1) { - for (int y = 0; y < goldenHeight; y += 1) { - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - failureImage.setPixel(x, y, goldenPixel); - } - } + await tester.runAsync(() async { + final goldenWidth = mismatch.golden!.image.width; + final goldenHeight = mismatch.golden!.image.height; - // Copy screenshot to top right corner. - for (int x = 0; x < screenshotWidth; x += 1) { - for (int y = 0; y < screenshotHeight; y += 1) { - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - failureImage.setPixel(maxWidth + x, y, screenshotPixel); - } - } + final screenshotWidth = mismatch.screenshot!.image.width; + final screenshotHeight = mismatch.screenshot!.image.height; - // Paint mismatch images. - final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); - for (int x = 0; x < maxWidth; x += 1) { - for (int y = 0; y < maxHeight; y += 1) { - if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { - // This pixel doesn't exist in the golden, or it doesn't exist in the - // screenshot. Therefore, we have nothing to compare. Treat this pixel - // as a max severity difference. + final maxWidth = max(goldenWidth, screenshotWidth); + final maxHeight = max(goldenHeight, screenshotHeight); - // Paint this pixel in the absolute diff image. - failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + final failureImage = Image( + width: maxWidth * 2, + height: maxHeight * 2, + ); - // Paint this pixel in the relative severity diff image. - failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); + // Copy golden to top left corner. + for (int x = 0; x < goldenWidth; x += 1) { + for (int y = 0; y < goldenHeight; y += 1) { + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + failureImage.setPixel(x, y, goldenPixel); + } + } - continue; - } + // Copy screenshot to top right corner. + for (int x = 0; x < screenshotWidth; x += 1) { + for (int y = 0; y < screenshotHeight; y += 1) { + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + failureImage.setPixel(maxWidth + x, y, screenshotPixel); + } + } - // Check if the screenshot matches the golden. - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - final pixelsMatch = goldenPixel == screenshotPixel; - if (pixelsMatch) { - continue; - } + // Paint mismatch images. + final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); + for (int x = 0; x < maxWidth; x += 1) { + for (int y = 0; y < maxHeight; y += 1) { + if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { + // This pixel doesn't exist in the golden, or it doesn't exist in the + // screenshot. Therefore, we have nothing to compare. Treat this pixel + // as a max severity difference. // Paint this pixel in the absolute diff image. failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); // Paint this pixel in the relative severity diff image. - final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); - final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; - failureImage.setPixel( - goldenWidth + x, - goldenHeight + y, - ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), - ); + failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); + + continue; } - } - await encodePngFile( - "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", - failureImage, - ); - }); - } + // Check if the screenshot matches the golden. + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + final pixelsMatch = goldenPixel == screenshotPixel; + if (pixelsMatch) { + continue; + } - throw Exception("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); - } else { - FtgLog.pipeline.info("No golden mismatches found"); + // Paint this pixel in the absolute diff image. + failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + + // Paint this pixel in the relative severity diff image. + final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); + final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; + failureImage.setPixel( + goldenWidth + x, + goldenHeight + y, + ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), + ); + } + } + + await encodePngFile( + "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", + failureImage, + ); + }); } + + throw Exception("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); } String get _testFileDirectory => (goldenFileComparator as LocalFileComparator).basedir.path; @@ -580,6 +642,90 @@ class Gallery { "$_goldenDirectory$_fileName${includeExtension ? ".png" : ""}"; String get _goldenFailureDirectoryPath => "${_goldenDirectory}failures"; + + /// Prints the report in an human readable format to the console. + void _printReport(GoldenSceneReport report) { + if (report.totalFailed == 0 && // + report.missingCandidates.isEmpty && + report.extraCandidates.isEmpty) { + // All checks passed. Don't print anything. + return; + } + + final buffer = StringBuffer(); + + // Report the summary of passed/failed tests and missing/extra candidates. + buffer.write("Golden scene has failures: ${report.sceneDescription} ("); + buffer.write("✅ ${report.totalPassed}/${report.items.length}, "); + buffer.write("❌ ${report.totalFailed}/${report.items.length}"); + if (report.missingCandidates.isNotEmpty || report.extraCandidates.isNotEmpty) { + buffer.write(", ❓"); + + if (report.missingCandidates.isNotEmpty) { + buffer.write(" -${report.missingCandidates.length}"); + } + + if (report.extraCandidates.isNotEmpty) { + if (report.missingCandidates.isNotEmpty) { + buffer.write(" /"); + } + buffer.write(" +${report.extraCandidates.length}"); + } + } + buffer.writeln(")"); + + if (report.totalFailed > 0) { + buffer.writeln(""); + for (final item in report.items) { + if (item.status == GoldenTestStatus.success) { + buffer.writeln("✅ ${item.description}"); + continue; + } + + // This item has a failed check. + final mismatch = item.details // + .where((detail) => detail.mismatch != null) + .firstOrNull + ?.mismatch; + + switch (mismatch) { + case WrongSizeGoldenMismatch(): + buffer.writeln( + '"❌ ${item.description}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); + break; + case PixelGoldenMismatch(): + buffer.writeln( + '"❌ ${item.description}" has a ${mismatch.percent.toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); + break; + case MissingGoldenMismatch(): + // Don't print anything, missing goldens are reported at the end. + break; + default: + buffer.writeln('"❌ ${item.description}": ${mismatch!.describe}'); + break; + } + } + } + + if (report.missingCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Missing goldens:"); + for (final mismatch in report.missingCandidates) { + buffer.writeln('❓ "${mismatch.golden!.id}"'); + } + } + + if (report.extraCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Extra (unexpected) candidates:"); + for (final mismatch in report.extraCandidates) { + buffer.writeln('❓ "${mismatch.screenshot!.id}"'); + } + } + + // ignore: avoid_print + print(buffer.toString()); + } } /// Pumps a widget tree into the given [tester], wrapping its content within the given [decorator]. @@ -701,3 +847,80 @@ Widget defaultGalleryItemDecorator(WidgetTester tester, Widget content) { child: content, ); } + +/// A report of a golden scene test. +/// +/// Holds information to display the results of a golden scene test. +class GoldenSceneReport { + GoldenSceneReport({ + required this.sceneDescription, + required this.items, + required this.missingCandidates, + required this.extraCandidates, + required this.totalPassed, + required this.totalFailed, + }); + + /// The human readable description of the scene. + final String sceneDescription; + + /// The items found in the scene. + /// + /// Each item might be a successful or a failed golden check. + final List items; + + /// The golden candidates that were expected to be present in the scene, but were not found. + final List missingCandidates; + + /// The golden candidates that were found in the scene, but were not expected to be present. + final List extraCandidates; + + /// The total number of successful [items] in the scene. + final int totalPassed; + + /// The total number of failed [items] in the scene. + final int totalFailed; +} + +/// An item in a golden scene report. +/// +/// Each item represents a single gallery item that was found in both the original golden +/// and the candidate image. +class GoldenReportItem { + GoldenReportItem({ + required this.status, + required this.description, + required this.details, + }); + + /// Whether the gallery item passed or failed the golden check. + final GoldenTestStatus status; + + /// The description of the gallery item that was checked. + final String description; + + /// The details of the golden check for this item. + /// + /// Might contain both successful and failed checks. + final List details; +} + +class GoldenCheckDetail { + GoldenCheckDetail({ + required this.status, + required this.description, + this.mismatch, + }) : assert( + status != GoldenTestStatus.success || mismatch == null, + "A successful golden test cannot have a mismatch", + ); + + final GoldenTestStatus status; + final String description; + final GoldenMismatch? mismatch; +} + +enum GoldenTestStatus { + success, + failure, +} From 45a26c25dccc5489f28f3813badf87b832e2bc29 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Thu, 19 Jun 2025 16:26:14 -0300 Subject: [PATCH 2/7] PR updates --- lib/src/goldens/golden_comparisons.dart | 35 +- lib/src/scenes/failure_scene.dart | 83 +++++ lib/src/scenes/gallery.dart | 298 +++--------------- lib/src/scenes/golden_scene.dart | 99 ++++++ .../scenes/golden_scene_report_printer.dart | 87 +++++ 5 files changed, 328 insertions(+), 274 deletions(-) create mode 100644 lib/src/scenes/failure_scene.dart create mode 100644 lib/src/scenes/golden_scene_report_printer.dart diff --git a/lib/src/goldens/golden_comparisons.dart b/lib/src/goldens/golden_comparisons.dart index bc6d95c..9b25d14 100644 --- a/lib/src/goldens/golden_comparisons.dart +++ b/lib/src/goldens/golden_comparisons.dart @@ -11,8 +11,8 @@ GoldenCollectionMismatches compareGoldenCollections( // For every golden, look for missing and mismatching screenshots. for (final id in goldens.ids) { if (!screenshots.hasId(id)) { - mismatches[id] = MissingGoldenMismatch( - golden: goldens[id], + mismatches[id] = MissingCandidateMismatch( + golden: goldens[id]!, ); continue; } @@ -44,7 +44,7 @@ GoldenCollectionMismatches compareGoldenCollections( for (final id in screenshots.ids) { if (!goldens.hasId(id)) { mismatches[id] = MissingGoldenMismatch( - screenshot: screenshots[id], + screenshot: screenshots[id]!, ); } } @@ -101,7 +101,7 @@ class PixelGoldenMismatch extends GoldenMismatch { int get totalPixelCount => golden.size.width.toInt() * golden.size.height.toInt(); - double get percent => mismatchPixelCount / totalPixelCount * 100; + double get percent => mismatchPixelCount / totalPixelCount; @override String get describe => @@ -132,22 +132,29 @@ class WrongSizeGoldenMismatch extends GoldenMismatch { Map get describeStructured => throw UnimplementedError(); } -/// Attempted to compare a screenshot to a golden, but either the screenshot was never -/// generated, or the screenshot was generated for a golden that doesn't exist. +/// Attempted to compare a candidate to a golden, but the candidate was generated for a golden that doesn't exist. class MissingGoldenMismatch extends GoldenMismatch { MissingGoldenMismatch({ - super.golden, - super.screenshot, - }); + required GoldenImage screenshot, + }) : super(screenshot: screenshot); - GoldenImage get _existingGolden => golden ?? screenshot!; + @override + String get describe => + "A new screenshot was generated with ID '${screenshot!.id}', but there's no existing golden image with that ID."; @override - String get describe => "A new screenshot was generated with ID '${_existingGolden.id}', $_missingMessage"; + Map get describeStructured => throw UnimplementedError(); +} + +/// Attempted to compare a candidante to a golden, but the candidate was never generated. +class MissingCandidateMismatch extends GoldenMismatch { + MissingCandidateMismatch({ + required GoldenImage golden, + }) : super(golden: golden); - String get _missingMessage => golden != null // - ? "but no screenshot was generated with that ID." - : "but there's no existing golden image with that ID."; + @override + String get describe => + "A new screenshot was generated with ID '${golden!.id}', but no screenshot was generated with that ID."; @override Map get describeStructured => throw UnimplementedError(); diff --git a/lib/src/scenes/failure_scene.dart b/lib/src/scenes/failure_scene.dart new file mode 100644 index 0000000..02b2079 --- /dev/null +++ b/lib/src/scenes/failure_scene.dart @@ -0,0 +1,83 @@ +import 'dart:math'; +import 'dart:ui' as ui; + +import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart'; +import 'package:image/image.dart'; + +import 'package:flutter_test_goldens/flutter_test_goldens.dart'; + +/// Given a [mismatch] between a golden and a screenshot, generates an image +/// that shows the golden, the screenshot, and the differences between them. +Future generateFailureScene(GoldenMismatch mismatch) async { + final goldenWidth = mismatch.golden!.image.width; + final goldenHeight = mismatch.golden!.image.height; + + final screenshotWidth = mismatch.screenshot!.image.width; + final screenshotHeight = mismatch.screenshot!.image.height; + + final maxWidth = max(goldenWidth, screenshotWidth); + final maxHeight = max(goldenHeight, screenshotHeight); + + final failureImage = Image( + width: maxWidth * 2, + height: maxHeight * 2, + ); + + // Copy golden to top left corner. + for (int x = 0; x < goldenWidth; x += 1) { + for (int y = 0; y < goldenHeight; y += 1) { + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + failureImage.setPixel(x, y, goldenPixel); + } + } + + // Copy screenshot to top right corner. + for (int x = 0; x < screenshotWidth; x += 1) { + for (int y = 0; y < screenshotHeight; y += 1) { + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + failureImage.setPixel(maxWidth + x, y, screenshotPixel); + } + } + + // Paint mismatch images. + final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); + for (int x = 0; x < maxWidth; x += 1) { + for (int y = 0; y < maxHeight; y += 1) { + if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { + // This pixel doesn't exist in the golden, or it doesn't exist in the + // screenshot. Therefore, we have nothing to compare. Treat this pixel + // as a max severity difference. + + // Paint this pixel in the absolute diff image. + failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + + // Paint this pixel in the relative severity diff image. + failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); + + continue; + } + + // Check if the screenshot matches the golden. + final goldenPixel = mismatch.golden!.image.getPixel(x, y); + final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); + final pixelsMatch = goldenPixel == screenshotPixel; + if (pixelsMatch) { + continue; + } + + // Paint this pixel in the absolute diff image. + failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); + + // Paint this pixel in the relative severity diff image. + final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); + final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; + failureImage.setPixel( + goldenWidth + x, + goldenHeight + y, + ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), + ); + } + } + + return failureImage; +} diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index 53d9a0d..aa3d284 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -1,6 +1,5 @@ import 'dart:convert'; import 'dart:io'; -import 'dart:math'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; @@ -13,11 +12,12 @@ import 'package:flutter_test_goldens/src/goldens/golden_collections.dart'; import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart'; import 'package:flutter_test_goldens/src/goldens/golden_rendering.dart'; import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; -import 'package:flutter_test_goldens/src/goldens/pixel_comparisons.dart'; import 'package:flutter_test_goldens/src/logging.dart'; import 'package:flutter_test_goldens/src/png/png_metadata.dart'; +import 'package:flutter_test_goldens/src/scenes/failure_scene.dart'; import 'package:flutter_test_goldens/src/scenes/golden_files.dart'; import 'package:flutter_test_goldens/src/scenes/golden_scene.dart'; +import 'package:flutter_test_goldens/src/scenes/golden_scene_report_printer.dart'; import 'package:flutter_test_goldens/src/scenes/scene_layout.dart'; import 'package:image/image.dart'; import 'package:path/path.dart'; @@ -473,7 +473,7 @@ class Gallery { final mismatches = compareGoldenCollections(goldenCollection, screenshotCollection); final items = []; - final missingCandidates = []; + final missingCandidates = []; final extraCandidates = []; int totalPassed = 0; @@ -483,63 +483,53 @@ class Gallery { for (final mismatch in mismatches.mismatches.values) { FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); switch (mismatch) { - case MissingGoldenMismatch(screenshot: null): + case MissingCandidateMismatch(): // A golden candidate is missing. missingCandidates.add(mismatch); break; - case MissingGoldenMismatch(golden: null): + case MissingGoldenMismatch(): // We have a golden candidate, but not the original golden. extraCandidates.add(mismatch); break; } } + // For each candidate found in the scene, report whether it passed or failed. for (final screenshotId in screenshotCollection.ids) { if (!goldenCollection.hasId(screenshotId)) { + // This candidate is an extra candidate, i.e., it was found in the scene, + // but it doesn't have a golden counterpart. We already reported extra candidates + // above, so we can skip this candidate. continue; } final mismatch = mismatches.mismatches[screenshotId]; - - final status = mismatch != null // - ? GoldenTestStatus.failure - : GoldenTestStatus.success; - - if (status == GoldenTestStatus.success) { + if (mismatch == null) { + // The golden check passed. totalPassed += 1; + items.add( + GoldenReportItem.success(description: goldenCollection.imagesById[screenshotId]!.id), + ); } else { + // The golden check failed. totalFailed += 1; - } - - items.add( - GoldenReportItem( - status: status, - description: goldenCollection.imagesById[screenshotId]!.id, - details: [ - if (mismatch != null) + items.add( + GoldenReportItem.failure( + description: goldenCollection.imagesById[screenshotId]!.id, + details: [ GoldenCheckDetail( status: GoldenTestStatus.failure, description: mismatch.toString(), mismatch: mismatch, ), - ], - ), - ); + ], + ), + ); + } } - final report = GoldenSceneReport( - sceneDescription: _sceneDescription, - items: items, - missingCandidates: missingCandidates, - extraCandidates: extraCandidates, - totalPassed: totalPassed, - totalFailed: totalFailed, - ); - _printReport(report); - if (mismatches.mismatches.isEmpty) { FtgLog.pipeline.info("No golden mismatches found"); - return; } for (final mismatch in mismatches.mismatches.values) { @@ -551,75 +541,7 @@ class Gallery { Directory(_goldenFailureDirectoryPath).createSync(); await tester.runAsync(() async { - final goldenWidth = mismatch.golden!.image.width; - final goldenHeight = mismatch.golden!.image.height; - - final screenshotWidth = mismatch.screenshot!.image.width; - final screenshotHeight = mismatch.screenshot!.image.height; - - final maxWidth = max(goldenWidth, screenshotWidth); - final maxHeight = max(goldenHeight, screenshotHeight); - - final failureImage = Image( - width: maxWidth * 2, - height: maxHeight * 2, - ); - - // Copy golden to top left corner. - for (int x = 0; x < goldenWidth; x += 1) { - for (int y = 0; y < goldenHeight; y += 1) { - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - failureImage.setPixel(x, y, goldenPixel); - } - } - - // Copy screenshot to top right corner. - for (int x = 0; x < screenshotWidth; x += 1) { - for (int y = 0; y < screenshotHeight; y += 1) { - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - failureImage.setPixel(maxWidth + x, y, screenshotPixel); - } - } - - // Paint mismatch images. - final absoluteDiffColor = ColorUint32.rgb(255, 255, 0); - for (int x = 0; x < maxWidth; x += 1) { - for (int y = 0; y < maxHeight; y += 1) { - if (x >= goldenWidth || x >= screenshotWidth || y >= goldenHeight || y >= screenshotHeight) { - // This pixel doesn't exist in the golden, or it doesn't exist in the - // screenshot. Therefore, we have nothing to compare. Treat this pixel - // as a max severity difference. - - // Paint this pixel in the absolute diff image. - failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); - - // Paint this pixel in the relative severity diff image. - failureImage.setPixel(maxWidth + x, maxHeight + y, absoluteDiffColor); - - continue; - } - - // Check if the screenshot matches the golden. - final goldenPixel = mismatch.golden!.image.getPixel(x, y); - final screenshotPixel = mismatch.screenshot!.image.getPixel(x, y); - final pixelsMatch = goldenPixel == screenshotPixel; - if (pixelsMatch) { - continue; - } - - // Paint this pixel in the absolute diff image. - failureImage.setPixel(x, maxHeight + y, absoluteDiffColor); - - // Paint this pixel in the relative severity diff image. - final mismatchPercent = calculateColorMismatchPercent(goldenPixel, screenshotPixel); - final yellowAmount = ui.lerpDouble(0.2, 1.0, mismatchPercent)!; - failureImage.setPixel( - goldenWidth + x, - goldenHeight + y, - ColorUint32.rgb((255 * yellowAmount).round(), (255 * yellowAmount).round(), 0), - ); - } - } + final failureImage = await generateFailureScene(mismatch); await encodePngFile( "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", @@ -628,7 +550,19 @@ class Gallery { }); } - throw Exception("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); + final report = GoldenSceneReport( + sceneDescription: _sceneDescription, + items: items, + missingCandidates: missingCandidates, + extraCandidates: extraCandidates, + totalPassed: totalPassed, + totalFailed: totalFailed, + ); + _printReport(report); + + if (mismatches.mismatches.isNotEmpty) { + fail("Goldens failed with ${mismatches.mismatches.length} mismatch(es)"); + } } String get _testFileDirectory => (goldenFileComparator as LocalFileComparator).basedir.path; @@ -645,86 +579,7 @@ class Gallery { /// Prints the report in an human readable format to the console. void _printReport(GoldenSceneReport report) { - if (report.totalFailed == 0 && // - report.missingCandidates.isEmpty && - report.extraCandidates.isEmpty) { - // All checks passed. Don't print anything. - return; - } - - final buffer = StringBuffer(); - - // Report the summary of passed/failed tests and missing/extra candidates. - buffer.write("Golden scene has failures: ${report.sceneDescription} ("); - buffer.write("✅ ${report.totalPassed}/${report.items.length}, "); - buffer.write("❌ ${report.totalFailed}/${report.items.length}"); - if (report.missingCandidates.isNotEmpty || report.extraCandidates.isNotEmpty) { - buffer.write(", ❓"); - - if (report.missingCandidates.isNotEmpty) { - buffer.write(" -${report.missingCandidates.length}"); - } - - if (report.extraCandidates.isNotEmpty) { - if (report.missingCandidates.isNotEmpty) { - buffer.write(" /"); - } - buffer.write(" +${report.extraCandidates.length}"); - } - } - buffer.writeln(")"); - - if (report.totalFailed > 0) { - buffer.writeln(""); - for (final item in report.items) { - if (item.status == GoldenTestStatus.success) { - buffer.writeln("✅ ${item.description}"); - continue; - } - - // This item has a failed check. - final mismatch = item.details // - .where((detail) => detail.mismatch != null) - .firstOrNull - ?.mismatch; - - switch (mismatch) { - case WrongSizeGoldenMismatch(): - buffer.writeln( - '"❌ ${item.description}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); - break; - case PixelGoldenMismatch(): - buffer.writeln( - '"❌ ${item.description}" has a ${mismatch.percent.toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); - break; - case MissingGoldenMismatch(): - // Don't print anything, missing goldens are reported at the end. - break; - default: - buffer.writeln('"❌ ${item.description}": ${mismatch!.describe}'); - break; - } - } - } - - if (report.missingCandidates.isNotEmpty) { - buffer.writeln(""); - buffer.writeln("Missing goldens:"); - for (final mismatch in report.missingCandidates) { - buffer.writeln('❓ "${mismatch.golden!.id}"'); - } - } - - if (report.extraCandidates.isNotEmpty) { - buffer.writeln(""); - buffer.writeln("Extra (unexpected) candidates:"); - for (final mismatch in report.extraCandidates) { - buffer.writeln('❓ "${mismatch.screenshot!.id}"'); - } - } - - // ignore: avoid_print - print(buffer.toString()); + GoldenSceneReportPrinter().printReport(report); } } @@ -847,80 +702,3 @@ Widget defaultGalleryItemDecorator(WidgetTester tester, Widget content) { child: content, ); } - -/// A report of a golden scene test. -/// -/// Holds information to display the results of a golden scene test. -class GoldenSceneReport { - GoldenSceneReport({ - required this.sceneDescription, - required this.items, - required this.missingCandidates, - required this.extraCandidates, - required this.totalPassed, - required this.totalFailed, - }); - - /// The human readable description of the scene. - final String sceneDescription; - - /// The items found in the scene. - /// - /// Each item might be a successful or a failed golden check. - final List items; - - /// The golden candidates that were expected to be present in the scene, but were not found. - final List missingCandidates; - - /// The golden candidates that were found in the scene, but were not expected to be present. - final List extraCandidates; - - /// The total number of successful [items] in the scene. - final int totalPassed; - - /// The total number of failed [items] in the scene. - final int totalFailed; -} - -/// An item in a golden scene report. -/// -/// Each item represents a single gallery item that was found in both the original golden -/// and the candidate image. -class GoldenReportItem { - GoldenReportItem({ - required this.status, - required this.description, - required this.details, - }); - - /// Whether the gallery item passed or failed the golden check. - final GoldenTestStatus status; - - /// The description of the gallery item that was checked. - final String description; - - /// The details of the golden check for this item. - /// - /// Might contain both successful and failed checks. - final List details; -} - -class GoldenCheckDetail { - GoldenCheckDetail({ - required this.status, - required this.description, - this.mismatch, - }) : assert( - status != GoldenTestStatus.success || mismatch == null, - "A successful golden test cannot have a mismatch", - ); - - final GoldenTestStatus status; - final String description; - final GoldenMismatch? mismatch; -} - -enum GoldenTestStatus { - success, - failure, -} diff --git a/lib/src/scenes/golden_scene.dart b/lib/src/scenes/golden_scene.dart index be1fb5c..1c51f78 100644 --- a/lib/src/scenes/golden_scene.dart +++ b/lib/src/scenes/golden_scene.dart @@ -5,6 +5,7 @@ import 'package:flutter/material.dart' show Colors; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test_goldens/src/goldens/golden_camera.dart'; +import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart'; class GoldenScene extends StatelessWidget { const GoldenScene({ @@ -110,3 +111,101 @@ typedef GoldenPumper = Future Function( ); typedef GoldenSetup = FutureOr Function(WidgetTester tester); + +/// A report of a golden scene test. +/// +/// Holds information to display the results of a golden scene test. +class GoldenSceneReport { + GoldenSceneReport({ + required this.sceneDescription, + required this.items, + required this.missingCandidates, + required this.extraCandidates, + required this.totalPassed, + required this.totalFailed, + }); + + /// The human readable description of the scene. + final String sceneDescription; + + /// The items found in the scene. + /// + /// Each item might be a successful or a failed golden check. + final List items; + + /// The golden candidates that were expected to be present in the scene, but were not found. + final List missingCandidates; + + /// The golden candidates that were found in the scene, but were not expected to be present. + final List extraCandidates; + + /// The total number of successful [items] in the scene. + final int totalPassed; + + /// The total number of failed [items] in the scene. + final int totalFailed; +} + +/// An item in a golden scene report. +/// +/// Each item represents a single gallery item that was found in both the original golden +/// and the candidate image. +class GoldenReportItem { + GoldenReportItem({ + required this.status, + required this.description, + required this.details, + }); + + factory GoldenReportItem.success({ + required String description, + }) { + return GoldenReportItem( + status: GoldenTestStatus.success, + description: description, + details: [], + ); + } + + factory GoldenReportItem.failure({ + required String description, + required List details, + }) { + return GoldenReportItem( + status: GoldenTestStatus.failure, + description: description, + details: details, + ); + } + + /// Whether the gallery item passed or failed the golden check. + final GoldenTestStatus status; + + /// The description of the gallery item that was checked. + final String description; + + /// The details of the golden check for this item. + /// + /// Might contain both successful and failed checks. + final List details; +} + +class GoldenCheckDetail { + GoldenCheckDetail({ + required this.status, + required this.description, + this.mismatch, + }) : assert( + status != GoldenTestStatus.success || mismatch == null, + "A successful golden test cannot have a mismatch", + ); + + final GoldenTestStatus status; + final String description; + final GoldenMismatch? mismatch; +} + +enum GoldenTestStatus { + success, + failure, +} diff --git a/lib/src/scenes/golden_scene_report_printer.dart b/lib/src/scenes/golden_scene_report_printer.dart new file mode 100644 index 0000000..43fef82 --- /dev/null +++ b/lib/src/scenes/golden_scene_report_printer.dart @@ -0,0 +1,87 @@ +import 'package:flutter_test_goldens/flutter_test_goldens.dart'; + +class GoldenSceneReportPrinter { + void printReport(GoldenSceneReport report) { + if (report.totalFailed == 0 && // + report.missingCandidates.isEmpty && + report.extraCandidates.isEmpty) { + // All checks passed. Don't print anything. + return; + } + + final buffer = StringBuffer(); + + // Report the summary of passed/failed tests and missing/extra candidates. + buffer.write("Golden scene has failures: ${report.sceneDescription} ("); + buffer.write("✅ ${report.totalPassed}/${report.items.length}, "); + buffer.write("❌ ${report.totalFailed}/${report.items.length}"); + if (report.missingCandidates.isNotEmpty || report.extraCandidates.isNotEmpty) { + buffer.write(", ❓"); + + if (report.missingCandidates.isNotEmpty) { + buffer.write(" -${report.missingCandidates.length}"); + } + + if (report.extraCandidates.isNotEmpty) { + if (report.missingCandidates.isNotEmpty) { + buffer.write(" /"); + } + buffer.write(" +${report.extraCandidates.length}"); + } + } + buffer.writeln(")"); + + if (report.totalFailed > 0) { + buffer.writeln(""); + for (final item in report.items) { + if (item.status == GoldenTestStatus.success) { + buffer.writeln("✅ ${item.description}"); + continue; + } + + // This item has a failed check. + final mismatch = item.details // + .where((detail) => detail.mismatch != null) + .firstOrNull + ?.mismatch; + + switch (mismatch) { + case WrongSizeGoldenMismatch(): + buffer.writeln( + '"❌ ${item.description}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); + break; + case PixelGoldenMismatch(): + buffer.writeln( + '"❌ ${item.description}" has a ${(mismatch.percent * 100).toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); + break; + case MissingGoldenMismatch(): + case MissingCandidateMismatch(): + // Don't print anything, missing goldens are reported at the end. + break; + default: + buffer.writeln('"❌ ${item.description}": ${mismatch!.describe}'); + break; + } + } + } + + if (report.missingCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Missing goldens:"); + for (final mismatch in report.missingCandidates) { + buffer.writeln('❓ "${mismatch.golden!.id}"'); + } + } + + if (report.extraCandidates.isNotEmpty) { + buffer.writeln(""); + buffer.writeln("Extra (unexpected) candidates:"); + for (final mismatch in report.extraCandidates) { + buffer.writeln('❓ "${mismatch.screenshot!.id}"'); + } + } + + // ignore: avoid_print + print(buffer.toString()); + } +} From b10a612ff6afbbd83cc74640c68201d3904e2e46 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Thu, 19 Jun 2025 18:33:01 -0300 Subject: [PATCH 3/7] PR updates --- lib/src/goldens/golden_collections.dart | 8 ++- lib/src/goldens/golden_scenes.dart | 5 +- lib/src/scenes/failure_scene.dart | 2 +- lib/src/scenes/gallery.dart | 20 +++----- lib/src/scenes/golden_scene.dart | 50 ++++++++++--------- .../scenes/golden_scene_report_printer.dart | 8 +-- 6 files changed, 50 insertions(+), 43 deletions(-) diff --git a/lib/src/goldens/golden_collections.dart b/lib/src/goldens/golden_collections.dart index 38eaef0..c69db4a 100644 --- a/lib/src/goldens/golden_collections.dart +++ b/lib/src/goldens/golden_collections.dart @@ -1,5 +1,6 @@ import 'dart:ui'; +import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; import 'package:image/image.dart' as img; /// A collection of in-memory golden images or screenshot images. @@ -12,10 +13,15 @@ import 'package:image/image.dart' as img; /// corresponding golden image. /// {@endtemplate} class GoldenCollection { - GoldenCollection(this.imagesById); + GoldenCollection( + this.imagesById, { + required this.metadata, + }); final Map imagesById; + final GoldenSceneMetadata metadata; + List get ids => imagesById.keys.toList(growable: false); bool hasId(String id) => imagesById[id] != null; diff --git a/lib/src/goldens/golden_scenes.dart b/lib/src/goldens/golden_scenes.dart index abe3a4d..321c8de 100644 --- a/lib/src/goldens/golden_scenes.dart +++ b/lib/src/goldens/golden_scenes.dart @@ -100,7 +100,10 @@ GoldenCollection _extractCollectionFromScene(GoldenSceneMetadata sceneMetadata, ); } - return GoldenCollection(goldenImages); + return GoldenCollection( + goldenImages, + metadata: sceneMetadata, + ); } RenderRepaintBoundary? _findNearestRepaintBoundary(Finder bounds) { diff --git a/lib/src/scenes/failure_scene.dart b/lib/src/scenes/failure_scene.dart index 02b2079..c8ce071 100644 --- a/lib/src/scenes/failure_scene.dart +++ b/lib/src/scenes/failure_scene.dart @@ -8,7 +8,7 @@ import 'package:flutter_test_goldens/flutter_test_goldens.dart'; /// Given a [mismatch] between a golden and a screenshot, generates an image /// that shows the golden, the screenshot, and the differences between them. -Future generateFailureScene(GoldenMismatch mismatch) async { +Future paintGoldenMismatchImages(GoldenMismatch mismatch) async { final goldenWidth = mismatch.golden!.image.width; final goldenHeight = mismatch.golden!.image.height; diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index aa3d284..24ca832 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -472,13 +472,10 @@ class Gallery { FtgLog.pipeline.fine("Comparing goldens and screenshots"); final mismatches = compareGoldenCollections(goldenCollection, screenshotCollection); - final items = []; + final items = []; final missingCandidates = []; final extraCandidates = []; - int totalPassed = 0; - int totalFailed = 0; - FtgLog.pipeline.fine("Mismatches ($existingGoldenFileName):"); for (final mismatch in mismatches.mismatches.values) { FtgLog.pipeline.fine(" - ${mismatch.golden?.id ?? mismatch.screenshot?.id}: $mismatch"); @@ -506,16 +503,16 @@ class Gallery { final mismatch = mismatches.mismatches[screenshotId]; if (mismatch == null) { // The golden check passed. - totalPassed += 1; items.add( - GoldenReportItem.success(description: goldenCollection.imagesById[screenshotId]!.id), + GoldenReport.success( + goldenCollection.metadata.images.where((image) => image.id == screenshotId).first, + ), ); } else { // The golden check failed. - totalFailed += 1; items.add( - GoldenReportItem.failure( - description: goldenCollection.imagesById[screenshotId]!.id, + GoldenReport.failure( + metadata: goldenCollection.metadata.images.where((image) => image.id == screenshotId).first, details: [ GoldenCheckDetail( status: GoldenTestStatus.failure, @@ -541,7 +538,7 @@ class Gallery { Directory(_goldenFailureDirectoryPath).createSync(); await tester.runAsync(() async { - final failureImage = await generateFailureScene(mismatch); + final failureImage = await paintGoldenMismatchImages(mismatch); await encodePngFile( "$_goldenFailureDirectoryPath/failure_${existingGoldenFileName}_${mismatch.golden!.id}.png", @@ -552,11 +549,10 @@ class Gallery { final report = GoldenSceneReport( sceneDescription: _sceneDescription, + metadata: goldenCollection.metadata, items: items, missingCandidates: missingCandidates, extraCandidates: extraCandidates, - totalPassed: totalPassed, - totalFailed: totalFailed, ); _printReport(report); diff --git a/lib/src/scenes/golden_scene.dart b/lib/src/scenes/golden_scene.dart index 1c51f78..0aac0da 100644 --- a/lib/src/scenes/golden_scene.dart +++ b/lib/src/scenes/golden_scene.dart @@ -6,6 +6,7 @@ import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test_goldens/src/goldens/golden_camera.dart'; import 'package:flutter_test_goldens/src/goldens/golden_comparisons.dart'; +import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; class GoldenScene extends StatelessWidget { const GoldenScene({ @@ -114,24 +115,27 @@ typedef GoldenSetup = FutureOr Function(WidgetTester tester); /// A report of a golden scene test. /// -/// Holds information to display the results of a golden scene test. +/// Reports the success or failure of each individual golden in the scene, as well as +/// the missing candidates and candidates that have no corresponding golden. class GoldenSceneReport { GoldenSceneReport({ required this.sceneDescription, + required this.metadata, required this.items, required this.missingCandidates, required this.extraCandidates, - required this.totalPassed, - required this.totalFailed, }); /// The human readable description of the scene. final String sceneDescription; + /// The metadata of the scene, such as the golden images and their positions. + final GoldenSceneMetadata metadata; + /// The items found in the scene. /// /// Each item might be a successful or a failed golden check. - final List items; + final List items; /// The golden candidates that were expected to be present in the scene, but were not found. final List missingCandidates; @@ -140,49 +144,47 @@ class GoldenSceneReport { final List extraCandidates; /// The total number of successful [items] in the scene. - final int totalPassed; + int get totalPassed => items.where((e) => e.status == GoldenTestStatus.success).length; /// The total number of failed [items] in the scene. - final int totalFailed; + int get totalFailed => items.where((e) => e.status == GoldenTestStatus.failure).length; } /// An item in a golden scene report. /// /// Each item represents a single gallery item that was found in both the original golden /// and the candidate image. -class GoldenReportItem { - GoldenReportItem({ - required this.status, - required this.description, - required this.details, - }); - - factory GoldenReportItem.success({ - required String description, - }) { - return GoldenReportItem( +class GoldenReport { + factory GoldenReport.success(GoldenImageMetadata metadata) { + return GoldenReport( status: GoldenTestStatus.success, - description: description, + metadata: metadata, details: [], ); } - factory GoldenReportItem.failure({ - required String description, + factory GoldenReport.failure({ + required GoldenImageMetadata metadata, required List details, }) { - return GoldenReportItem( + return GoldenReport( status: GoldenTestStatus.failure, - description: description, + metadata: metadata, details: details, ); } + GoldenReport({ + required this.status, + required this.metadata, + required this.details, + }); + /// Whether the gallery item passed or failed the golden check. final GoldenTestStatus status; - /// The description of the gallery item that was checked. - final String description; + /// The metadata of the candidate image of this report. + final GoldenImageMetadata metadata; /// The details of the golden check for this item. /// diff --git a/lib/src/scenes/golden_scene_report_printer.dart b/lib/src/scenes/golden_scene_report_printer.dart index 43fef82..4e72415 100644 --- a/lib/src/scenes/golden_scene_report_printer.dart +++ b/lib/src/scenes/golden_scene_report_printer.dart @@ -35,7 +35,7 @@ class GoldenSceneReportPrinter { buffer.writeln(""); for (final item in report.items) { if (item.status == GoldenTestStatus.success) { - buffer.writeln("✅ ${item.description}"); + buffer.writeln("✅ ${item.metadata.id}"); continue; } @@ -48,18 +48,18 @@ class GoldenSceneReportPrinter { switch (mismatch) { case WrongSizeGoldenMismatch(): buffer.writeln( - '"❌ ${item.description}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); + '"❌ ${item.metadata.id}" has an unexpected size (expected: ${mismatch.golden.size}, actual: ${mismatch.screenshot.size})'); break; case PixelGoldenMismatch(): buffer.writeln( - '"❌ ${item.description}" has a ${(mismatch.percent * 100).toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); + '"❌ ${item.metadata.id}" has a ${(mismatch.percent * 100).toStringAsFixed(2)}% (${mismatch.mismatchPixelCount}px) mismatch'); break; case MissingGoldenMismatch(): case MissingCandidateMismatch(): // Don't print anything, missing goldens are reported at the end. break; default: - buffer.writeln('"❌ ${item.description}": ${mismatch!.describe}'); + buffer.writeln('"❌ ${item.metadata.id}": ${mismatch!.describe}'); break; } } From 831e1275e03bedf4cff4589b5dfa513c47e5c0ee Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Fri, 20 Jun 2025 19:10:44 -0300 Subject: [PATCH 4/7] PR updates --- lib/src/goldens/golden_collections.dart | 7 +--- lib/src/goldens/golden_scenes.dart | 13 +++---- lib/src/scenes/film_strip.dart | 3 +- lib/src/scenes/gallery.dart | 19 ++++------ lib/src/scenes/golden_scene.dart | 35 +++++-------------- .../scenes/golden_scene_report_printer.dart | 8 ++--- 6 files changed, 28 insertions(+), 57 deletions(-) diff --git a/lib/src/goldens/golden_collections.dart b/lib/src/goldens/golden_collections.dart index c69db4a..2f8c0c2 100644 --- a/lib/src/goldens/golden_collections.dart +++ b/lib/src/goldens/golden_collections.dart @@ -13,15 +13,10 @@ import 'package:image/image.dart' as img; /// corresponding golden image. /// {@endtemplate} class GoldenCollection { - GoldenCollection( - this.imagesById, { - required this.metadata, - }); + GoldenCollection(this.imagesById); final Map imagesById; - final GoldenSceneMetadata metadata; - List get ids => imagesById.keys.toList(growable: false); bool hasId(String id) => imagesById[id] != null; diff --git a/lib/src/goldens/golden_scenes.dart b/lib/src/goldens/golden_scenes.dart index 321c8de..3bb4294 100644 --- a/lib/src/goldens/golden_scenes.dart +++ b/lib/src/goldens/golden_scenes.dart @@ -15,7 +15,7 @@ import 'package:image/image.dart'; /// /// This function loads the scene image from the [file], extracts each individual golden /// image from the scene, and then returns all of those golden images as a [GoldenCollection]. -GoldenCollection extractGoldenCollectionFromSceneFile(File file) { +(GoldenCollection, GoldenSceneMetadata) extractGoldenCollectionFromSceneFile(File file) { FtgLog.pipeline.fine("Extracting golden collection from golden image."); // Read the scene PNG data into memory. @@ -38,7 +38,7 @@ GoldenCollection extractGoldenCollectionFromSceneFile(File file) { } // Extract the golden images from the scene image. - return _extractCollectionFromScene(sceneMetadata, sceneImage); + return (_extractCollectionFromScene(sceneMetadata, sceneImage), sceneMetadata); } /// Extracts a [GoldenCollection] from a golden scene within the current widget tree. @@ -100,10 +100,7 @@ GoldenCollection _extractCollectionFromScene(GoldenSceneMetadata sceneMetadata, ); } - return GoldenCollection( - goldenImages, - metadata: sceneMetadata, - ); + return GoldenCollection(goldenImages); } RenderRepaintBoundary? _findNearestRepaintBoundary(Finder bounds) { @@ -131,6 +128,7 @@ RenderRepaintBoundary? _findNearestRepaintBoundary(Finder bounds) { class GoldenSceneMetadata { static GoldenSceneMetadata fromJson(Map json) { return GoldenSceneMetadata( + description: json["description"] ?? "", images: [ for (final imageJson in (json["images"] as List)) // GoldenImageMetadata.fromJson(imageJson), @@ -139,13 +137,16 @@ class GoldenSceneMetadata { } const GoldenSceneMetadata({ + required this.description, required this.images, }); + final String description; final List images; Map toJson() { return { + "description": description, "images": images.map((image) => image.toJson()).toList(growable: false), }; } diff --git a/lib/src/scenes/film_strip.dart b/lib/src/scenes/film_strip.dart index 5515e3d..1d4d322 100644 --- a/lib/src/scenes/film_strip.dart +++ b/lib/src/scenes/film_strip.dart @@ -277,6 +277,7 @@ class FilmStrip { // Lookup and return metadata for the position and size of each golden image // within the gallery. return GoldenSceneMetadata( + description: goldenName, images: [ for (final golden in renderablePhotos.keys) GoldenImageMetadata( @@ -339,7 +340,7 @@ class FilmStrip { // TODO: report error in structured way. throw Exception("Can't compare goldens. Golden file doesn't exist: ${goldenFile.path}"); } - final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); + final (goldenCollection, metadata) = extractGoldenCollectionFromSceneFile(goldenFile); FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); late final GoldenCollection screenshotCollection; diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index 24ca832..14f27f2 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -396,6 +396,7 @@ class Gallery { // Lookup and return metadata for the position and size of each golden image // within the gallery. return GoldenSceneMetadata( + description: _sceneDescription, images: [ for (final golden in renderablePhotos.keys) GoldenImageMetadata( @@ -459,7 +460,7 @@ class Gallery { // TODO: report error in structured way. throw Exception("Can't compare goldens. Golden file doesn't exist: ${goldenFile.path}"); } - final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); + final (goldenCollection, metadata) = extractGoldenCollectionFromSceneFile(goldenFile); // Extract scene metadata from the current widget tree. FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); @@ -505,21 +506,15 @@ class Gallery { // The golden check passed. items.add( GoldenReport.success( - goldenCollection.metadata.images.where((image) => image.id == screenshotId).first, + metadata.images.where((image) => image.id == screenshotId).first, ), ); } else { // The golden check failed. items.add( GoldenReport.failure( - metadata: goldenCollection.metadata.images.where((image) => image.id == screenshotId).first, - details: [ - GoldenCheckDetail( - status: GoldenTestStatus.failure, - description: mismatch.toString(), - mismatch: mismatch, - ), - ], + metadata: metadata.images.where((image) => image.id == screenshotId).first, + mismatch: mismatch, ), ); } @@ -548,8 +543,8 @@ class Gallery { } final report = GoldenSceneReport( - sceneDescription: _sceneDescription, - metadata: goldenCollection.metadata, + //sceneDescription: _sceneDescription, + metadata: metadata, items: items, missingCandidates: missingCandidates, extraCandidates: extraCandidates, diff --git a/lib/src/scenes/golden_scene.dart b/lib/src/scenes/golden_scene.dart index 0aac0da..10a87ad 100644 --- a/lib/src/scenes/golden_scene.dart +++ b/lib/src/scenes/golden_scene.dart @@ -119,16 +119,12 @@ typedef GoldenSetup = FutureOr Function(WidgetTester tester); /// the missing candidates and candidates that have no corresponding golden. class GoldenSceneReport { GoldenSceneReport({ - required this.sceneDescription, required this.metadata, required this.items, required this.missingCandidates, required this.extraCandidates, }); - /// The human readable description of the scene. - final String sceneDescription; - /// The metadata of the scene, such as the golden images and their positions. final GoldenSceneMetadata metadata; @@ -159,26 +155,28 @@ class GoldenReport { return GoldenReport( status: GoldenTestStatus.success, metadata: metadata, - details: [], ); } factory GoldenReport.failure({ required GoldenImageMetadata metadata, - required List details, + required GoldenMismatch mismatch, }) { return GoldenReport( status: GoldenTestStatus.failure, metadata: metadata, - details: details, + mismatch: mismatch, ); } GoldenReport({ required this.status, required this.metadata, - required this.details, - }); + this.mismatch, + }) : assert( + status == GoldenTestStatus.success || mismatch != null, + "A failure report must have a mismatch.", + ); /// Whether the gallery item passed or failed the golden check. final GoldenTestStatus status; @@ -186,24 +184,9 @@ class GoldenReport { /// The metadata of the candidate image of this report. final GoldenImageMetadata metadata; - /// The details of the golden check for this item. + /// The failure details of the gallery item, if it failed the golden check. /// - /// Might contain both successful and failed checks. - final List details; -} - -class GoldenCheckDetail { - GoldenCheckDetail({ - required this.status, - required this.description, - this.mismatch, - }) : assert( - status != GoldenTestStatus.success || mismatch == null, - "A successful golden test cannot have a mismatch", - ); - - final GoldenTestStatus status; - final String description; + /// Non-`null` if [status] is [GoldenTestStatus.failure] and `null` otherwise. final GoldenMismatch? mismatch; } diff --git a/lib/src/scenes/golden_scene_report_printer.dart b/lib/src/scenes/golden_scene_report_printer.dart index 4e72415..92af41a 100644 --- a/lib/src/scenes/golden_scene_report_printer.dart +++ b/lib/src/scenes/golden_scene_report_printer.dart @@ -12,7 +12,7 @@ class GoldenSceneReportPrinter { final buffer = StringBuffer(); // Report the summary of passed/failed tests and missing/extra candidates. - buffer.write("Golden scene has failures: ${report.sceneDescription} ("); + buffer.write("Golden scene has failures: ${report.metadata.description} ("); buffer.write("✅ ${report.totalPassed}/${report.items.length}, "); buffer.write("❌ ${report.totalFailed}/${report.items.length}"); if (report.missingCandidates.isNotEmpty || report.extraCandidates.isNotEmpty) { @@ -40,11 +40,7 @@ class GoldenSceneReportPrinter { } // This item has a failed check. - final mismatch = item.details // - .where((detail) => detail.mismatch != null) - .firstOrNull - ?.mismatch; - + final mismatch = item.mismatch; switch (mismatch) { case WrongSizeGoldenMismatch(): buffer.writeln( From 0de9b9b2b580f15f22f5da65b0523680eb8c0355 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Fri, 20 Jun 2025 19:36:48 -0300 Subject: [PATCH 5/7] Separate metadata extraction --- lib/src/goldens/golden_scenes.dart | 37 ++++++++++++++++++++++++------ lib/src/scenes/film_strip.dart | 2 +- lib/src/scenes/gallery.dart | 3 ++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/lib/src/goldens/golden_scenes.dart b/lib/src/goldens/golden_scenes.dart index 3bb4294..493d397 100644 --- a/lib/src/goldens/golden_scenes.dart +++ b/lib/src/goldens/golden_scenes.dart @@ -1,5 +1,6 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'dart:ui' as ui; import 'package:flutter/rendering.dart'; @@ -15,20 +16,17 @@ import 'package:image/image.dart'; /// /// This function loads the scene image from the [file], extracts each individual golden /// image from the scene, and then returns all of those golden images as a [GoldenCollection]. -(GoldenCollection, GoldenSceneMetadata) extractGoldenCollectionFromSceneFile(File file) { +GoldenCollection extractGoldenCollectionFromSceneFile(File file) { FtgLog.pipeline.fine("Extracting golden collection from golden image."); // Read the scene PNG data into memory. final scenePngBytes = file.readAsBytesSync(); // Extract scene metadata from PNG. - final pngText = scenePngBytes.readTextMetadata(); - final sceneJsonText = pngText["flutter_test_goldens"]; - if (sceneJsonText == null) { + final sceneMetadata = _extractGoldenSceneMetadataFromBytes(scenePngBytes); + if (sceneMetadata == null) { throw Exception("Golden image is missing scene metadata: ${file.path}"); } - final sceneJson = JsonDecoder().convert(sceneJsonText); - final sceneMetadata = GoldenSceneMetadata.fromJson(sceneJson); // Decode PNG data to an image. final sceneImage = decodePng(scenePngBytes); @@ -38,7 +36,7 @@ import 'package:image/image.dart'; } // Extract the golden images from the scene image. - return (_extractCollectionFromScene(sceneMetadata, sceneImage), sceneMetadata); + return _extractCollectionFromScene(sceneMetadata, sceneImage); } /// Extracts a [GoldenCollection] from a golden scene within the current widget tree. @@ -84,6 +82,31 @@ Future extractGoldenCollectionFromSceneWidgetTree( return _extractCollectionFromScene(sceneMetadata, treeImage); } +/// Extracts then golden scene metadata within the given image [file]. +GoldenSceneMetadata extractGoldenSceneMetadataFromFile(File file) { + // Read the scene PNG data into memory. + final scenePngBytes = file.readAsBytesSync(); + + // Extract scene metadata from PNG. + final sceneMetadata = _extractGoldenSceneMetadataFromBytes(scenePngBytes); + if (sceneMetadata == null) { + throw Exception("Golden image is missing scene metadata: ${file.path}"); + } + + return sceneMetadata; +} + +GoldenSceneMetadata? _extractGoldenSceneMetadataFromBytes(Uint8List pngBytes) { + // Extract scene metadata from PNG. + final pngText = pngBytes.readTextMetadata(); + final sceneJsonText = pngText["flutter_test_goldens"]; + if (sceneJsonText == null) { + return null; + } + final sceneJson = JsonDecoder().convert(sceneJsonText); + return GoldenSceneMetadata.fromJson(sceneJson); +} + GoldenCollection _extractCollectionFromScene(GoldenSceneMetadata sceneMetadata, Image sceneImage) { // Cut each golden image out of the scene. final goldenImages = {}; diff --git a/lib/src/scenes/film_strip.dart b/lib/src/scenes/film_strip.dart index 1d4d322..8d84045 100644 --- a/lib/src/scenes/film_strip.dart +++ b/lib/src/scenes/film_strip.dart @@ -340,7 +340,7 @@ class FilmStrip { // TODO: report error in structured way. throw Exception("Can't compare goldens. Golden file doesn't exist: ${goldenFile.path}"); } - final (goldenCollection, metadata) = extractGoldenCollectionFromSceneFile(goldenFile); + final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); late final GoldenCollection screenshotCollection; diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index 14f27f2..c5e25ae 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -460,7 +460,8 @@ class Gallery { // TODO: report error in structured way. throw Exception("Can't compare goldens. Golden file doesn't exist: ${goldenFile.path}"); } - final (goldenCollection, metadata) = extractGoldenCollectionFromSceneFile(goldenFile); + final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); + final metadata = extractGoldenSceneMetadataFromFile(goldenFile); // Extract scene metadata from the current widget tree. FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); From 8b18ceec092a45d9572db0d91bb00deda578ca52 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Fri, 20 Jun 2025 19:52:14 -0300 Subject: [PATCH 6/7] PR updates --- lib/src/goldens/golden_scenes.dart | 33 +++++------------------------- lib/src/scenes/gallery.dart | 11 +++++++++- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/lib/src/goldens/golden_scenes.dart b/lib/src/goldens/golden_scenes.dart index 493d397..a25ac05 100644 --- a/lib/src/goldens/golden_scenes.dart +++ b/lib/src/goldens/golden_scenes.dart @@ -1,6 +1,5 @@ import 'dart:convert'; import 'dart:io'; -import 'dart:typed_data'; import 'dart:ui' as ui; import 'package:flutter/rendering.dart'; @@ -23,10 +22,13 @@ GoldenCollection extractGoldenCollectionFromSceneFile(File file) { final scenePngBytes = file.readAsBytesSync(); // Extract scene metadata from PNG. - final sceneMetadata = _extractGoldenSceneMetadataFromBytes(scenePngBytes); - if (sceneMetadata == null) { + final pngText = scenePngBytes.readTextMetadata(); + final sceneJsonText = pngText["flutter_test_goldens"]; + if (sceneJsonText == null) { throw Exception("Golden image is missing scene metadata: ${file.path}"); } + final sceneJson = JsonDecoder().convert(sceneJsonText); + final sceneMetadata = GoldenSceneMetadata.fromJson(sceneJson); // Decode PNG data to an image. final sceneImage = decodePng(scenePngBytes); @@ -82,31 +84,6 @@ Future extractGoldenCollectionFromSceneWidgetTree( return _extractCollectionFromScene(sceneMetadata, treeImage); } -/// Extracts then golden scene metadata within the given image [file]. -GoldenSceneMetadata extractGoldenSceneMetadataFromFile(File file) { - // Read the scene PNG data into memory. - final scenePngBytes = file.readAsBytesSync(); - - // Extract scene metadata from PNG. - final sceneMetadata = _extractGoldenSceneMetadataFromBytes(scenePngBytes); - if (sceneMetadata == null) { - throw Exception("Golden image is missing scene metadata: ${file.path}"); - } - - return sceneMetadata; -} - -GoldenSceneMetadata? _extractGoldenSceneMetadataFromBytes(Uint8List pngBytes) { - // Extract scene metadata from PNG. - final pngText = pngBytes.readTextMetadata(); - final sceneJsonText = pngText["flutter_test_goldens"]; - if (sceneJsonText == null) { - return null; - } - final sceneJson = JsonDecoder().convert(sceneJsonText); - return GoldenSceneMetadata.fromJson(sceneJson); -} - GoldenCollection _extractCollectionFromScene(GoldenSceneMetadata sceneMetadata, Image sceneImage) { // Cut each golden image out of the scene. final goldenImages = {}; diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index c5e25ae..4f76706 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -461,7 +461,16 @@ class Gallery { throw Exception("Can't compare goldens. Golden file doesn't exist: ${goldenFile.path}"); } final goldenCollection = extractGoldenCollectionFromSceneFile(goldenFile); - final metadata = extractGoldenSceneMetadataFromFile(goldenFile); + + // Extract scene metadata from the existing golden file. + final scenePngBytes = goldenFile.readAsBytesSync(); + final pngText = scenePngBytes.readTextMetadata(); + final sceneJsonText = pngText["flutter_test_goldens"]; + if (sceneJsonText == null) { + throw Exception("Golden image is missing scene metadata: ${goldenFile.path}"); + } + final sceneJson = JsonDecoder().convert(sceneJsonText); + final metadata = GoldenSceneMetadata.fromJson(sceneJson); // Extract scene metadata from the current widget tree. FtgLog.pipeline.fine("Extracting golden collection from current widget tree (screenshots)."); From a91178290d1cf79bb87bd7db5792a976fedaa803 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Fri, 20 Jun 2025 20:14:06 -0300 Subject: [PATCH 7/7] PR updates --- lib/src/goldens/golden_collections.dart | 1 - lib/src/scenes/gallery.dart | 10 +++++----- lib/src/scenes/golden_scene.dart | 11 ++++++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/src/goldens/golden_collections.dart b/lib/src/goldens/golden_collections.dart index 2f8c0c2..38eaef0 100644 --- a/lib/src/goldens/golden_collections.dart +++ b/lib/src/goldens/golden_collections.dart @@ -1,6 +1,5 @@ import 'dart:ui'; -import 'package:flutter_test_goldens/src/goldens/golden_scenes.dart'; import 'package:image/image.dart' as img; /// A collection of in-memory golden images or screenshot images. diff --git a/lib/src/scenes/gallery.dart b/lib/src/scenes/gallery.dart index 4f76706..716f4bd 100644 --- a/lib/src/scenes/gallery.dart +++ b/lib/src/scenes/gallery.dart @@ -511,19 +511,20 @@ class Gallery { continue; } + // Find the golden metadata for this candidate. + final goldenMetadata = metadata.images.where((image) => image.id == screenshotId).first; + final mismatch = mismatches.mismatches[screenshotId]; if (mismatch == null) { // The golden check passed. items.add( - GoldenReport.success( - metadata.images.where((image) => image.id == screenshotId).first, - ), + GoldenReport.success(goldenMetadata), ); } else { // The golden check failed. items.add( GoldenReport.failure( - metadata: metadata.images.where((image) => image.id == screenshotId).first, + metadata: goldenMetadata, mismatch: mismatch, ), ); @@ -553,7 +554,6 @@ class Gallery { } final report = GoldenSceneReport( - //sceneDescription: _sceneDescription, metadata: metadata, items: items, missingCandidates: missingCandidates, diff --git a/lib/src/scenes/golden_scene.dart b/lib/src/scenes/golden_scene.dart index 10a87ad..5903e32 100644 --- a/lib/src/scenes/golden_scene.dart +++ b/lib/src/scenes/golden_scene.dart @@ -143,13 +143,18 @@ class GoldenSceneReport { int get totalPassed => items.where((e) => e.status == GoldenTestStatus.success).length; /// The total number of failed [items] in the scene. + /// + /// Only candidates that have a corresponding golden image and failed the golden check + /// count as a failure. + /// + /// See [missingCandidates] for candidates that were expected but not found, + /// and [extraCandidates] for candidates that were found but not expected. int get totalFailed => items.where((e) => e.status == GoldenTestStatus.failure).length; } -/// An item in a golden scene report. +/// A report of success or failure for a single golden within a scene. /// -/// Each item represents a single gallery item that was found in both the original golden -/// and the candidate image. +/// A [GoldenReport] holds the test results for a candidate that has a corresponding golden. class GoldenReport { factory GoldenReport.success(GoldenImageMetadata metadata) { return GoldenReport(