Skip to content

Commit c9c17d0

Browse files
Added pixel tolerances to Gallery and SingleShot. Added altered version of Flutter's standard matcher, which also allows pixel tolerances (pulled from super_editor).
1 parent 43fefb6 commit c9c17d0

File tree

6 files changed

+199
-11
lines changed

6 files changed

+199
-11
lines changed

lib/flutter_test_goldens.dart

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1+
export 'src/flutter/flutter_camera.dart';
2+
export 'src/flutter/flutter_golden_matcher.dart';
13
export 'src/flutter/flutter_test_extensions.dart';
24
export 'src/fonts/fonts.dart';
35
export 'src/fonts/icons.dart';
4-
export 'src/flutter/flutter_camera.dart';
56
export 'src/goldens/golden_collections.dart';
67
export 'src/goldens/golden_comparisons.dart';
78
export 'src/goldens/golden_rendering.dart';
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import 'dart:io';
2+
import 'dart:typed_data';
3+
4+
import 'package:flutter/foundation.dart';
5+
import 'package:flutter_test/flutter_test.dart';
6+
import 'package:path/path.dart';
7+
8+
/// A matcher that expects given content to match the golden file referenced
9+
/// by [key], allowing up to [maxPixelMismatchCount] different pixels before
10+
/// considering the test to be a failure.
11+
///
12+
/// Typically, the [key] is expected to be a relative file path from the given
13+
/// test file, to the golden file, e.g., "goldens/my-golden-name.png".
14+
///
15+
/// This matcher can be used by calling it in `expectLater()`, e.g.,
16+
///
17+
/// await expectLater(
18+
/// find.byType(MaterialApp),
19+
/// matchesGoldenFileWithPixelAllowance("goldens/my-golden-name.png", 20),
20+
/// );
21+
///
22+
/// Typically, Flutter's golden system describes mismatches in terms of percentages.
23+
/// But percentages are difficult to depend upon. Sometimes a relatively large percentage
24+
/// doesn't matter, and sometimes a tiny percentage is critical. When it comes to ignoring
25+
/// irrelevant mismatches, it's often more convenient to work in terms of pixels. This
26+
/// matcher lets developers specify a maximum pixel mismatch count, instead of relying on
27+
/// percentage differences across the entire golden image.
28+
MatchesGoldenFile matchesGoldenFileWithPixelAllowance(Object key, int maxPixelMismatchCount, {int? version}) {
29+
if (key is Uri) {
30+
return MatchesGoldenFileWithPixelAllowance(key, maxPixelMismatchCount, version);
31+
} else if (key is String) {
32+
return MatchesGoldenFileWithPixelAllowance.forStringPath(key, maxPixelMismatchCount, version);
33+
}
34+
throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}');
35+
}
36+
37+
/// A special version of [MatchesGoldenFile] that allows a specified number of
38+
/// pixels to be different between golden files before considering the test to
39+
/// be a failure.
40+
///
41+
/// Typically, this matcher is expected to be created by calling
42+
/// [matchesGoldenFileWithPixelAllowance].
43+
class MatchesGoldenFileWithPixelAllowance extends MatchesGoldenFile {
44+
/// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
45+
/// file at the relative path within the [key] URI.
46+
///
47+
/// The [key] URI should be a relative path from the executing test's
48+
/// directory to the golden file, e.g., "goldens/my-golden-name.png".
49+
MatchesGoldenFileWithPixelAllowance(super.key, this._maxPixelMismatchCount, [super.version]);
50+
51+
/// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
52+
/// file at the relative [path].
53+
///
54+
/// The [path] should be relative to the executing test's directory, e.g.,
55+
/// "goldens/my-golden-name.png".
56+
MatchesGoldenFileWithPixelAllowance.forStringPath(String path, this._maxPixelMismatchCount, [int? version])
57+
: super.forStringPath(path, version);
58+
59+
final int _maxPixelMismatchCount;
60+
61+
@override
62+
Future<String?> matchAsync(dynamic item) async {
63+
// Cache the current goldenFileComparator so we can restore
64+
// it after the test.
65+
final originalComparator = goldenFileComparator;
66+
67+
try {
68+
goldenFileComparator = PixelDiffGoldenComparator(
69+
(goldenFileComparator as LocalFileComparator).basedir.path,
70+
pixelCount: _maxPixelMismatchCount,
71+
);
72+
73+
return await super.matchAsync(item);
74+
} finally {
75+
goldenFileComparator = originalComparator;
76+
}
77+
}
78+
}
79+
80+
/// A golden file comparator that allows a specified number of pixels
81+
/// to be different between the golden image file and the test image file, and
82+
/// still pass.
83+
class PixelDiffGoldenComparator extends LocalFileComparator {
84+
PixelDiffGoldenComparator(
85+
String testBaseDirectory, {
86+
required int pixelCount,
87+
}) : _testBaseDirectory = testBaseDirectory,
88+
_maxPixelMismatchCount = pixelCount,
89+
super(Uri.parse(testBaseDirectory));
90+
91+
@override
92+
Uri get basedir => Uri.parse(_testBaseDirectory);
93+
94+
/// The file system path to the directory that holds the currently executing
95+
/// Dart test file.
96+
final String _testBaseDirectory;
97+
98+
/// The maximum number of mismatched pixels for which this pixel test
99+
/// is considered a success/pass.
100+
final int _maxPixelMismatchCount;
101+
102+
@override
103+
Future<bool> compare(Uint8List imageBytes, Uri golden) async {
104+
// Note: the incoming `golden` Uri is a partial path from the currently
105+
// executing test directory to the golden file, e.g., "goldens/my-test.png".
106+
final result = await GoldenFileComparator.compareLists(
107+
imageBytes,
108+
await getGoldenBytes(golden),
109+
);
110+
111+
if (result.passed) {
112+
return true;
113+
}
114+
115+
final diffImage = result.diffs!.entries.first.value;
116+
final pixelCount = diffImage.width * diffImage.height;
117+
final pixelMismatchCount = pixelCount * result.diffPercent;
118+
119+
if (pixelMismatchCount <= _maxPixelMismatchCount) {
120+
return true;
121+
}
122+
123+
// Paint the golden diffs and images to failure files.
124+
await generateFailureOutput(result, golden, basedir);
125+
throw FlutterError(
126+
"Pixel test failed. ${result.diffPercent.toStringAsFixed(2)}% diff, $pixelMismatchCount pixel count diff (max allowed pixel mismatch count is $_maxPixelMismatchCount)");
127+
}
128+
129+
@override
130+
@protected
131+
Future<List<int>> getGoldenBytes(Uri golden) async {
132+
final File goldenFile = _getGoldenFile(golden);
133+
if (!goldenFile.existsSync()) {
134+
fail('Could not be compared against non-existent file: "$golden"');
135+
}
136+
final List<int> goldenBytes = await goldenFile.readAsBytes();
137+
return goldenBytes;
138+
}
139+
140+
File _getGoldenFile(Uri golden) => File(join(_testBaseDirectory, fromUri(golden.path)));
141+
}

lib/src/goldens/golden_comparisons.dart

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ import 'package:flutter_test_goldens/src/logging.dart';
44
/// Compares new [screenshots] to existing [goldens] and reports any mismatches between them.
55
GoldenCollectionMismatches compareGoldenCollections(
66
ScreenshotCollection goldens,
7-
ScreenshotCollection screenshots,
8-
) {
7+
ScreenshotCollection screenshots, {
8+
Map<String, int> tolerances = const {},
9+
}) {
910
final mismatches = <String, GoldenMismatch>{};
1011

1112
// For every golden, look for missing and mismatching screenshots.
@@ -30,7 +31,8 @@ GoldenCollectionMismatches compareGoldenCollections(
3031

3132
// The golden and screenshot have the same size. Look for a pixel mismatch.
3233
final mismatchPixelCount = _calculatePixelMismatch(golden, screenshot);
33-
if (mismatchPixelCount > 0) {
34+
final tolerance = tolerances[golden.id] ?? 0;
35+
if (mismatchPixelCount > tolerance) {
3436
mismatches[id] = PixelGoldenMismatch(
3537
golden: golden,
3638
screenshot: screenshot,

lib/src/scenes/gallery.dart

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ class Gallery {
109109
BoxConstraints? constraints,
110110
Finder? boundsFinder,
111111
GoldenSetup? setup,
112+
int tolerancePx = 0,
112113
required Widget widget,
113114
}) {
114115
assert(
@@ -131,6 +132,7 @@ class Gallery {
131132
constraints: constraints,
132133
boundsFinder: boundsFinder,
133134
setup: setup,
135+
tolerancePx: tolerancePx,
134136
child: widget,
135137
);
136138
}
@@ -146,6 +148,7 @@ class Gallery {
146148
constraints: constraints,
147149
boundsFinder: boundsFinder,
148150
setup: setup,
151+
tolerancePx: tolerancePx,
149152
child: widget,
150153
);
151154

@@ -162,6 +165,7 @@ class Gallery {
162165
BoxConstraints? constraints,
163166
Finder? boundsFinder,
164167
GoldenSetup? setup,
168+
int tolerancePx = 0,
165169
required WidgetBuilder builder,
166170
}) {
167171
assert(
@@ -184,6 +188,7 @@ class Gallery {
184188
constraints: constraints,
185189
boundsFinder: boundsFinder,
186190
setup: setup,
191+
tolerancePx: tolerancePx,
187192
builder: builder,
188193
);
189194
}
@@ -199,6 +204,7 @@ class Gallery {
199204
constraints: constraints,
200205
boundsFinder: boundsFinder,
201206
setup: setup,
207+
tolerancePx: tolerancePx,
202208
builder: builder,
203209
);
204210

@@ -231,6 +237,7 @@ class Gallery {
231237
BoxConstraints? constraints,
232238
Finder? boundsFinder,
233239
GoldenSetup? setup,
240+
int tolerancePx = 0,
234241
required GoldenSceneItemPumper pumper,
235242
}) {
236243
assert(
@@ -253,6 +260,7 @@ class Gallery {
253260
constraints: constraints,
254261
boundsFinder: boundsFinder,
255262
setup: setup,
263+
tolerancePx: tolerancePx,
256264
pumper: pumper,
257265
);
258266
}
@@ -268,6 +276,7 @@ class Gallery {
268276
constraints: constraints,
269277
boundsFinder: boundsFinder,
270278
setup: setup,
279+
tolerancePx: tolerancePx,
271280
pumper: pumper,
272281
);
273282

@@ -301,11 +310,7 @@ class Gallery {
301310
// Compare to existing goldens.
302311
FtgLog.pipeline.finer("Comparing existing goldens...");
303312
// TODO: Return a success/failure report that we can publish to the test output.
304-
await _compareGoldens(
305-
tester,
306-
_fileName,
307-
screenshots,
308-
);
313+
await _compareGoldens(tester, _fileName, screenshots);
309314
FtgLog.pipeline.finer("Done comparing goldens.");
310315
}
311316

@@ -545,7 +550,11 @@ Image.memory(
545550

546551
// Compare goldens in the scene.
547552
FtgLog.pipeline.fine("Comparing goldens and screenshots");
548-
final mismatches = compareGoldenCollections(goldenCollection, ScreenshotCollection(candidateCollection));
553+
final mismatches = compareGoldenCollections(
554+
goldenCollection,
555+
ScreenshotCollection(candidateCollection),
556+
tolerances: _requests.map((id, request) => MapEntry(id, request.tolerancePx)),
557+
);
549558

550559
final items = <GoldenReport>[];
551560
final missingCandidates = <MissingCandidateMismatch>[];
@@ -659,6 +668,7 @@ class GalleryGoldenRequest {
659668
this.constraints,
660669
Finder? boundsFinder,
661670
this.setup,
671+
this.tolerancePx = 0,
662672
required this.child,
663673
}) : pumper = null,
664674
builder = null {
@@ -673,6 +683,7 @@ class GalleryGoldenRequest {
673683
this.constraints,
674684
Finder? boundsFinder,
675685
this.setup,
686+
this.tolerancePx = 0,
676687
required this.builder,
677688
}) : pumper = null,
678689
child = null {
@@ -687,6 +698,7 @@ class GalleryGoldenRequest {
687698
this.constraints,
688699
Finder? boundsFinder,
689700
this.setup,
701+
this.tolerancePx = 0,
690702
required this.pumper,
691703
}) : builder = null,
692704
child = null {
@@ -721,6 +733,18 @@ class GalleryGoldenRequest {
721733
/// into the widget tree, but before the screenshot is taken.
722734
final GoldenSetup? setup;
723735

736+
/// {@template tolerance}
737+
/// The number of mismatched pixels that are permitted for this item before triggering
738+
/// a test failure.
739+
///
740+
/// Tolerance is designed primarily for situations where local runs can't match CI runs.
741+
/// For example, there are situations where using an Ubuntu Docker runner locally produces
742+
/// different screenshots than the GitHub Ubuntu runner. When this happens, there's no
743+
/// good answer. In such cases, it typically suffices to add a tolerance for the small number
744+
/// of pixels that don't match.
745+
/// {@endtemplate}
746+
final int tolerancePx;
747+
724748
/// The [GalleryItemPumper] that creates this gallery item, or `null` if this gallery
725749
/// item is created with a [builder] or a [child].
726750
final GoldenSceneItemPumper? pumper;

lib/src/scenes/single_shot.dart

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,15 @@ class SingleShotConfigurator {
7878
);
7979
}
8080

81+
SingleShotConfigurator withTolerance(int tolerancePx) {
82+
_ensureStepNotComplete("tolerance");
83+
84+
return SingleShotConfigurator(
85+
_config.copyWith(tolerancePx: tolerancePx),
86+
{..._stepsCompleted, "tolerance"},
87+
);
88+
}
89+
8190
void _ensureStepNotComplete(String name) {
8291
if (!_stepsCompleted.contains(name)) {
8392
return;
@@ -104,6 +113,7 @@ class SingleShotConfigurator {
104113
widget: _config.widget!,
105114
constraints: _config.constraints,
106115
boundsFinder: _config.boundsFinder,
116+
tolerancePx: _config.tolerancePx ?? 0,
107117
setup: _config.setup,
108118
);
109119
} else if (_config.builder != null) {
@@ -113,6 +123,7 @@ class SingleShotConfigurator {
113123
constraints: _config.constraints,
114124
builder: _config.builder!,
115125
boundsFinder: _config.boundsFinder,
126+
tolerancePx: _config.tolerancePx ?? 0,
116127
setup: _config.setup,
117128
);
118129
} else {
@@ -122,6 +133,7 @@ class SingleShotConfigurator {
122133
constraints: _config.constraints,
123134
pumper: _config.pumper!,
124135
boundsFinder: _config.boundsFinder,
136+
tolerancePx: _config.tolerancePx ?? 0,
125137
setup: _config.setup,
126138
);
127139
}
@@ -142,6 +154,7 @@ class SingleShotConfiguration {
142154
this.builder,
143155
this.pumper,
144156
this.setup,
157+
this.tolerancePx,
145158
this.boundsFinder,
146159
});
147160

@@ -161,6 +174,9 @@ class SingleShotConfiguration {
161174

162175
final SceneLayout? sceneLayout;
163176

177+
/// {@macro tolerance}
178+
final int? tolerancePx;
179+
164180
final Widget? widget;
165181
final WidgetBuilder? builder;
166182
final GoldenSceneItemPumper? pumper;
@@ -176,6 +192,7 @@ class SingleShotConfiguration {
176192
BoxConstraints? constraints,
177193
GoldenSceneItemScaffold? itemScaffold,
178194
SceneLayout? sceneLayout,
195+
int? tolerancePx,
179196
Widget? widget,
180197
WidgetBuilder? builder,
181198
GoldenSceneItemPumper? pumper,
@@ -189,6 +206,7 @@ class SingleShotConfiguration {
189206
constraints: constraints ?? this.constraints,
190207
itemScaffold: itemScaffold ?? this.itemScaffold,
191208
sceneLayout: sceneLayout ?? this.sceneLayout,
209+
tolerancePx: tolerancePx ?? this.tolerancePx,
192210
widget: widget ?? this.widget,
193211
builder: builder ?? this.builder,
194212
pumper: pumper ?? this.pumper,

0 commit comments

Comments
 (0)