Skip to content

Commit 619222d

Browse files
Run golden tests in CI (Resolves #36) (#60)
* Added support for pixel tolerances * Regenerated all goldens using golden_runner to pass GitHub CI
1 parent 1e124fc commit 619222d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+263
-25
lines changed

.github/workflows/pr_validation.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,23 @@ on:
33
pull_request:
44

55
jobs:
6+
test_goldens:
7+
runs-on: ubuntu-latest
8+
steps:
9+
# Checkout the repository
10+
- uses: actions/checkout@v3
11+
12+
# Setup Flutter environment
13+
- uses: subosito/flutter-action@v2
14+
with:
15+
channel: "stable"
16+
17+
# Download all the packages that the app uses
18+
- run: flutter pub get
19+
20+
# Run all tests
21+
- run: flutter test test_goldens
22+
623
build_website:
724
runs-on: ubuntu-latest
825
defaults:

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
test/**/failures/
2+
test_goldens/**/failures/
3+
14
# Miscellaneous
25
*.class
36
*.log

LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2022 Declarative, Inc.
1+
Copyright (c) 2025 Declarative, Inc.
22

33
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
44

doc/website/source/styles/docs_page_layout.scss

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -263,15 +263,15 @@ main.page-content {
263263

264264
p {
265265
margin-bottom: 1.5em;
266+
}
266267

267-
code {
268-
padding: 3px 6px;
269-
background: #7f00a6;
270-
border: 1px solid #a218cc;
271-
border-radius: 4px;
268+
code {
269+
padding: 3px 6px;
270+
background: #7f00a6;
271+
border: 1px solid #a218cc;
272+
border-radius: 4px;
272273

273-
color: WHITE;
274-
}
274+
color: WHITE;
275275
}
276276

277277
li {

golden_tester.Dockerfile

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM ubuntu:latest
2+
3+
ENV FLUTTER_HOME=${HOME}/sdks/flutter
4+
ENV PATH ${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin
5+
6+
USER root
7+
8+
RUN apt update
9+
10+
RUN apt install -y git curl unzip
11+
12+
# Print the Ubuntu version. Useful when there are failing tests.
13+
RUN cat /etc/lsb-release
14+
15+
# Invalidate the cache when flutter pushes a new commit.
16+
ADD https://api.github.com/repos/flutter/flutter/git/refs/heads/master ./flutter-latest-master
17+
18+
RUN git clone https://github.com/flutter/flutter.git ${FLUTTER_HOME}
19+
20+
RUN flutter doctor
21+
22+
# Copy the whole repo.
23+
# We need this because we use local dependencies.
24+
COPY ./ /golden_tester

lib/flutter_test_goldens.dart

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1+
export 'src/flutter/flutter_camera.dart';
2+
export 'src/flutter/flutter_golden_matcher.dart';
13
export 'src/flutter/flutter_test_extensions.dart';
24
export 'src/fonts/fonts.dart';
35
export 'src/fonts/icons.dart';
4-
export 'src/flutter/flutter_camera.dart';
56
export 'src/goldens/golden_collections.dart';
67
export 'src/goldens/golden_comparisons.dart';
78
export 'src/goldens/golden_rendering.dart';
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import 'dart:io';
2+
import 'dart:typed_data';
3+
4+
import 'package:flutter/foundation.dart';
5+
import 'package:flutter_test/flutter_test.dart';
6+
import 'package:path/path.dart';
7+
8+
/// A matcher that expects given content to match the golden file referenced
9+
/// by [key], allowing up to [maxPixelMismatchCount] different pixels before
10+
/// considering the test to be a failure.
11+
///
12+
/// Typically, the [key] is expected to be a relative file path from the given
13+
/// test file, to the golden file, e.g., "goldens/my-golden-name.png".
14+
///
15+
/// This matcher can be used by calling it in `expectLater()`, e.g.,
16+
///
17+
/// await expectLater(
18+
/// find.byType(MaterialApp),
19+
/// matchesGoldenFileWithPixelAllowance("goldens/my-golden-name.png", 20),
20+
/// );
21+
///
22+
/// Typically, Flutter's golden system describes mismatches in terms of percentages.
23+
/// But percentages are difficult to depend upon. Sometimes a relatively large percentage
24+
/// doesn't matter, and sometimes a tiny percentage is critical. When it comes to ignoring
25+
/// irrelevant mismatches, it's often more convenient to work in terms of pixels. This
26+
/// matcher lets developers specify a maximum pixel mismatch count, instead of relying on
27+
/// percentage differences across the entire golden image.
28+
MatchesGoldenFile matchesGoldenFileWithPixelAllowance(Object key, int maxPixelMismatchCount, {int? version}) {
29+
if (key is Uri) {
30+
return MatchesGoldenFileWithPixelAllowance(key, maxPixelMismatchCount, version);
31+
} else if (key is String) {
32+
return MatchesGoldenFileWithPixelAllowance.forStringPath(key, maxPixelMismatchCount, version);
33+
}
34+
throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}');
35+
}
36+
37+
/// A special version of [MatchesGoldenFile] that allows a specified number of
38+
/// pixels to be different between golden files before considering the test to
39+
/// be a failure.
40+
///
41+
/// Typically, this matcher is expected to be created by calling
42+
/// [matchesGoldenFileWithPixelAllowance].
43+
class MatchesGoldenFileWithPixelAllowance extends MatchesGoldenFile {
44+
/// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
45+
/// file at the relative path within the [key] URI.
46+
///
47+
/// The [key] URI should be a relative path from the executing test's
48+
/// directory to the golden file, e.g., "goldens/my-golden-name.png".
49+
MatchesGoldenFileWithPixelAllowance(super.key, this._maxPixelMismatchCount, [super.version]);
50+
51+
/// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
52+
/// file at the relative [path].
53+
///
54+
/// The [path] should be relative to the executing test's directory, e.g.,
55+
/// "goldens/my-golden-name.png".
56+
MatchesGoldenFileWithPixelAllowance.forStringPath(String path, this._maxPixelMismatchCount, [int? version])
57+
: super.forStringPath(path, version);
58+
59+
final int _maxPixelMismatchCount;
60+
61+
@override
62+
Future<String?> matchAsync(dynamic item) async {
63+
// Cache the current goldenFileComparator so we can restore
64+
// it after the test.
65+
final originalComparator = goldenFileComparator;
66+
67+
try {
68+
goldenFileComparator = PixelDiffGoldenComparator(
69+
(goldenFileComparator as LocalFileComparator).basedir.path,
70+
pixelCount: _maxPixelMismatchCount,
71+
);
72+
73+
return await super.matchAsync(item);
74+
} finally {
75+
goldenFileComparator = originalComparator;
76+
}
77+
}
78+
}
79+
80+
/// A golden file comparator that allows a specified number of pixels
81+
/// to be different between the golden image file and the test image file, and
82+
/// still pass.
83+
class PixelDiffGoldenComparator extends LocalFileComparator {
84+
PixelDiffGoldenComparator(
85+
String testBaseDirectory, {
86+
required int pixelCount,
87+
}) : _testBaseDirectory = testBaseDirectory,
88+
_maxPixelMismatchCount = pixelCount,
89+
super(Uri.parse(testBaseDirectory));
90+
91+
@override
92+
Uri get basedir => Uri.parse(_testBaseDirectory);
93+
94+
/// The file system path to the directory that holds the currently executing
95+
/// Dart test file.
96+
final String _testBaseDirectory;
97+
98+
/// The maximum number of mismatched pixels for which this pixel test
99+
/// is considered a success/pass.
100+
final int _maxPixelMismatchCount;
101+
102+
@override
103+
Future<bool> compare(Uint8List imageBytes, Uri golden) async {
104+
// Note: the incoming `golden` Uri is a partial path from the currently
105+
// executing test directory to the golden file, e.g., "goldens/my-test.png".
106+
final result = await GoldenFileComparator.compareLists(
107+
imageBytes,
108+
await getGoldenBytes(golden),
109+
);
110+
111+
if (result.passed) {
112+
return true;
113+
}
114+
115+
final diffImage = result.diffs!.entries.first.value;
116+
final pixelCount = diffImage.width * diffImage.height;
117+
final pixelMismatchCount = pixelCount * result.diffPercent;
118+
119+
if (pixelMismatchCount <= _maxPixelMismatchCount) {
120+
return true;
121+
}
122+
123+
// Paint the golden diffs and images to failure files.
124+
await generateFailureOutput(result, golden, basedir);
125+
throw FlutterError(
126+
"Pixel test failed. ${result.diffPercent.toStringAsFixed(2)}% diff, $pixelMismatchCount pixel count diff (max allowed pixel mismatch count is $_maxPixelMismatchCount)");
127+
}
128+
129+
@override
130+
@protected
131+
Future<List<int>> getGoldenBytes(Uri golden) async {
132+
final File goldenFile = _getGoldenFile(golden);
133+
if (!goldenFile.existsSync()) {
134+
fail('Could not be compared against non-existent file: "$golden"');
135+
}
136+
final List<int> goldenBytes = await goldenFile.readAsBytes();
137+
return goldenBytes;
138+
}
139+
140+
File _getGoldenFile(Uri golden) => File(join(_testBaseDirectory, fromUri(golden.path)));
141+
}

lib/src/goldens/golden_comparisons.dart

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ import 'package:flutter_test_goldens/src/logging.dart';
44
/// Compares new [screenshots] to existing [goldens] and reports any mismatches between them.
55
GoldenCollectionMismatches compareGoldenCollections(
66
ScreenshotCollection goldens,
7-
ScreenshotCollection screenshots,
8-
) {
7+
ScreenshotCollection screenshots, {
8+
Map<String, int> tolerances = const {},
9+
}) {
910
final mismatches = <String, GoldenMismatch>{};
1011

1112
// For every golden, look for missing and mismatching screenshots.
@@ -30,7 +31,8 @@ GoldenCollectionMismatches compareGoldenCollections(
3031

3132
// The golden and screenshot have the same size. Look for a pixel mismatch.
3233
final mismatchPixelCount = _calculatePixelMismatch(golden, screenshot);
33-
if (mismatchPixelCount > 0) {
34+
final tolerance = tolerances[golden.id] ?? 0;
35+
if (mismatchPixelCount > tolerance) {
3436
mismatches[id] = PixelGoldenMismatch(
3537
golden: golden,
3638
screenshot: screenshot,

0 commit comments

Comments
 (0)