Run golden tests in CI (Resolves #36)

.github/workflows/pr_validation.yaml

@@ -3,6 +3,23 @@ on:
   pull_request:
 
 jobs:
+  test_goldens:
+    runs-on: ubuntu-latest
+    steps:
+      # Checkout the repository
+      - uses: actions/checkout@v3
+
+      # Setup Flutter environment
+      - uses: subosito/flutter-action@v2
+        with:
+          channel: "stable"
+
+      # Download all the packages that the app uses
+      - run: flutter pub get
+
+      # Run all tests
+      - run: flutter test test_goldens
+
   build_website:
     runs-on: ubuntu-latest
     defaults:

.gitignore

@@ -1,3 +1,6 @@
+test/**/failures/
+test_goldens/**/failures/
+
 # Miscellaneous
 *.class
 *.log

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2022 Declarative, Inc.
+Copyright (c) 2025 Declarative, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

doc/website/source/styles/docs_page_layout.scss

@@ -263,15 +263,15 @@ main.page-content {
 
   p {
     margin-bottom: 1.5em;
+  }
 
-    code {
-      padding: 3px 6px;
-      background: #7f00a6;
-      border: 1px solid #a218cc;
-      border-radius: 4px;
+  code {
+    padding: 3px 6px;
+    background: #7f00a6;
+    border: 1px solid #a218cc;
+    border-radius: 4px;
 
-      color: WHITE;
-    }
+    color: WHITE;
   }
 
   li {

golden_tester.Dockerfile

@@ -0,0 +1,24 @@
+FROM ubuntu:latest
+
+ENV FLUTTER_HOME=${HOME}/sdks/flutter 
+ENV PATH ${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin
+
+USER root
+
+RUN apt update
+
+RUN apt install -y git curl unzip
+
+# Print the Ubuntu version. Useful when there are failing tests.
+RUN cat /etc/lsb-release
+
+# Invalidate the cache when flutter pushes a new commit.
+ADD https://api.github.com/repos/flutter/flutter/git/refs/heads/master ./flutter-latest-master
+
+RUN git clone https://github.com/flutter/flutter.git ${FLUTTER_HOME}
+
+RUN flutter doctor
+
+# Copy the whole repo.
+# We need this because we use local dependencies.
+COPY ./ /golden_tester

lib/flutter_test_goldens.dart

@@ -1,7 +1,8 @@
+export 'src/flutter/flutter_camera.dart';
+export 'src/flutter/flutter_golden_matcher.dart';
 export 'src/flutter/flutter_test_extensions.dart';
 export 'src/fonts/fonts.dart';
 export 'src/fonts/icons.dart';
-export 'src/flutter/flutter_camera.dart';
 export 'src/goldens/golden_collections.dart';
 export 'src/goldens/golden_comparisons.dart';
 export 'src/goldens/golden_rendering.dart';

lib/src/flutter/flutter_golden_matcher.dart

@@ -0,0 +1,141 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:path/path.dart';
+
+/// A matcher that expects given content to match the golden file referenced
+/// by [key], allowing up to [maxPixelMismatchCount] different pixels before
+/// considering the test to be a failure.
+///
+/// Typically, the [key] is expected to be a relative file path from the given
+/// test file, to the golden file, e.g., "goldens/my-golden-name.png".
+///
+/// This matcher can be used by calling it in `expectLater()`, e.g.,
+///
+///     await expectLater(
+///       find.byType(MaterialApp),
+///       matchesGoldenFileWithPixelAllowance("goldens/my-golden-name.png", 20),
+///     );
+///
+/// Typically, Flutter's golden system describes mismatches in terms of percentages.
+/// But percentages are difficult to depend upon. Sometimes a relatively large percentage
+/// doesn't matter, and sometimes a tiny percentage is critical. When it comes to ignoring
+/// irrelevant mismatches, it's often more convenient to work in terms of pixels. This
+/// matcher lets developers specify a maximum pixel mismatch count, instead of relying on
+/// percentage differences across the entire golden image.
+MatchesGoldenFile matchesGoldenFileWithPixelAllowance(Object key, int maxPixelMismatchCount, {int? version}) {
+  if (key is Uri) {
+    return MatchesGoldenFileWithPixelAllowance(key, maxPixelMismatchCount, version);
+  } else if (key is String) {
+    return MatchesGoldenFileWithPixelAllowance.forStringPath(key, maxPixelMismatchCount, version);
+  }
+  throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}');
+}
+
+/// A special version of [MatchesGoldenFile] that allows a specified number of
+/// pixels to be different between golden files before considering the test to
+/// be a failure.
+///
+/// Typically, this matcher is expected to be created by calling
+/// [matchesGoldenFileWithPixelAllowance].
+class MatchesGoldenFileWithPixelAllowance extends MatchesGoldenFile {
+  /// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
+  /// file at the relative path within the [key] URI.
+  ///
+  /// The [key] URI should be a relative path from the executing test's
+  /// directory to the golden file, e.g., "goldens/my-golden-name.png".
+  MatchesGoldenFileWithPixelAllowance(super.key, this._maxPixelMismatchCount, [super.version]);
+
+  /// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden
+  /// file at the relative [path].
+  ///
+  /// The [path] should be relative to the executing test's directory, e.g.,
+  /// "goldens/my-golden-name.png".
+  MatchesGoldenFileWithPixelAllowance.forStringPath(String path, this._maxPixelMismatchCount, [int? version])
+      : super.forStringPath(path, version);
+
+  final int _maxPixelMismatchCount;
+
+  @override
+  Future<String?> matchAsync(dynamic item) async {
+    // Cache the current goldenFileComparator so we can restore
+    // it after the test.
+    final originalComparator = goldenFileComparator;
+
+    try {
+      goldenFileComparator = PixelDiffGoldenComparator(
+        (goldenFileComparator as LocalFileComparator).basedir.path,
+        pixelCount: _maxPixelMismatchCount,
+      );
+
+      return await super.matchAsync(item);
+    } finally {
+      goldenFileComparator = originalComparator;
+    }
+  }
+}
+
+/// A golden file comparator that allows a specified number of pixels
+/// to be different between the golden image file and the test image file, and
+/// still pass.
+class PixelDiffGoldenComparator extends LocalFileComparator {
+  PixelDiffGoldenComparator(
+    String testBaseDirectory, {
+    required int pixelCount,
+  })  : _testBaseDirectory = testBaseDirectory,
+        _maxPixelMismatchCount = pixelCount,
+        super(Uri.parse(testBaseDirectory));
+
+  @override
+  Uri get basedir => Uri.parse(_testBaseDirectory);
+
+  /// The file system path to the directory that holds the currently executing
+  /// Dart test file.
+  final String _testBaseDirectory;
+
+  /// The maximum number of mismatched pixels for which this pixel test
+  /// is considered a success/pass.
+  final int _maxPixelMismatchCount;
+
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
+    // Note: the incoming `golden` Uri is a partial path from the currently
+    // executing test directory to the golden file, e.g., "goldens/my-test.png".
+    final result = await GoldenFileComparator.compareLists(
+      imageBytes,
+      await getGoldenBytes(golden),
+    );
+
+    if (result.passed) {
+      return true;
+    }
+
+    final diffImage = result.diffs!.entries.first.value;
+    final pixelCount = diffImage.width * diffImage.height;
+    final pixelMismatchCount = pixelCount * result.diffPercent;
+
+    if (pixelMismatchCount <= _maxPixelMismatchCount) {
+      return true;
+    }
+
+    // Paint the golden diffs and images to failure files.
+    await generateFailureOutput(result, golden, basedir);
+    throw FlutterError(
+        "Pixel test failed. ${result.diffPercent.toStringAsFixed(2)}% diff, $pixelMismatchCount pixel count diff (max allowed pixel mismatch count is $_maxPixelMismatchCount)");
+  }
+
+  @override
+  @protected
+  Future<List<int>> getGoldenBytes(Uri golden) async {
+    final File goldenFile = _getGoldenFile(golden);
+    if (!goldenFile.existsSync()) {
+      fail('Could not be compared against non-existent file: "$golden"');
+    }
+    final List<int> goldenBytes = await goldenFile.readAsBytes();
+    return goldenBytes;
+  }
+
+  File _getGoldenFile(Uri golden) => File(join(_testBaseDirectory, fromUri(golden.path)));
+}

lib/src/goldens/golden_comparisons.dart

@@ -4,8 +4,9 @@ import 'package:flutter_test_goldens/src/logging.dart';
 /// Compares new [screenshots] to existing [goldens] and reports any mismatches between them.
 GoldenCollectionMismatches compareGoldenCollections(
   ScreenshotCollection goldens,
-  ScreenshotCollection screenshots,
-) {
+  ScreenshotCollection screenshots, {
+  Map<String, int> tolerances = const {},
+}) {
   final mismatches = <String, GoldenMismatch>{};
 
   // For every golden, look for missing and mismatching screenshots.
@@ -30,7 +31,8 @@ GoldenCollectionMismatches compareGoldenCollections(
 
     // The golden and screenshot have the same size. Look for a pixel mismatch.
     final mismatchPixelCount = _calculatePixelMismatch(golden, screenshot);
-    if (mismatchPixelCount > 0) {
+    final tolerance = tolerances[golden.id] ?? 0;
+    if (mismatchPixelCount > tolerance) {
       mismatches[id] = PixelGoldenMismatch(
         golden: golden,
         screenshot: screenshot,