Try golden-testing on a Mokey (bringup: true), retry on an emulator (flutter#163029)

Towards flutter#162362 (see if retries
help VD on an emulator).
Towards flutter#163025 (see if we can
golden-test on a device).

May the odds be in our favor.

Author: matanlurey

.ci.yaml

@@ -1548,6 +1548,20 @@ targets:
           {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"}
         ]
 
+  - name: Linux_mokey android_engine_vulkan_tests
+    recipe: flutter/flutter_drone
+    # TODO(matanlurey): https://github.com/flutter/flutter/issues/163025.
+    bringup: true
+    timeout: 60
+    properties:
+      shard: android_engine_vulkan_tests
+      tags: >
+         ["framework", "hostonly", "shard", "linux"]
+      dependencies: >-
+        [
+          {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"}
+        ]
+
   - name: Linux_android_emu android_engine_opengles_tests
     recipe: flutter/flutter_drone
     timeout: 60

dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt

@@ -49,6 +49,14 @@ class NativeDriverSupportPlugin :
                 val versionMap = mapOf("version" to Build.VERSION.SDK_INT)
                 result.success(versionMap)
             }
+            "is_emulator" -> {
+                val isEmulator =
+                    when {
+                        Build.MODEL.contains("gphone") -> true
+                        else -> false
+                    }
+                result.success(mapOf("emulator" to isEmulator))
+            }
             "ping" -> {
                 result.success(null)
             }

dev/integration_tests/android_engine_test/test_driver/_unstable_gold_retry.dart

@@ -0,0 +1,56 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// @docImport 'package:android_driver_extensions/skia_gold.dart';
+library;
+
+// Similar to `flutter_test`, we ignore the implementation import.
+// ignore: implementation_imports
+import 'package:android_driver_extensions/native_driver.dart';
+import 'package:matcher/src/expect/async_matcher.dart';
+import 'package:matcher/src/interfaces.dart';
+
+/// Invokes [matchesGoldenFile] with optional [retries] if a comparison fails.
+AsyncMatcher matchesGoldenFileWithRetries(Object key, {int? version, int retries = 2}) {
+  final AsyncMatcher delegate = matchesGoldenFile(key, version: version);
+  if (retries == 0) {
+    return delegate;
+  }
+  return _AsyncMatcherWithRetries(delegate, retries: retries);
+}
+
+final class _AsyncMatcherWithRetries extends AsyncMatcher {
+  _AsyncMatcherWithRetries(this._delegate, {required int retries}) : _retries = retries {
+    if (retries < 1) {
+      throw RangeError.value(retries, 'retries', 'Must be at least 1');
+    }
+  }
+
+  final AsyncMatcher _delegate;
+  int _retries;
+
+  @override
+  Description describe(Description description) {
+    description = _delegate.describe(description);
+    description.add('Retries remaining: $_retries');
+    return description;
+  }
+
+  @override
+  Future<String?> matchAsync(Object? item) async {
+    while (true) {
+      final Object? error = await _delegate.matchAsync(item);
+      if (error == null) {
+        return null;
+      }
+      print('Failed: $error');
+      if (--_retries == 0) {
+        return 'Retries exceeded. Giving up.';
+      } else {
+        print('Retrying... $_retries retries left.');
+      }
+      assert(_retries >= 0, 'Unreachable');
+    }
+  }
+}

dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart

@@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
 import '../_luci_skia_gold_prelude.dart';
+import '../_unstable_gold_retry.dart';
 
 /// For local debugging, a (local) golden-file is required as a baseline:
 ///
@@ -49,22 +50,10 @@ void main() async {
     // See:
     // - Vulkan: https://github.com/flutter/flutter/issues/162362
     // - OpenGLES: https://github.com/flutter/flutter/issues/162363
-    int retriesLeft = 2;
-    do {
-      try {
-        await expectLater(
-          nativeDriver.screenshot(),
-          matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'),
-        );
-        break;
-      } on TestFailure catch (e) {
-        if (retriesLeft == 0) {
-          rethrow;
-        }
-        print('Caught: $e. Retrying...');
-        retriesLeft--;
-      }
-    } while (retriesLeft > 0);
+    await expectLater(
+      nativeDriver.screenshot(),
+      matchesGoldenFileWithRetries('$goldenPrefix.blue_orange_gradient_portrait.png'),
+    );
   }, timeout: Timeout.none);
 
   test('should rotate landscape and screenshot the gradient', () async {

dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart

@@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
 import '../_luci_skia_gold_prelude.dart';
+import '../_unstable_gold_retry.dart';
 
 /// For local debugging, a (local) golden-file is required as a baseline:
 ///
@@ -28,6 +29,9 @@ void main() async {
   late final FlutterDriver flutterDriver;
   late final NativeDriver nativeDriver;
 
+  late final bool isEmulator;
+  late final bool isVulkan;
+
   setUpAll(() async {
     if (isLuci) {
       await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant');
@@ -42,6 +46,13 @@ void main() async {
     if (await nativeDriver.sdkVersion case final int version when version < 23) {
       fail('Requires SDK >= 23, got $version');
     }
+
+    // TODO(matanlurey): https://github.com/flutter/flutter/issues/162362#issuecomment-2649555821.
+    isEmulator = await nativeDriver.isEmulator;
+    isVulkan = goldenVariant.contains('vulkan');
+    if (isEmulator && isVulkan) {
+      print('Detected running on a vulkan emulator. Will retry certain failures');
+    }
   });
 
   tearDownAll(() async {
@@ -60,13 +71,19 @@ void main() async {
     await nativeDriver.rotateToLandscape();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'),
+      matchesGoldenFileWithRetries(
+        '$goldenPrefix.blue_orange_gradient_landscape_rotated.png',
+        retries: isEmulator && isVulkan ? 2 : 0,
+      ),
     );
 
     await nativeDriver.rotateResetDefault();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'),
+      matchesGoldenFileWithRetries(
+        '$goldenPrefix.blue_orange_gradient_portait_rotated_back.png',
+        retries: isEmulator && isVulkan ? 2 : 0,
+      ),
     );
   }, timeout: Timeout.none);
 }

dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart

@@ -77,6 +77,12 @@ final class AndroidNativeDriver implements NativeDriver {
     return result['version']! as int;
   }
 
+  @override
+  Future<bool> get isEmulator async {
+    final Map<String, Object?> result = await _driver.sendCommand(NativeCommand.getIsEmulator);
+    return result['emulator']! as bool;
+  }
+
   /// Waits for 2 seconds before completing.
   ///
   /// There is no perfect way, outside of polling, to know when the device is

dev/tools/android_driver_extensions/lib/src/common.dart

@@ -30,6 +30,9 @@ final class NativeCommand extends Command {
   /// Gets the SDK version code.
   static const NativeCommand getSdkVersion = NativeCommand('sdk_version');
 
+  /// Gets whether the device is an emulator.
+  static const NativeCommand getIsEmulator = NativeCommand('is_emulator');
+
   /// The method to call on the plugin.
   final String method;
 

dev/tools/android_driver_extensions/lib/src/driver.dart

@@ -45,9 +45,12 @@ abstract interface class NativeDriver {
   /// ```
   Future<Duration> ping();
 
-  /// Returns the SDK version.
+  /// The SDK version.
   Future<int> get sdkVersion;
 
+  /// Whether the device is an emulator.
+  Future<bool> get isEmulator;
+
   /// Take a screenshot using a platform-specific mechanism.
   ///
   /// The image is returned as an opaque handle that can be used to retrieve