Scaffolding for NativeDriver and AndroidNativeDriver for taking screenshots using adb. (flutter#152194)

Closes flutter#152189.

I have next to no clue how to configure this to run on CI, so bear with me as I rediscover the wheel.

Author: matanlurey

.ci.yaml

@@ -1232,6 +1232,15 @@ targets:
       - bin/**
       - .ci.yaml
 
+  - name: Linux_android_emu flutter_driver_android_test
+    recipe: flutter/flutter_drone
+    timeout: 60
+    bringup: true
+    properties:
+      shard: flutter_driver_android
+      tags: >
+         ["framework", "hostonly", "shard", "linux"]
+
   - name: Linux realm_checker
     recipe: flutter/flutter_drone
     timeout: 60

TESTOWNERS

@@ -324,6 +324,7 @@
 # coverage @goderbauer @flutter/infra
 # customer_testing @Piinks @flutter/framework
 # docs @Piinks @flutter/framework
+# flutter_driver_android_test @matanlurey @johnmccutchan
 # flutter_packaging @christopherfujino @flutter/infra
 # flutter_plugins @stuartmorgan @flutter/plugin
 # framework_tests @Piinks @flutter/framework

dev/bots/suite_runners/run_flutter_driver_android_tests.dart

@@ -0,0 +1,17 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:path/path.dart' as path;
+import '../utils.dart';
+
+Future<void> runFlutterDriverAndroidTests() async {
+  print('Running Flutter Driver Android tests...');
+
+  await runDartTest(
+    path.join(flutterRoot, 'packages', 'flutter_driver'),
+    testPaths: <String>[
+      'test/src/native_tests/android',
+    ],
+  );
+}

dev/bots/test.dart

@@ -63,6 +63,7 @@ import 'suite_runners/run_android_java11_integration_tool_tests.dart';
 import 'suite_runners/run_android_preview_integration_tool_tests.dart';
 import 'suite_runners/run_customer_testing_tests.dart';
 import 'suite_runners/run_docs_tests.dart';
+import 'suite_runners/run_flutter_driver_android_tests.dart';
 import 'suite_runners/run_flutter_packages_tests.dart';
 import 'suite_runners/run_framework_coverage_tests.dart';
 import 'suite_runners/run_framework_tests.dart';
@@ -142,6 +143,7 @@ Future<void> main(List<String> args) async {
       'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests,
       // All web integration tests
       'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner,
+      'flutter_driver_android': runFlutterDriverAndroidTests,
       'flutter_plugins': flutterPackagesRunner,
       'skp_generator': skpGeneratorTestsRunner,
       'realm_checker': realmCheckerTestRunner,

packages/flutter_driver/lib/src/native/README.md

@@ -0,0 +1,24 @@
+# Flutter Native Driver
+
+An experiment in adding platform-aware functionality to `flutter_driver`.
+
+Project tracking: <https://github.com/orgs/flutter/projects/154>.
+
+We'd like to be able to test, within `flutter/flutter` (and friends):
+
+- Does a web-view load and render the expected content?
+- Unexpected changes with the native OS, i.e. Android edge-to-edge
+- Impeller rendering on Android using a real GPU (not swift_shader or Skia)
+- Does an app correctly respond to application backgrounding and resume?
+- Interact with native UI elements (not rendered by Flutter) and observe output
+- Native text/keyboard input (IMEs, virtual keyboards, anything a11y related)
+
+This project is tracking augmenting `flutter_driver` towards these goals.
+
+If the project is not successful, the experiment will be turned-down and the
+code removed or repurposed.
+
+---
+
+_Questions?_ Ask in the `#hackers-tests` channel on the Flutter Discord or
+`@matanlurey` or `@johnmccutchan` on GitHub.

packages/flutter_driver/lib/src/native/android.dart

@@ -0,0 +1,185 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Examples can assume:
+// import 'package:flutter_driver/src/native/android.dart';
+
+import 'dart:io' as io;
+import 'dart:typed_data';
+
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as p;
+
+import 'driver.dart';
+
+/// Drives an Android device or emulator that is running a Flutter application.
+final class AndroidNativeDriver implements NativeDriver {
+  /// Creates a new Android native driver with the provided configuration.
+  ///
+  /// The [tempDirectory] argument can be used to specify a custom directory
+  /// where the driver will store temporary files. If not provided, a temporary
+  /// directory will be created in the system's temporary directory.
+  @visibleForTesting
+  AndroidNativeDriver({
+    required AndroidDeviceTarget target,
+    String? adbPath,
+    io.Directory? tempDirectory,
+  }) : _adbPath = adbPath ?? 'adb',
+       _target = target,
+       _tmpDir = tempDirectory ?? io.Directory.systemTemp.createTempSync('flutter_driver.');
+
+  /// Connects to a device or emulator identified by [target].
+  static Future<AndroidNativeDriver> connect({
+    AndroidDeviceTarget target = const AndroidDeviceTarget.onlyEmulatorOrDevice(),
+  }) async {
+    final AndroidNativeDriver driver = AndroidNativeDriver(target: target);
+    await driver._smokeTest();
+    return driver;
+  }
+
+  Future<void> _smokeTest() async {
+    final io.ProcessResult version = await io.Process.run(
+      _adbPath,
+      const <String>['version'],
+    );
+    if (version.exitCode != 0) {
+      throw StateError('Failed to run `$_adbPath version`: ${version.stderr}');
+    }
+
+    final io.ProcessResult devices = await io.Process.run(
+      _adbPath,
+      <String>[
+        ..._target._toAdbArgs(),
+        'shell',
+        'echo',
+        'connected',
+      ],
+    );
+    if (devices.exitCode != 0) {
+      throw StateError('Failed to connect to target: ${devices.stderr}');
+    }
+  }
+
+  final String _adbPath;
+  final AndroidDeviceTarget _target;
+  final io.Directory _tmpDir;
+
+  @override
+  Future<void> close() async {
+    await _tmpDir.delete(recursive: true);
+  }
+
+  @override
+  Future<NativeScreenshot> screenshot() async {
+    final io.ProcessResult result = await io.Process.run(
+      _adbPath,
+      <String>[
+        ..._target._toAdbArgs(),
+        'exec-out',
+        'screencap',
+        '-p',
+      ],
+      stdoutEncoding: null,
+    );
+
+    if (result.exitCode != 0) {
+      throw StateError('Failed to take screenshot: ${result.stderr}');
+    }
+
+    final Uint8List bytes = result.stdout as Uint8List;
+    return _AdbScreencap(bytes, _tmpDir);
+  }
+}
+
+final class _AdbScreencap implements NativeScreenshot {
+  const _AdbScreencap(this._bytes, this._tmpDir);
+
+  /// Raw bytes of the screenshot in PNG format.
+  final Uint8List _bytes;
+
+  /// Temporary directory to default to when saving the screenshot.
+  final io.Directory _tmpDir;
+
+  static int _lastScreenshotId = 0;
+
+  @override
+  Future<String> saveAs([String? path]) async {
+    final int id = _lastScreenshotId++;
+    path ??= p.join(_tmpDir.path, '$id.png');
+    await io.File(path).writeAsBytes(_bytes);
+    return path;
+  }
+
+  @override
+  Future<Uint8List> readAsBytes() async => _bytes;
+}
+
+/// Represents a target device running Android.
+sealed class AndroidDeviceTarget {
+  /// Represents a device with the given [serialNumber].
+  ///
+  /// This is the recommended way to target a specific device, and uses the
+  /// device's serial number, as reported by `adb devices`, to identify the
+  /// device:
+  ///
+  /// ```sh
+  /// $ adb devices
+  /// List of devices attached
+  /// emulator-5554   device
+  /// ```
+  ///
+  /// In this example, the serial number is `emulator-5554`:
+  ///
+  /// ```dart
+  /// const AndroidDeviceTarget target = AndroidDeviceTarget.bySerial('emulator-5554');
+  /// ```
+  const factory AndroidDeviceTarget.bySerial(String serialNumber) = _SerialDeviceTarget;
+
+  /// Represents the only running emulator _or_ connected device.
+  ///
+  /// This is equivalent to using `adb` without `-e`, `-d`, or `-s`.
+  const factory AndroidDeviceTarget.onlyEmulatorOrDevice() = _SingleAnyTarget;
+
+  /// Represents the only running emulator on the host machine.
+  ///
+  /// This is equivalent to using `adb -e`, a _single_ emulator must be running.
+  const factory AndroidDeviceTarget.onlyRunningEmulator() = _SingleEmulatorTarget;
+
+  /// Represents the only connected device on the host machine.
+  ///
+  /// This is equivalent to using `adb -d`, a _single_ device must be connected.
+  const factory AndroidDeviceTarget.onlyConnectedDevice() = _SingleDeviceTarget;
+
+  /// Returns the arguments to pass to `adb` to target this device.
+  List<String> _toAdbArgs();
+}
+
+final class _SerialDeviceTarget implements AndroidDeviceTarget {
+  const _SerialDeviceTarget(this.serialNumber);
+  final String serialNumber;
+
+  @override
+  List<String> _toAdbArgs() => <String>['-s', serialNumber];
+}
+
+final class _SingleEmulatorTarget implements AndroidDeviceTarget {
+  const _SingleEmulatorTarget();
+
+  @override
+  List<String> _toAdbArgs() => const <String>['-e'];
+}
+
+final class _SingleDeviceTarget implements AndroidDeviceTarget {
+  const _SingleDeviceTarget();
+
+  @override
+  List<String> _toAdbArgs() => const <String>['-d'];
+}
+
+final class _SingleAnyTarget implements AndroidDeviceTarget {
+  const _SingleAnyTarget();
+
+  @override
+  List<String> _toAdbArgs() => const <String>[];
+}

packages/flutter_driver/lib/src/native/driver.dart

@@ -0,0 +1,54 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// @docImport 'package:flutter_driver/flutter_driver.dart';
+library;
+
+import 'dart:typed_data';
+
+/// Drives a native device or emulator that is running a Flutter application.
+///
+/// Unlike [FlutterDriver], a [NativeDriver] is backed by a platform specific
+/// implementation that might interact with out-of-process services, such as
+/// `adb` for Android or `ios-deploy` for iOS, and might require additional
+/// setup (e.g., adding test-only plugins to the application under test) for
+/// full functionality.
+///
+/// API that is available directly on [NativeDriver] is considered _lowest
+/// common denominator_ and is guaranteed to work on all platforms supported by
+/// Flutter Driver unless otherwise noted. Platform-specific functionality that
+/// _cannot_ be exposed through this interface is available through
+/// platform-specific extensions.
+abstract interface class NativeDriver {
+  /// Closes the native driver and releases any resources associated with it.
+  ///
+  /// After calling this method, the driver is no longer usable.
+  Future<void> close();
+
+  /// Take a screenshot using a platform-specific mechanism.
+  ///
+  /// The image is returned as an opaque handle that can be used to retrieve
+  /// the screenshot data or to compare it with another screenshot, and may
+  /// include platform-specific system UI elements, such as the status bar or
+  /// navigation bar.
+  Future<NativeScreenshot> screenshot();
+}
+
+/// An opaque handle to a screenshot taken on a native device.
+///
+/// Unlike [FlutterDriver.screenshot], the screenshot represented by this handle
+/// is generated by a platform-specific mechanism and is often already stored
+/// on disk. The handle can be used to retrieve the screenshot data or to
+/// compare it with another screenshot.
+abstract interface class NativeScreenshot {
+  /// Saves the screenshot to a file at the specified [path].
+  ///
+  /// If [path] is not provided, a temporary file will be created.
+  ///
+  /// Returns the path to the saved file.
+  Future<String> saveAs([String? path]);
+
+  /// Reads the screenshot as a PNG-formatted list of bytes.
+  Future<Uint8List> readAsBytes();
+}

packages/flutter_driver/test/src/native_tests/android/README.md

@@ -0,0 +1,15 @@
+# `AndroidNativeDriver` Tests
+
+This directory are tests that require an Android device or emulator to run.
+
+To run locally, connect an Android device or start an emulator and run:
+
+```bash
+# Assumuing your current working directory is `packages/flutter_driver`.\
+
+$ flutter test test/src/native_tests/android
+```
+
+On CI, these tests are run via [`run_flutter_driver_android_tests.dart`][ci].
+
+[ci]: ../../../../../../dev/bots/suite_runners/run_flutter_driver_android_tests.dart

packages/flutter_driver/test/src/native_tests/android/screenshot_test.dart

@@ -0,0 +1,25 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+import 'dart:typed_data';
+
+import 'package:flutter_driver/src/native/android.dart';
+import 'package:flutter_driver/src/native/driver.dart';
+import 'package:test/test.dart';
+
+void main() async {
+  test('should connect to an Android device and take a screenshot', () async {
+    final NativeDriver driver = await AndroidNativeDriver.connect();
+    final NativeScreenshot screenshot = await driver.screenshot();
+
+    final Uint8List bytes = await screenshot.readAsBytes();
+    expect(bytes.length, greaterThan(0));
+
+    final String path = await screenshot.saveAs();
+    expect(io.File(path).readAsBytesSync(), bytes);
+
+    await driver.close();
+  });
+}