[camera_android_camerax] Support NV21 (#9853)

Implements NV21 support for image streaming.

Also bumps the CameraX version to `1.5.0-rc01`! To do this, I needed to (1) bump the minimum SDK version to 23 and (2) modify a couple of unrelated tests.

Fixes flutter/flutter#145961.

## Pre-Review Checklist

**Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed.

[^1]: Regular contributors who have demonstrated familiarity with the repository guidelines only need to comment if the PR is not auto-exempted by repo tooling.

Author: camsim99

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.21
+
+* Implements NV21 support for image streaming.
+
 ## 0.6.20+3
 
 * Bumps com.google.guava:guava from 33.4.0-android to 33.4.8-android.

packages/camera/camera_android_camerax/README.md

@@ -73,7 +73,9 @@ in the merged Android manifest of your app, then take the following steps to rem
     tools:node="remove" />
 ```
 
-### Allowing image streaming in the background
+### Notes on image streaming
+
+#### Allowing image streaming in the background
 
 As of Android 14, to allow for background image streaming, you will need to specify the foreground
 [`TYPE_CAMERA`][12] foreground service permission in your app's manifest. Specifically, in
@@ -86,6 +88,12 @@ As of Android 14, to allow for background image streaming, you will need to spec
 </manifest>
 ```
 
+#### Configuring NV21 image format
+
+If you initialize a `CameraController` with `ImageFormatGroup.nv21`, then streamed images will
+still have the `ImageFormatGroup.yuv420` format, but their image data will be formatted in NV21.
+See https://developer.android.com/reference/kotlin/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21().
+
 ## Contributing
 
 For more information on contributing to this plugin, see [`CONTRIBUTING.md`](CONTRIBUTING.md).

packages/camera/camera_android_camerax/android/build.gradle

@@ -40,8 +40,8 @@ android {
     }
 
     defaultConfig {
-        // Many of the CameraX APIs require API 21.
-        minSdkVersion 21
+        // CameraX APIs require API 23 or later.
+        minSdk = 23
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
@@ -72,7 +72,7 @@ android {
 
 dependencies {
     // CameraX core library using the camera2 implementation must use same version number.
-    def camerax_version = "1.5.0-beta01"
+    def camerax_version = "1.5.0-rc01"
     implementation "androidx.camera:camera-core:${camerax_version}"
     implementation "androidx.camera:camera-camera2:${camerax_version}"
     implementation "androidx.camera:camera-lifecycle:${camerax_version}"

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt

@@ -1,7 +1,7 @@
 // Copyright 2013 The Flutter Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Autogenerated from Pigeon (v25.3.2), do not edit directly.
+// Autogenerated from Pigeon (v25.5.0), do not edit directly.
 // See also: https://pub.dev/packages/pigeon
 @file:Suppress("UNCHECKED_CAST", "ArrayInDataClass")
 
@@ -4761,7 +4761,8 @@ abstract class PigeonApiImageAnalysis(
 ) {
   abstract fun pigeon_defaultConstructor(
       resolutionSelector: androidx.camera.core.resolutionselector.ResolutionSelector?,
-      targetRotation: Long?
+      targetRotation: Long?,
+      outputImageFormat: Long?
   ): androidx.camera.core.ImageAnalysis
 
   abstract fun resolutionSelector(
@@ -4800,10 +4801,12 @@ abstract class PigeonApiImageAnalysis(
             val resolutionSelectorArg =
                 args[1] as androidx.camera.core.resolutionselector.ResolutionSelector?
             val targetRotationArg = args[2] as Long?
+            val outputImageFormatArg = args[3] as Long?
             val wrapped: List<Any?> =
                 try {
                   api.pigeonRegistrar.instanceManager.addDartCreatedInstance(
-                      api.pigeon_defaultConstructor(resolutionSelectorArg, targetRotationArg),
+                      api.pigeon_defaultConstructor(
+                          resolutionSelectorArg, targetRotationArg, outputImageFormatArg),
                       pigeon_identifierArg)
                   listOf(null)
                 } catch (exception: Throwable) {

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageAnalysisProxyApi.java

@@ -21,14 +21,21 @@ class ImageAnalysisProxyApi extends PigeonApiImageAnalysis {
   @NonNull
   @Override
   public ImageAnalysis pigeon_defaultConstructor(
-      @Nullable ResolutionSelector resolutionSelector, @Nullable Long targetRotation) {
+      @Nullable ResolutionSelector resolutionSelector,
+      @Nullable Long targetRotation,
+      @Nullable Long outputImageFormat) {
     final ImageAnalysis.Builder builder = new ImageAnalysis.Builder();
     if (resolutionSelector != null) {
       builder.setResolutionSelector(resolutionSelector);
     }
     if (targetRotation != null) {
       builder.setTargetRotation(targetRotation.intValue());
     }
+
+    if (outputImageFormat != null) {
+      builder.setOutputImageFormat(outputImageFormat.intValue());
+    }
+
     return builder.build();
   }
 

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/Camera2CameraInfoTest.java

@@ -69,7 +69,7 @@ public void getCameraCharacteristic_returnsCorrespondingValueOfKeyWhenKeyNotReco
     assertEquals(value, api.getCameraCharacteristic(instance, key));
   }
 
-  @Config(minSdk = 21)
+  @Config(minSdk = 23)
   @SuppressWarnings("unchecked")
   @Test
   public void getCameraCharacteristic_returnsExpectedCameraHardwareLevelWhenRequested() {

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageAnalysisTest.java

@@ -32,11 +32,13 @@ public void pigeon_defaultConstructor_createsExpectedImageAnalysisInstance() {
 
     final ResolutionSelector mockResolutionSelector = new ResolutionSelector.Builder().build();
     final long targetResolution = Surface.ROTATION_0;
+    final long outputImageFormat = ImageAnalysis.OUTPUT_IMAGE_FORMAT_NV21;
     final ImageAnalysis imageAnalysis =
-        api.pigeon_defaultConstructor(mockResolutionSelector, targetResolution);
+        api.pigeon_defaultConstructor(mockResolutionSelector, targetResolution, outputImageFormat);
 
     assertEquals(imageAnalysis.getResolutionSelector(), mockResolutionSelector);
     assertEquals(imageAnalysis.getTargetRotation(), Surface.ROTATION_0);
+    assertEquals(imageAnalysis.getOutputImageFormat(), ImageAnalysis.OUTPUT_IMAGE_FORMAT_NV21);
   }
 
   @Test

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/PendingRecordingTest.java

@@ -98,7 +98,7 @@ public void start_callsStartOnInstance() {
           .when(() -> ContextCompat.getMainExecutor(any()))
           .thenAnswer((Answer<Executor>) invocation -> mock(Executor.class));
 
-      when(instance.start(any(), any())).thenReturn(value);
+      when(instance.start(any(Executor.class), any())).thenReturn(value);
 
       assertEquals(value, api.start(instance, listener));
     }

packages/camera/camera_android_camerax/example/lib/camera_image.dart

@@ -83,6 +83,9 @@ ImageFormatGroup _asImageFormatGroup(dynamic rawFormat) {
       // android.graphics.ImageFormat.YUV_420_888
       case 35:
         return ImageFormatGroup.yuv420;
+      // android.graphics.ImageFormat.NV21
+      case 17:
+        return ImageFormatGroup.nv21;
       // android.graphics.ImageFormat.JPEG
       case 256:
         return ImageFormatGroup.jpeg;

packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart

@@ -174,15 +174,30 @@ class AndroidCameraCameraX extends CameraPlatform {
   @visibleForTesting
   StreamController<CameraImageData>? cameraImageDataStreamController;
 
-  /// Constant representing the multi-plane Android YUV 420 image format.
+  /// Constant representing the multi-plane Android YUV 420 image format used by ImageProxy.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888.
-  static const int imageFormatYuv420_888 = 35;
+  static const int imageProxyFormatYuv420_888 = 35;
 
-  /// Constant representing the compressed JPEG image format.
+  /// Constant representing the NV21 image format used by ImageProxy.
+  ///
+  /// See https://developer.android.com/reference/android/graphics/ImageFormat#NV21.
+  static const int imageProxyFormatNv21 = 17;
+
+  /// Constant representing the compressed JPEG image format used by ImageProxy.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat#JPEG.
-  static const int imageFormatJpeg = 256;
+  static const int imageProxyFormatJpeg = 256;
+
+  /// Constant representing the YUV 420 image format used for configuring ImageAnalysis.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_YUV_420_888()
+  static const int imageAnalysisOutputImageFormatYuv420_888 = 1;
+
+  /// Constant representing the NV21 image format used for configuring ImageAnalysis.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21().
+  static const int imageAnalysisOutputImageFormatNv21 = 3;
 
   /// Error code indicating a [ZoomState] was requested, but one has not been
   /// set for the camera in use.
@@ -269,6 +284,12 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// A map to associate a [CameraInfo] with its camera name.
   final Map<String, CameraInfo> _savedCameras = <String, CameraInfo>{};
 
+  /// The preset resolution selector for the camera.
+  ResolutionSelector? _presetResolutionSelector;
+
+  /// The ID of the surface texture that the camera preview is drawn to.
+  late int _flutterSurfaceTextureId;
+
   /// Returns list of all available cameras and their descriptions.
   @override
   Future<List<CameraDescription>> availableCameras() async {
@@ -380,8 +401,9 @@ class AndroidCameraCameraX extends CameraPlatform {
     );
     // Determine ResolutionSelector and QualitySelector based on
     // resolutionPreset for camera UseCases.
-    final ResolutionSelector? presetResolutionSelector =
-        _getResolutionSelectorFromPreset(mediaSettings?.resolutionPreset);
+    _presetResolutionSelector = _getResolutionSelectorFromPreset(
+      mediaSettings?.resolutionPreset,
+    );
     final QualitySelector? presetQualitySelector =
         _getQualitySelectorFromPreset(mediaSettings?.resolutionPreset);
 
@@ -391,55 +413,26 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Configure Preview instance.
     preview = proxy.newPreview(
-      resolutionSelector: presetResolutionSelector,
+      resolutionSelector: _presetResolutionSelector,
       /* use CameraX default target rotation */ targetRotation: null,
     );
-    final int flutterSurfaceTextureId = await preview!.setSurfaceProvider(
+    _flutterSurfaceTextureId = await preview!.setSurfaceProvider(
       systemServicesManager,
     );
 
     // Configure ImageCapture instance.
     imageCapture = proxy.newImageCapture(
-      resolutionSelector: presetResolutionSelector,
+      resolutionSelector: _presetResolutionSelector,
       /* use CameraX default target rotation */ targetRotation:
           await deviceOrientationManager.getDefaultDisplayRotation(),
     );
 
-    // Configure ImageAnalysis instance.
-    // Defaults to YUV_420_888 image format.
-    imageAnalysis = proxy.newImageAnalysis(
-      resolutionSelector: presetResolutionSelector,
-      /* use CameraX default target rotation */ targetRotation: null,
-    );
-
     // Configure VideoCapture and Recorder instances.
     recorder = proxy.newRecorder(qualitySelector: presetQualitySelector);
     videoCapture = proxy.withOutputVideoCapture(videoOutput: recorder!);
 
-    // Bind configured UseCases to ProcessCameraProvider instance & mark Preview
-    // instance as bound but not paused. Video capture is bound at first use
-    // instead of here.
-    camera = await processCameraProvider!.bindToLifecycle(
-      cameraSelector!,
-      <UseCase>[preview!, imageCapture!, imageAnalysis!],
-    );
-    await _updateCameraInfoAndLiveCameraState(flutterSurfaceTextureId);
-    previewInitiallyBound = true;
-    _previewIsPaused = false;
-
     // Retrieve info required for correcting the rotation of the camera preview
     // if necessary.
-
-    final Camera2CameraInfo camera2CameraInfo = proxy.fromCamera2CameraInfo(
-      cameraInfo: cameraInfo!,
-    );
-    sensorOrientationDegrees =
-        ((await camera2CameraInfo.getCameraCharacteristic(
-                  proxy.sensorOrientationCameraCharacteristics(),
-                ))!
-                as int)
-            .toDouble();
-
     sensorOrientationDegrees = cameraDescription.sensorOrientation.toDouble();
     _handlesCropAndRotation = await preview!
         .surfaceProducerHandlesCropAndRotation();
@@ -449,35 +442,59 @@ class AndroidCameraCameraX extends CameraPlatform {
     _initialDefaultDisplayRotation = await deviceOrientationManager
         .getDefaultDisplayRotation();
 
-    return flutterSurfaceTextureId;
+    return _flutterSurfaceTextureId;
   }
 
   /// Initializes the camera on the device.
   ///
-  /// Since initialization of a camera does not directly map as an operation to
-  /// the CameraX library, this method just retrieves information about the
-  /// camera and sends a [CameraInitializedEvent].
+  /// Specifically, this method:
+  ///  * Configures the [ImageAnalysis] instance according to the specified
+  ///   [imageFormatGroup]
+  ///  * Binds the configured [Preview], [ImageCapture], and [ImageAnalysis]
+  ///    instances to the [ProcessCameraProvider] instance.
+  ///  * Retrieves information about the camera and sends a [CameraInitializedEvent].
   ///
   /// [imageFormatGroup] is used to specify the image format used for image
-  /// streaming, but CameraX currently only supports YUV_420_888 (supported by
-  /// Flutter) and RGBA (not supported by Flutter). CameraX uses YUV_420_888
-  /// by default, so [imageFormatGroup] is not used.
+  /// streaming, but CameraX currently only supports YUV_420_888 (the CameraX default),
+  /// NV21, and RGBA (not supported by Flutter).
   @override
   Future<void> initializeCamera(
     int cameraId, {
     ImageFormatGroup imageFormatGroup = ImageFormatGroup.unknown,
   }) async {
-    // Configure CameraInitializedEvent to send as representation of a
-    // configured camera:
-    // Retrieve preview resolution.
+    // If preview has not been created, then no camera has been created, which signals that
+    // createCamera was not called before initializeCamera.
     if (preview == null) {
-      // No camera has been created; createCamera must be called before initializeCamera.
       throw CameraException(
         'cameraNotFound',
         "Camera not found. Please call the 'create' method before calling 'initialize'",
       );
     }
+    // Configure ImageAnalysis instance.
+    // Defaults to YUV_420_888 image format.
+    imageAnalysis = proxy.newImageAnalysis(
+      resolutionSelector: _presetResolutionSelector,
+      outputImageFormat: _imageAnalysisOutputFormatFromImageFormatGroup(
+        imageFormatGroup,
+      ),
+      /* use CameraX default target rotation */ targetRotation: null,
+    );
+
+    // Bind configured UseCases to ProcessCameraProvider instance & mark Preview
+    // instance as bound but not paused. Video capture is bound at first use
+    // instead of here.
+    camera = await processCameraProvider!.bindToLifecycle(
+      cameraSelector!,
+      <UseCase>[preview!, imageCapture!, imageAnalysis!],
+    );
+    await _updateCameraInfoAndLiveCameraState(_flutterSurfaceTextureId);
+    previewInitiallyBound = true;
+    _previewIsPaused = false;
+
+    // Configure CameraInitializedEvent to send as representation of a
+    // configured camera:
 
+    // Retrieve preview resolution.
     final ResolutionInfo previewResolutionInfo = (await preview!
         .getResolutionInfo())!;
 
@@ -1214,6 +1231,10 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// implementation using a broadcast [StreamController], which does not
   /// support those operations.
   ///
+  /// If the camera was initialized with [ImageFormatGroup.nv21], then the
+  /// streamed images will still have format [ImageFormatGroup.yuv420], but
+  /// their image data will be formatted in NV21.
+  ///
   /// [cameraId] and [options] are not used.
   @override
   Stream<CameraImageData> onStreamedFrameAvailable(
@@ -1326,14 +1347,30 @@ class AndroidCameraCameraX extends CameraPlatform {
     await imageAnalysis!.clearAnalyzer();
   }
 
-  /// Converts between Android ImageFormat constants and [ImageFormatGroup]s.
+  /// Converts [ImageFormatGroup]s to Android ImageAnalysis output format constants.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis.
+  int? _imageAnalysisOutputFormatFromImageFormatGroup(dynamic format) {
+    switch (format) {
+      case ImageFormatGroup.yuv420:
+        return imageAnalysisOutputImageFormatYuv420_888;
+      case ImageFormatGroup.nv21:
+        return imageAnalysisOutputImageFormatNv21;
+    }
+
+    return null;
+  }
+
+  /// Converts from Android ImageFormat constants to [ImageFormatGroup]s.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat.
   ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) {
     switch (data) {
-      case imageFormatYuv420_888: // android.graphics.ImageFormat.YUV_420_888
+      case imageProxyFormatYuv420_888: // android.graphics.ImageFormat.YUV_420_888
         return ImageFormatGroup.yuv420;
-      case imageFormatJpeg: // android.graphics.ImageFormat.JPEG
+      case imageProxyFormatNv21: // android.graphics.ImageFormat.NV21
+        return ImageFormatGroup.nv21;
+      case imageProxyFormatJpeg: // android.graphics.ImageFormat.JPEG
         return ImageFormatGroup.jpeg;
     }