[camera_android_camerax] Fix NV21 Format (#10022)

When NV21 image format is requested for streaming images, this PR ensures that:

1. The NV21-compatible (but actually YUV_420_888) three planes per image are converted to a single NV21 plane (Y + interleaved VU planes).
2. The single `CameraImagePlane` created has overridden raw and camera image format NV21.*

This should make this package compatible with [google_ml_kit_flutter](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master) 🤞 I tested this change with the Barcode scanner example. Fixes flutter/flutter#174923.

*The conversion will fail if the image is actually not NV21 compatible for some reason, so this should never be a false positive.

_Note: Uses code inspired by [googlesamples/mlkit](https://github.com/googlesamples/mlkit/blob/da17257a78b9beedb57b7a9795b911296ae970a0/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java)._

## Pre-Review Checklist

**Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed.

[^1]: Regular contributors who have demonstrated familiarity with the repository guidelines only need to comment if the PR is not auto-exempted by repo tooling.

Author: camsim99

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.23
+
+* Converts NV21-compatible streamed images to NV21 when requested. In doing so,
+this plugin should now be compatible with [google_ml_kit_flutter](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master).
+
 ## 0.6.22
 
 * Implements `setDescriptionWhileRecording`.

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt

@@ -597,6 +597,12 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
    */
   abstract fun getPigeonApiImageProxy(): PigeonApiImageProxy
 
+  /**
+   * An implementation of [PigeonApiImageProxyUtils] used to add a new Dart instance of
+   * `ImageProxyUtils` to the Dart `InstanceManager`.
+   */
+  abstract fun getPigeonApiImageProxyUtils(): PigeonApiImageProxyUtils
+
   /**
    * An implementation of [PigeonApiPlaneProxy] used to add a new Dart instance of `PlaneProxy` to
    * the Dart `InstanceManager`.
@@ -738,6 +744,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
     PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, getPigeonApiAnalyzer())
     PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, getPigeonApiLiveData())
     PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxy())
+    PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxyUtils())
     PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, getPigeonApiQualitySelector())
     PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, getPigeonApiFallbackStrategy())
     PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, getPigeonApiCameraControl())
@@ -785,6 +792,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
     PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, null)
+    PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, null)
@@ -916,6 +924,8 @@ private class CameraXLibraryPigeonProxyApiBaseCodec(
       registrar.getPigeonApiLiveData().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.core.ImageProxy) {
       registrar.getPigeonApiImageProxy().pigeon_newInstance(value) {}
+    } else if (value is ImageProxyUtils) {
+      registrar.getPigeonApiImageProxyUtils().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.core.ImageProxy.PlaneProxy) {
       registrar.getPigeonApiPlaneProxy().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.video.QualitySelector) {
@@ -5372,6 +5382,83 @@ abstract class PigeonApiImageProxy(
     }
   }
 }
+/** Utils for working with [ImageProxy]s. */
+@Suppress("UNCHECKED_CAST")
+abstract class PigeonApiImageProxyUtils(
+    open val pigeonRegistrar: CameraXLibraryPigeonProxyApiRegistrar
+) {
+  /**
+   * Returns a single Byte Buffer that is representative of the [planes] that are NV21 compatible.
+   */
+  abstract fun getNv21Buffer(
+      imageWidth: Long,
+      imageHeight: Long,
+      planes: List<androidx.camera.core.ImageProxy.PlaneProxy>
+  ): ByteArray
+
+  companion object {
+    @Suppress("LocalVariableName")
+    fun setUpMessageHandlers(binaryMessenger: BinaryMessenger, api: PigeonApiImageProxyUtils?) {
+      val codec = api?.pigeonRegistrar?.codec ?: CameraXLibraryPigeonCodec()
+      run {
+        val channel =
+            BasicMessageChannel<Any?>(
+                binaryMessenger,
+                "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.getNv21Buffer",
+                codec)
+        if (api != null) {
+          channel.setMessageHandler { message, reply ->
+            val args = message as List<Any?>
+            val imageWidthArg = args[0] as Long
+            val imageHeightArg = args[1] as Long
+            val planesArg = args[2] as List<androidx.camera.core.ImageProxy.PlaneProxy>
+            val wrapped: List<Any?> =
+                try {
+                  listOf(api.getNv21Buffer(imageWidthArg, imageHeightArg, planesArg))
+                } catch (exception: Throwable) {
+                  CameraXLibraryPigeonUtils.wrapError(exception)
+                }
+            reply.reply(wrapped)
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
+    }
+  }
+
+  @Suppress("LocalVariableName", "FunctionName")
+  /** Creates a Dart instance of ImageProxyUtils and attaches it to [pigeon_instanceArg]. */
+  fun pigeon_newInstance(pigeon_instanceArg: ImageProxyUtils, callback: (Result<Unit>) -> Unit) {
+    if (pigeonRegistrar.ignoreCallsToDart) {
+      callback(
+          Result.failure(
+              CameraXError("ignore-calls-error", "Calls to Dart are being ignored.", "")))
+    } else if (pigeonRegistrar.instanceManager.containsInstance(pigeon_instanceArg)) {
+      callback(Result.success(Unit))
+    } else {
+      val pigeon_identifierArg =
+          pigeonRegistrar.instanceManager.addHostCreatedInstance(pigeon_instanceArg)
+      val binaryMessenger = pigeonRegistrar.binaryMessenger
+      val codec = pigeonRegistrar.codec
+      val channelName =
+          "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance"
+      val channel = BasicMessageChannel<Any?>(binaryMessenger, channelName, codec)
+      channel.send(listOf(pigeon_identifierArg)) {
+        if (it is List<*>) {
+          if (it.size > 1) {
+            callback(
+                Result.failure(CameraXError(it[0] as String, it[1] as String, it[2] as String?)))
+          } else {
+            callback(Result.success(Unit))
+          }
+        } else {
+          callback(Result.failure(CameraXLibraryPigeonUtils.createConnectionError(channelName)))
+        }
+      }
+    }
+  }
+}
 /**
  * A plane proxy which has an analogous interface as `android.media.Image.Plane`.
  *

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java

@@ -0,0 +1,72 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/* Utilities for working with {@code ImageProxy}s. */
+public class ImageProxyUtils {
+
+  /**
+   * Converts list of {@link PlaneProxy}s in YUV_420_888 format (with VU planes in NV21 layout) to a
+   * single NV21 {@code ByteBuffer}.
+   */
+  @NonNull
+  public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    if (!areUVPlanesNV21(planes, width, height)) {
+      throw new IllegalArgumentException(
+          "Provided UV planes are not in NV21 layout and thus cannot be converted.");
+    }
+
+    int imageSize = width * height;
+    int nv21Size = imageSize + 2 * (imageSize / 4);
+    byte[] nv21Bytes = new byte[nv21Size];
+
+    // Copy Y plane.
+    ByteBuffer yBuffer = planes.get(0).getBuffer();
+    yBuffer.rewind();
+    yBuffer.get(nv21Bytes, 0, imageSize);
+
+    // Copy interleaved VU plane (NV21 layout).
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+
+    vBuffer.rewind();
+    uBuffer.rewind();
+    vBuffer.get(nv21Bytes, imageSize, 1);
+    uBuffer.get(nv21Bytes, imageSize + 1, 2 * imageSize / 4 - 1);
+
+    return ByteBuffer.wrap(nv21Bytes);
+  }
+
+  public static boolean areUVPlanesNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    int imageSize = width * height;
+
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+
+    // Backup buffer properties.
+    int vBufferPosition = vBuffer.position();
+    int uBufferLimit = uBuffer.limit();
+
+    // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+    vBuffer.position(vBufferPosition + 1);
+    // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+    uBuffer.limit(uBufferLimit - 1);
+
+    // Check that the buffers are equal and have the expected number of elements.
+    boolean areNV21 =
+        (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+    // Restore buffers to their initial state.
+    vBuffer.position(vBufferPosition);
+    uBuffer.limit(uBufferLimit);
+
+    return areNV21;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java

@@ -0,0 +1,37 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * ProxyApi implementation for {@link ImageProxyUtils}. This class may handle instantiating native
+ * object instances that are attached to a Dart instance or handle method calls on the associated
+ * native class or an instance of that class.
+ */
+public class ImageProxyUtilsProxyApi extends PigeonApiImageProxyUtils {
+  ImageProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) {
+    super(pigeonRegistrar);
+  }
+
+  // List<? extends PlaneProxy> can be considered the same as List<PlaneProxy>.
+  @SuppressWarnings("unchecked")
+  @NonNull
+  @Override
+  public byte[] getNv21Buffer(
+      long imageWidth, long imageHeight, @NonNull List<? extends PlaneProxy> planes) {
+    final ByteBuffer nv21Buffer =
+        ImageProxyUtils.planesToNV21(
+            (List<PlaneProxy>) planes, (int) imageWidth, (int) imageHeight);
+
+    byte[] bytes = new byte[nv21Buffer.remaining()];
+    nv21Buffer.get(bytes, 0, bytes.length);
+
+    return bytes;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java

@@ -425,4 +425,10 @@ public PigeonApiMeteringPointFactory getPigeonApiMeteringPointFactory() {
   public CameraPermissionsErrorProxyApi getPigeonApiCameraPermissionsError() {
     return new CameraPermissionsErrorProxyApi(this);
   }
+
+  @NonNull
+  @Override
+  public PigeonApiImageProxyUtils getPigeonApiImageProxyUtils() {
+    return new ImageProxyUtilsProxyApi(this);
+  }
 }

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java

@@ -0,0 +1,48 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.mockito.Mockito.mockStatic;
+
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+public class ImageProxyUtilsApiTest {
+
+  @Test
+  public void getNv21Buffer_returnsExpectedBytes() {
+    final PigeonApiImageProxyUtils api = new TestProxyApiRegistrar().getPigeonApiImageProxyUtils();
+
+    List<PlaneProxy> planes =
+        Arrays.asList(
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class));
+    long width = 4;
+    long height = 2;
+    byte[] expectedBytes = new byte[] {1, 2, 3, 4, 5};
+    ByteBuffer mockBuffer = ByteBuffer.wrap(expectedBytes);
+
+    try (MockedStatic<ImageProxyUtils> mockedStatic = mockStatic(ImageProxyUtils.class)) {
+      mockedStatic
+          .when(
+              () ->
+                  ImageProxyUtils.planesToNV21(
+                      Mockito.anyList(), Mockito.anyInt(), Mockito.anyInt()))
+          .thenReturn(mockBuffer);
+
+      byte[] result = api.getNv21Buffer(width, height, planes);
+
+      assertArrayEquals(expectedBytes, result);
+      mockedStatic.verify(() -> ImageProxyUtils.planesToNV21(planes, (int) width, (int) height));
+    }
+  }
+}