fix(android): much faster photo taking. However to make it fast it is prefered to set properties like pictureSize, captureMode ... on the view instead of as a arg of takePicture. If you do pass them to takePicture the whole configuration needs to be updated/set before taking the photo which is slow

Author: farfromrefug

packages/ui-cameraview/platforms/android/include.gradle

@@ -4,14 +4,14 @@ import groovy.json.JsonSlurper
 dependencies {
   implementation(name:'widgets-release', ext:'aar')
 
-  def camerax_version = "1.2.3"
-  implementation "androidx.camera:camera-core:${camerax_version}"
-  implementation "androidx.camera:camera-camera2:${camerax_version}"
-  implementation "androidx.camera:camera-lifecycle:${camerax_version}"
-  implementation "androidx.camera:camera-video:${camerax_version}"
-
-  implementation "androidx.camera:camera-view:${camerax_version}"
-  implementation "androidx.camera:camera-extensions:${camerax_version}"
+  def cameraxVersion =  project.hasProperty("cameraxVersion") ? project.cameraxVersion : "1.2.3"
+  implementation "androidx.camera:camera-core:${cameraxVersion}"
+  implementation "androidx.camera:camera-camera2:${cameraxVersion}"
+  implementation "androidx.camera:camera-lifecycle:${cameraxVersion}"
+  implementation "androidx.camera:camera-video:${cameraxVersion}"
+
+  implementation "androidx.camera:camera-view:${cameraxVersion}"
+  implementation "androidx.camera:camera-extensions:${cameraxVersion}"
 
   def androidxVersion =  project.hasProperty("androidxVersion") ? project.androidxVersion : "1.6.0"
   implementation "androidx.core:core:$androidxVersion"

packages/ui-cameraview/platforms/android/java/com/nativescript/cameraview/BitmapUtils.kt

@@ -1,5 +1,6 @@
 package com.nativescript.cameraview
 
+import android.annotation.SuppressLint
 import android.annotation.TargetApi
 import android.content.ContentResolver
 import android.graphics.*
@@ -8,9 +9,11 @@ import android.net.Uri
 import android.os.Build
 import android.provider.MediaStore
 import android.util.Log
+import androidx.annotation.OptIn
 import androidx.annotation.RequiresApi
 import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.ImageProxy
+import androidx.camera.core.internal.utils.ImageUtil
 import androidx.exifinterface.media.ExifInterface
 import java.io.ByteArrayOutputStream
 import java.io.IOException
@@ -39,58 +42,40 @@ import java.nio.ByteBuffer
 object BitmapUtils {
     private const val TAG = "BitmapUtils"
 
-
-    /** Converts NV21 format byte buffer to bitmap.  */
-    fun getBitmap(data: ByteArray, metadata: FrameMetadata): Bitmap? {
+    // based on androidx.camera.core.ImageSaver#imageToJpegByteArray(),
+    // optimized to avoid extracting uncropped image twice and to close ImageProxy sooner
+    @SuppressLint("RestrictedApi")
+    @Throws(ImageUtil.CodecFailedException::class)
+    public fun extractJpegBytes(image: ImageProxy, jpegQuality: Int): ByteArray {
         try {
-            val image = YuvImage(
-                data, ImageFormat.NV21, metadata.width, metadata.height, null
-            )
-            val stream = ByteArrayOutputStream()
-            image.compressToJpeg(Rect(0, 0, metadata.width, metadata.height), 80, stream)
-            val bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size())
-            stream.close()
-            return rotateBitmap(bmp, metadata.rotation, false, false)
-        } catch (e: Exception) {
-            Log.e("VisionProcessorBase", "Error: " + e.message)
+            var cropRect = if (ImageUtil.shouldCropImage(image)) image.cropRect else null
+            val imageFormat = image.format
+
+            var origJpegBytes = if (imageFormat == ImageFormat.JPEG) {
+                ImageUtil.jpegImageToJpegByteArray(image)
+            } else if (imageFormat == ImageFormat.YUV_420_888) {
+                ImageUtil.yuvImageToJpegByteArray(image, cropRect, jpegQuality)
+            } else {
+                throw IllegalStateException("unknown imageFormat $imageFormat")
+            }
+            return origJpegBytes!!;
+        } finally {
+            /*
+             from javadoc of the Image class:
+             Since Images are often directly produced or consumed by hardware components, they are
+             a limited resource shared across the system, and should be closed as soon as
+             they are no longer needed.
+             */
+            image.close()
         }
-        return null
     }
-
-
-    /** Converts NV21 format byte buffer to bitmap.  */
-    fun getBitmap(data: ByteBuffer, metadata: FrameMetadata): Bitmap? {
-        data.rewind()
-        val imageInBuffer = ByteArray(data.limit())
-        data[imageInBuffer, 0, imageInBuffer.size]
-        try {
-            val image = YuvImage(
-                imageInBuffer, ImageFormat.NV21, metadata.width, metadata.height, null
-            )
-            val stream = ByteArrayOutputStream()
-            image.compressToJpeg(Rect(0, 0, metadata.width, metadata.height), 80, stream)
-            val bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size())
-            stream.close()
-            return rotateBitmap(bmp, metadata.rotation, false, false)
-        } catch (e: Exception) {
-            Log.e("VisionProcessorBase", "Error: " + e.message)
-        }
-        return null
+    @OptIn(ExperimentalGetImage::class) public fun byteArrayFromProxy(image: ImageProxy, jpegQuality: Int): ByteArray? {
+        return extractJpegBytes(image,jpegQuality);
     }
-
-    /** Converts a YUV_420_888 image from CameraX API to a bitmap.  */
-    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
-    @ExperimentalGetImage
-    fun getBitmap(image: ImageProxy): Bitmap? {
-        val frameMetadata = FrameMetadata.Builder()
-            .setWidth(image.width)
-            .setHeight(image.height)
-            .setRotation(image.imageInfo.rotationDegrees)
-            .build()
-        val nv21Buffer = yuv420ThreePlanesToNV21(
-            image.image!!.planes, image.width, image.height
-        )
-        return getBitmap(nv21Buffer, frameMetadata)
+    @OptIn(ExperimentalGetImage::class) public fun getBitmap(image: ImageProxy, jpegQuality: Int): Bitmap {
+        val byteArray = extractJpegBytes(image, jpegQuality);
+        var bm = BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size);
+        return bm;
     }
 
     /** Rotates a bitmap if it is converted from a bytebuffer.  */
@@ -169,113 +154,4 @@ object BitmapUtils {
         }
         return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
     }
-
-    /**
-     * Converts YUV_420_888 to NV21 bytebuffer.
-     *
-     *
-     * The NV21 format consists of a single byte array containing the Y, U and V values. For an
-     * image of size S, the first S positions of the array contain all the Y values. The remaining
-     * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both
-     * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain
-     * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU
-     *
-     *
-     * YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled
-     * by a factor of 2 in both dimensions. [Image.getPlanes] returns an array with the Y, U and
-     * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into
-     * the first part of the NV21 array. The U and V planes may already have the representation in the
-     * NV21 format. This happens if the planes share the same buffer, the V buffer is one position
-     * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy
-     * them to the NV21 array.
-     */
-    @RequiresApi(Build.VERSION_CODES.KITKAT)
-    private fun yuv420ThreePlanesToNV21(
-        yuv420888planes: Array<Image.Plane>, width: Int, height: Int
-    ): ByteBuffer {
-        val imageSize = width * height
-        val out = ByteArray(imageSize + 2 * (imageSize / 4))
-        if (areUVPlanesNV21(yuv420888planes, width, height)) {
-            // Copy the Y values.
-            yuv420888planes[0].buffer[out, 0, imageSize]
-            val uBuffer = yuv420888planes[1].buffer
-            val vBuffer = yuv420888planes[2].buffer
-            // Get the first V value from the V buffer, since the U buffer does not contain it.
-            vBuffer[out, imageSize, 1]
-            // Copy the first U value and the remaining VU values from the U buffer.
-            uBuffer[out, imageSize + 1, 2 * imageSize / 4 - 1]
-        } else {
-            // Fallback to copying the UV values one by one, which is slower but also works.
-            // Unpack Y.
-            unpackPlane(yuv420888planes[0], width, height, out, 0, 1)
-            // Unpack U.
-            unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2)
-            // Unpack V.
-            unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2)
-        }
-        return ByteBuffer.wrap(out)
-    }
-
-    /** Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format.  */
-    @RequiresApi(Build.VERSION_CODES.KITKAT)
-    private fun areUVPlanesNV21(planes: Array<Image.Plane>, width: Int, height: Int): Boolean {
-        val imageSize = width * height
-        val uBuffer = planes[1].buffer
-        val vBuffer = planes[2].buffer
-
-        // Backup buffer properties.
-        val vBufferPosition = vBuffer.position()
-        val uBufferLimit = uBuffer.limit()
-
-        // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
-        vBuffer.position(vBufferPosition + 1)
-        // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
-        uBuffer.limit(uBufferLimit - 1)
-
-        // Check that the buffers are equal and have the expected number of elements.
-        val areNV21 =
-            vBuffer.remaining() == 2 * imageSize / 4 - 2 && vBuffer.compareTo(uBuffer) == 0
-
-        // Restore buffers to their initial state.
-        vBuffer.position(vBufferPosition)
-        uBuffer.limit(uBufferLimit)
-        return areNV21
-    }
-
-    /**
-     * Unpack an image plane into a byte array.
-     *
-     *
-     * The input plane data will be copied in 'out', starting at 'offset' and every pixel will be
-     * spaced by 'pixelStride'. Note that there is no row padding on the output.
-     */
-    @TargetApi(Build.VERSION_CODES.KITKAT)
-    private fun unpackPlane(
-        plane: Image.Plane, width: Int, height: Int, out: ByteArray, offset: Int, pixelStride: Int
-    ) {
-        val buffer = plane.buffer
-        buffer.rewind()
-
-        // Compute the size of the current plane.
-        // We assume that it has the aspect ratio as the original image.
-        val numRow = (buffer.limit() + plane.rowStride - 1) / plane.rowStride
-        if (numRow == 0) {
-            return
-        }
-        val scaleFactor = height / numRow
-        val numCol = width / scaleFactor
-
-        // Extract the data in the output buffer.
-        var outputPos = offset
-        var rowStart = 0
-        for (row in 0 until numRow) {
-            var inputPos = rowStart
-            for (col in 0 until numCol) {
-                out[outputPos] = buffer[inputPos]
-                outputPos += pixelStride
-                inputPos += plane.pixelStride
-            }
-            rowStart += plane.rowStride
-        }
-    }
 }

packages/ui-cameraview/platforms/android/java/com/nativescript/cameraview/CameraBase.kt

@@ -15,6 +15,8 @@ import android.util.AttributeSet
 import android.util.Log
 import android.view.OrientationEventListener
 import android.view.Surface
+import android.widget.FrameLayout
+import android.widget.LinearLayout
 import androidx.core.app.ActivityCompat
 import androidx.core.content.ContextCompat
 import java.io.IOException
@@ -60,11 +62,11 @@ abstract class CameraBase @JvmOverloads constructor(
     abstract fun stopPreview()
     abstract fun startRecording()
     abstract fun stopRecording()
-    abstract fun takePhoto(jsonStringOptions: String)
+    abstract fun takePhoto(jsonStringOptions: String? = null)
     abstract fun hasFlash(): Boolean
     abstract fun cameraRecording(): Boolean
     abstract fun toggleCamera()
-    abstract fun getSupportedRatios(): Array<String>
+//    abstract fun getSupportedRatios(): Array<String>
     abstract fun getAvailablePictureSizes(ratio: String): Array<Size>
     abstract fun getAllAvailablePictureSizes(): Array<Size>
     abstract var displayRatio: String

packages/ui-cameraview/platforms/android/java/com/nativescript/cameraview/CameraEventListener.kt

@@ -8,6 +8,7 @@
 package com.nativescript.cameraview
 
 import android.media.Image
+import android.graphics.Bitmap
 import androidx.camera.core.ImageInfo
 import java.io.File
 import java.lang.Exception
@@ -17,7 +18,7 @@ interface CameraEventListener {
     fun onCameraOpen()
     fun onCameraClose()
     fun onCameraPhoto(file: File?)
-    fun onCameraPhotoImage(image: Image?, info: ImageInfo, processor: ImageAsyncProcessor)
+    fun onCameraPhotoImage(image: Bitmap?, info: ImageInfo/* , processor: ImageAsyncProcessor */)
     fun onCameraVideo(file: File?)
     fun onCameraAnalysis(analysis: ImageAnalysis)
     fun onCameraError(message: String, ex: Exception)