Bounds property of barcode scan result not the same for iOS and Android #1288

Author: EddyVerbruggen

src/mlkit/barcodescanning/index.android.ts

@@ -1,6 +1,5 @@
 import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitScanBarcodesOnDeviceOptions, MLKitScanBarcodesOnDeviceResult, MLKitScanBarcodesResultBounds } from "./";
-import { MLKitVisionOptions } from "../index";
 import { BarcodeFormat, MLKitBarcodeScanner as MLKitBarcodeScannerBase } from "./barcodescanning-common";
 import * as application from "tns-core-modules/application";
 
@@ -68,14 +67,21 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
         };
 
         if (barcodes && barcodes.size() > 0) {
+
+          const image: android.graphics.Bitmap = this.lastVisionImage ? this.lastVisionImage.getBitmap() : null;
+
           // see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
           for (let i = 0; i < barcodes.size(); i++) {
             const barcode = barcodes.get(i);
             result.barcodes.push({
               value: barcode.getRawValue(),
               format: BarcodeFormat[barcode.getFormat()],
               android: barcode,
-              bounds: boundingBoxToBounds(barcode.getBoundingBox())
+              bounds: boundingBoxToBounds(barcode.getBoundingBox()),
+              image: !image ? null : {
+                width: image.getWidth(),
+                height: image.getHeight()
+              }
             });
           }
 
@@ -124,6 +130,9 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
     try {
       const firebaseVisionBarcodeDetector = getBarcodeDetector(options.formats);
 
+      const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
+      const firImage = com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
+
       const onSuccessListener = new gmsTasks.OnSuccessListener({
         onSuccess: barcodes => {
           const result = <MLKitScanBarcodesOnDeviceResult>{
@@ -138,7 +147,11 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
                 value: barcode.getRawValue(),
                 format: BarcodeFormat[barcode.getFormat()],
                 android: barcode,
-                bounds: boundingBoxToBounds(barcode.getBoundingBox())
+                bounds: boundingBoxToBounds(barcode.getBoundingBox()),
+                image: {
+                  width: image.getWidth(),
+                  height: image.getHeight()
+                }
               });
             }
           }
@@ -153,7 +166,7 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
       });
 
       firebaseVisionBarcodeDetector
-          .detectInImage(getImage(options))
+          .detectInImage(firImage)
           .addOnSuccessListener(onSuccessListener)
           .addOnFailureListener(onFailureListener);
 
@@ -163,8 +176,3 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
     }
   });
 }
-
-function getImage(options: MLKitVisionOptions): any /* com.google.firebase.ml.vision.common.FirebaseVisionImage */ {
-  const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
-  return com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
-}

src/mlkit/barcodescanning/index.d.ts

@@ -17,7 +17,11 @@ export interface MLKitScanBarcodesResultBounds {
 export interface MLKitScanBarcodesResultBarcode {
   value: string;
   format: string;
-  bounds?: MLKitScanBarcodesResultBounds;
+  bounds: MLKitScanBarcodesResultBounds;
+  image: {
+    width: number;
+    height: number;
+  }
   ios?: any;
   android?: any;
 }

src/mlkit/barcodescanning/index.ios.ts

@@ -46,6 +46,8 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
         for (let i = 0, l = barcodes.count; i < l; i++) {
           const barcode: FIRVisionBarcode = barcodes.objectAtIndex(i);
           const image: UIImage = this.lastVisionImage;
+          let imageWidth = image.size.width;
+          let imageHeight = image.size.height;
 
           // the iOS image is rotated, so compensate for it when reporting these
           let { x, y } = barcode.frame.origin;
@@ -54,6 +56,7 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
           if (image) {
             const origX = x;
             const origWidth = width;
+            const origImageWidth = imageWidth;
 
             if (iosUtils.isLandscape()) {
               if (UIDevice.currentDevice.orientation === UIDeviceOrientation.LandscapeRight) {
@@ -67,9 +70,9 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
               y = origX;
               width = height;
               height = origWidth;
+              imageWidth = imageHeight;
+              imageHeight = origImageWidth;
             }
-
-            console.log("iosUtils.isLandscape(): " + iosUtils.isLandscape() + ", deviceOrientation: " + UIDevice.currentDevice.orientation);
           }
 
           result.barcodes.push({
@@ -85,6 +88,10 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
                 width,
                 height
               }
+            },
+            image: {
+              width: imageWidth,
+              height: imageHeight
             }
           });
         }
@@ -122,7 +129,10 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
     try {
       const barcodeDetector = getBarcodeDetector(options.formats);
 
-      barcodeDetector.detectInImageCompletion(getImage(options), (barcodes: NSArray<FIRVisionBarcode>, error: NSError) => {
+      const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
+      const firImage = FIRVisionImage.alloc().initWithImage(image);
+
+      barcodeDetector.detectInImageCompletion(firImage, (barcodes: NSArray<FIRVisionBarcode>, error: NSError) => {
         if (error !== null) {
           reject(error.localizedDescription);
 
@@ -137,7 +147,11 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
               value: barcode.rawValue,
               format: BarcodeFormat[barcode.format],
               ios: barcode,
-              bounds: barcode.frame
+              bounds: barcode.frame,
+              image: {
+                width: image.size.width,
+                height: image.size.height
+              }
             });
           }
           resolve(result);
@@ -149,8 +163,3 @@ export function scanBarcodesOnDevice(options: MLKitScanBarcodesOnDeviceOptions):
     }
   });
 }
-
-function getImage(options: MLKitVisionOptions): FIRVisionImage {
-  const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
-  return FIRVisionImage.alloc().initWithImage(image);
-}