Bounds property of barcode scan result not the same for iOS and Android #1288

Author: EddyVerbruggen

src/mlkit/barcodescanning/index.d.ts

@@ -17,10 +17,9 @@ export interface MLKitScanBarcodesResultBounds {
 export interface MLKitScanBarcodesResultBarcode {
   value: string;
   format: string;
+  bounds?: MLKitScanBarcodesResultBounds;
   ios?: any;
   android?: any;
-  bounds?: MLKitScanBarcodesResultBounds;
-  // TODO details
 }
 
 export interface MLKitScanBarcodesOnDeviceResult extends MLKitVisionResult {

src/mlkit/barcodescanning/index.ios.ts

@@ -1,4 +1,5 @@
 import { ImageSource } from "tns-core-modules/image-source";
+import { ios as iosUtils } from "tns-core-modules/utils/utils";
 import { MLKitScanBarcodesOnDeviceOptions, MLKitScanBarcodesOnDeviceResult } from "./index";
 import { MLKitVisionOptions } from "../index";
 import { BarcodeFormat, MLKitBarcodeScanner as MLKitBarcodeScannerBase } from "./barcodescanning-common";
@@ -44,11 +45,47 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
 
         for (let i = 0, l = barcodes.count; i < l; i++) {
           const barcode: FIRVisionBarcode = barcodes.objectAtIndex(i);
+          const image: UIImage = this.lastVisionImage;
+
+          // the iOS image is rotated, so compensate for it when reporting these
+          let { x, y } = barcode.frame.origin;
+          let { width, height } = barcode.frame.size;
+
+          if (image) {
+            const origX = x;
+            const origWidth = width;
+
+            if (iosUtils.isLandscape()) {
+              if (UIDevice.currentDevice.orientation === UIDeviceOrientation.LandscapeRight) {
+                // the image is rotated 180 degrees
+                x = image.size.width - (width + x);
+                y = image.size.height - (height + y);
+              }
+            } else {
+              // the image is rotated 90 degrees to the left
+              x = image.size.height - (height + y);
+              y = origX;
+              width = height;
+              height = origWidth;
+            }
+
+            console.log("iosUtils.isLandscape(): " + iosUtils.isLandscape() + ", deviceOrientation: " + UIDevice.currentDevice.orientation);
+          }
+
           result.barcodes.push({
             value: barcode.rawValue,
             format: BarcodeFormat[barcode.format],
             ios: barcode,
-            bounds: barcode.frame
+            bounds: {
+              origin: {
+                x,
+                y
+              },
+              size: {
+                width,
+                height
+              }
+            }
           });
         }
 

src/mlkit/mlkit-cameraview-common.ts

@@ -28,7 +28,7 @@ export const pauseProperty = new Property<MLKitCameraView, boolean>({
 export abstract class MLKitCameraView extends ContentView {
   static scanResultEvent: string = "scanResult";
 
-  protected lastVisionImage;
+  public lastVisionImage;
 
   protected processEveryNthFrame: number;
   protected preferFrontCamera: boolean;

src/mlkit/mlkit-cameraview.android.ts

@@ -26,7 +26,7 @@ export abstract class MLKitCameraView extends MLKitCameraViewBase {
   private bytesToByteBuffer = new Map();
   private pendingFrameData = null;
   protected rotation;
-  protected lastVisionImage;
+  public lastVisionImage;
   private detector: any;
   private camera;
 

src/mlkit/mlkit-cameraview.d.ts

@@ -1,7 +1,7 @@
 import { MLKitCameraView as MLKitCameraViewBase } from "./mlkit-cameraview-common";
 
 export declare abstract class MLKitCameraView extends MLKitCameraViewBase {
-  protected lastVisionImage;
+  public lastVisionImage;
 
   protected abstract createDetector(): any;
 

src/mlkit/mlkit-cameraview.ios.ts

@@ -9,6 +9,8 @@ export abstract class MLKitCameraView extends MLKitCameraViewBase {
   private previewLayer: AVCaptureVideoPreviewLayer;
   private cameraView: TNSMLKitCameraView;
 
+  public lastVisionImage;
+
   disposeNativeView(): void {
     super.disposeNativeView();
     if (this.captureSession) {
@@ -217,6 +219,8 @@ class TNSMLKitCameraViewDelegateImpl extends NSObject implements TNSMLKitCameraV
       this.detectorBusy = false;
     };
 
+    this.owner.get().lastVisionImage = image;
+
     if (this.detector.detectInImageCompletion) {
       this.detector.detectInImageCompletion(this.uiImageToFIRVisionImage(image), (result, error) => {
         this.onSuccessListener(result, error);