#699 Add ML Kit support (iOS camera stream)

Author: EddyVerbruggen

.gitignore

@@ -7,6 +7,8 @@ package-lock.json
 build/
 *.log.*
 src/**/*.d.ts
+src/platforms/ios_lib/TNSMLKitCamera/TNSMLKitCamera.xcodeproj/project.xcworkspace
+src/platforms/ios_lib/TNSMLKitCamera/TNSMLKitCamera.xcodeproj/xcuserdata
 src/platforms/ios/Podfile
 src/platforms/ios/build.xcconfig
 src/platforms/android/include.gradle

demo-ng/app/app.module.ts

@@ -5,6 +5,12 @@ import { AppComponent } from "./app.component";
 
 import { ItemsComponent } from "./item/items.component";
 
+import { registerElement } from "nativescript-angular/element-registry";
+
+registerElement("MLKitBarcodeScanner", () => require("nativescript-plugin-firebase/mlkit/barcodescanning").MLKitBarcodeScanner);
+// registerElement("MLKitFaceRecognition", () => require("nativescript-plugin-firebase/mlkit/facedetection").MLKitFaceRecognition);
+// registerElement("MLKitTextRecognition", () => require("nativescript-plugin-firebase/mlkit/textrecognition").MLKitTextRecognition);
+
 @NgModule({
   bootstrap: [
     AppComponent

demo-ng/app/item/items.component.html

@@ -5,6 +5,13 @@
   <StackLayout class="page">
     <Label text="After pressing those buttons, check the console log" textWrap="true"></Label>
 
+    <MLKitBarcodeScanner
+        width="400"
+        height="300"
+        formats="QR_CODE, EAN_8"
+        (scanResult)="onBarcodeScanResult($event)">
+    </MLKitBarcodeScanner>
+
     <Label text="Authentication" class="h2"></Label>
     <Button text="login anonymously" (tap)="loginAnonymously()" class="button button-user"></Button>
 

demo-ng/app/item/items.component.ts

@@ -2,6 +2,7 @@ import { Component, NgZone } from "@angular/core";
 import { firestore } from "nativescript-plugin-firebase";
 import { Observable } from "rxjs/Observable";
 import { City } from "../model/City";
+import { MLKitScanBarcodesResult } from "../../../src/mlkit/barcodescanning";
 
 const firebase = require("nativescript-plugin-firebase/app");
 const firebaseWebApi = require("nativescript-plugin-firebase/app");
@@ -26,6 +27,11 @@ export class ItemsComponent {
     // AngularFireModule.initializeApp({});
   }
 
+  onBarcodeScanResult(event): void {
+    const result: MLKitScanBarcodesResult = event.value;
+    console.log("Received barcode(s): " + JSON.stringify(result));
+  }
+
   public loginAnonymously(): void {
     firebase.auth().signInAnonymously()
         .then(() => console.log("Logged in"))

src/.npmignore

@@ -5,6 +5,7 @@ tsconfig.json
 references.d.ts
 platforms/android/libraryproject/
 platforms/ios/typings/
+platforms/ios_lib/
 platforms/android/typings/
 platforms/web
 platforms/ios/Podfile

src/mlkit/barcodescanning/index.d.ts

@@ -2,8 +2,6 @@ import { MLKitOptions } from "../";
 import { BarcodeFormat } from "./barcodescanning-common";
 import { MLKitResult, MLKitCameraView } from "../index";
 
-export declare const BarcodeFormat: typeof BarcodeFormat;
-
 export interface MLKitScanBarcodesResult extends MLKitResult {
   barcodes: Array<{
     value: string;

src/mlkit/barcodescanning/index.ios.ts

@@ -1,37 +1,68 @@
 import { ImageSource } from "tns-core-modules/image-source";
-import { BarcodeFormat } from "./barcodescanning-common";
-import { MLKitCameraView as MLKitBarcodeScannerBase } from "../mlkit-cameraview";
 import { MLKitScanBarcodesOptions, MLKitScanBarcodesResult } from "./index";
 import { MLKitOptions } from "../index";
+import { BarcodeFormat, MLKitBarcodeScanner as MLKitBarcodeScannerBase } from "./barcodescanning-common";
 
 export { BarcodeFormat };
 
 export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
-
   protected createDetector(): any {
-  }
-
-  protected createFailureListener(): any {
+    let formats: Array<BarcodeFormat>;
+    if (this.formats) {
+      formats = [];
+      const requestedFormats = this.formats.split(",");
+      requestedFormats.forEach(format => formats.push(BarcodeFormat[format.trim().toUpperCase()]))
+    }
+    return getBarcodeDetector(formats);
   }
 
   protected createSuccessListener(): any {
+    return (barcodes: NSArray<FIRVisionBarcode>, error: NSError) => {
+      if (error !== null) {
+        console.log(error.localizedDescription);
+
+      } else if (barcodes !== null) {
+        const result = <MLKitScanBarcodesResult>{
+          barcodes: []
+        };
+
+        for (let i = 0, l = barcodes.count; i < l; i++) {
+          const barcode: FIRVisionBarcode = barcodes.objectAtIndex(i);
+          result.barcodes.push({
+            value: barcode.rawValue,
+            format: BarcodeFormat[barcode.format]
+          });
+        }
+
+        this.notify({
+          eventName: MLKitBarcodeScanner.scanResultEvent,
+          object: this,
+          value: result
+        });
+      }
+    }
   }
+}
 
-  // public onLayout(left: number, top: number, right: number, bottom: number): void {
-  //   super.onLayout(left, top, right, bottom);
-  //   if (this._hasSupport && this.ios) {
-  //     this._reader.previewLayer.frame = this.ios.layer.bounds;
-  //   }
-  // }
+function getBarcodeDetector(formats?: Array<BarcodeFormat>): any {
+  if (formats && formats.length > 0) {
+    // TODO
+    const barcodeDetector: FIRVisionBarcodeDetector = FIRVision.vision().barcodeDetector();
+    return barcodeDetector;
+    // const firebaseVisionBarcodeDetectorOptions =
+    //     new com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetectorOptions.Builder()
+    //         .setBarcodeFormats(formats[0], formats) // the seconds argument is a varargs.. let's make it easy and just do it like this
+    //         .build();
+    // return com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionBarcodeDetector(firebaseVisionBarcodeDetectorOptions);
+  } else {
+    return FIRVision.vision().barcodeDetector();
+  }
 }
 
 export function scanBarcodes(options: MLKitScanBarcodesOptions): Promise<MLKitScanBarcodesResult> {
   return new Promise((resolve, reject) => {
     try {
-      const firVision: FIRVision = FIRVision.vision();
-      // TODO pass in formats
-      const barcodeDetector: FIRVisionBarcodeDetector = firVision.barcodeDetector();
-      // const textDetector: FIRVisionBarcodeDetector = firVision.barcodeDetectorWithOptions();
+      const barcodeDetector = getBarcodeDetector(options.formats);
 
       barcodeDetector.detectInImageCompletion(getImage(options), (barcodes: NSArray<FIRVisionBarcode>, error: NSError) => {
         if (error !== null) {

src/mlkit/facedetection/index.d.ts

@@ -1,5 +1,4 @@
 import { MLKitOptions } from "../";
-import { BarcodeFormat } from "./barcodescanning-common";
 import { MLKitCameraView, MLKitResult } from "../index";
 
 export interface MLKitDetectFacesResult extends MLKitResult {

src/mlkit/facedetection/index.ios.ts

@@ -0,0 +1,49 @@
+import { ImageSource } from "tns-core-modules/image-source";
+import { MLKitDetectFacesOptions, MLKitDetectFacesResult } from "./";
+import { MLKitOptions } from "../index";
+
+// export class MLKitFaceDetection extends MLKitFaceDetectionBase {
+//   constructor() {
+//     super();
+//     console.log(">>> MLKitFaceDetection constr");
+//   }
+// }
+
+export function detectFaces(options: MLKitDetectFacesOptions): Promise<MLKitDetectFacesResult> {
+  return new Promise((resolve, reject) => {
+    try {
+      const firVision: FIRVision = FIRVision.vision();
+      const faceDetector: FIRVisionFaceDetector = firVision.faceDetector();
+
+      faceDetector.detectInImageCompletion(getImage(options), (faces: NSArray<FIRVisionFace>, error: NSError) => {
+        if (error !== null) {
+          reject(error.localizedDescription);
+
+        } else if (faces !== null) {
+          const result = <MLKitDetectFacesResult>{
+            faces: []
+          };
+
+          for (let i = 0, l = faces.count; i < l; i++) {
+            const face: FIRVisionFace = faces.objectAtIndex(i);
+            result.faces.push({
+              smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
+              leftEyeOpenProbability: face.hasLeftEyeOpenProbability ? face.leftEyeOpenProbability : undefined,
+              rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined
+            });
+          }
+          resolve(result);
+        }
+      });
+    } catch (ex) {
+      console.log("Error in firebase.mlkit.detectFaces: " + ex);
+      reject(ex);
+    }
+  });
+}
+
+// TODO move
+function getImage(options: MLKitOptions): FIRVisionImage {
+  const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
+  return FIRVisionImage.alloc().initWithImage(image);
+}

src/mlkit/index.ts

@@ -20,6 +20,9 @@ export interface MLKitResult {
   imageSource?: ImageSource;
 }
 
+export declare class MLKitCameraView {
+}
+
 export declare class MLKitBarcodeScanner {
 }