#699 Add ML Kit support (ML Kit demo++, iOS rotation issues fixed, lower memory footprint)

Author: EddyVerbruggen

demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.html

@@ -4,14 +4,14 @@
 
 <GridLayout rows="auto, auto, *">
 
-  <Label row="0" text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 10th frame. These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>
+  <Label row="0" text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 5th frame (default 10). These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>
 
   <MLKitBarcodeScanner
       row="1"
       width="300"
       height="340"
       formats="QR_CODE, EAN_8, EAN_13"
-      processEveryNthFrame="10"
+      processEveryNthFrame="5"
       (scanResult)="onBarcodeScanResult($event)">
   </MLKitBarcodeScanner>
 

demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html

@@ -4,13 +4,13 @@
 
 <GridLayout rows="auto, auto, auto, auto, *">
 
-  <Label row="0" text="The scanner has been configured to detect faces every 5th frame. You can tweak this in your usage of the plugin." textWrap="true"></Label>
+  <Label row="0" text="The scanner has been configured to detect faces every 30th frame (default is 10). You can tweak this in your usage of the plugin." textWrap="true"></Label>
 
   <MLKitFaceDetection
       row="1"
       width="300"
       height="340"
-      processEveryNthFrame="5"
+      processEveryNthFrame="30"
       (scanResult)="onFaceDetectionResult($event)">
   </MLKitFaceDetection>
 

demo-ng/app/tabs/mlkit/mlkit.component.html

@@ -1,11 +1,12 @@
-<StackLayout class="tab-content">
+<GridLayout rows="auto, auto, *" class="tab-content">
 
-  <Label text="Pick the image processing source:" textWrap="true"></Label>
+  <Label row="0" text="Pick the image processing source:" textWrap="true"></Label>
 
-  <StackLayout orientation="horizontal" horizontalAlignment="center">
-    <Button text="Cameraroll" (tap)="fromCameraroll()" class="button button-mlkit"></Button>
-    <Button text="Cam picture" (tap)="fromCameraPicture()" class="button button-mlkit"></Button>
-    <Button text="Cam feed" (tap)="fromCameraFeed()" class="button button-mlkit"></Button>
+  <StackLayout row="1" orientation="horizontal" horizontalAlignment="center">
+    <Button text="Cameraroll" (tap)="fromCameraroll()" class="button"></Button>
+    <Button text="Cam picture" (tap)="fromCameraPicture()" class="button"></Button>
+    <Button text="Cam feed" (tap)="fromCameraFeed()" class="button"></Button>
   </StackLayout>
 
-</StackLayout>
+  <Image row="2" [src]="pickedImage" width="240" horizontalAlignment="center"></Image>
+</GridLayout>

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -1,4 +1,4 @@
-import { Component } from "@angular/core";
+import { Component, NgZone } from "@angular/core";
 import { ImageSource } from "tns-core-modules/image-source";
 
 import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
@@ -22,6 +22,8 @@ const firebase = require("nativescript-plugin-firebase");
 })
 export class MLKitComponent {
 
+  pickedImage: ImageSource;
+
   private mlkitFeatures: Array<string> = [
     "Text recognition",
     "Barcode scanning",
@@ -30,7 +32,8 @@ export class MLKitComponent {
     "Landmark recognition (cloud)"
   ];
 
-  constructor(private routerExtensions: RouterExtensions) {
+  constructor(private routerExtensions: RouterExtensions,
+              private zone: NgZone) {
   }
 
   fromCameraFeed(): void {
@@ -78,11 +81,13 @@ export class MLKitComponent {
       width: 800,
       height: 800,
       keepAspectRatio: true,
-      saveToGallery: false,
+      saveToGallery: true,
       cameraFacing: "rear"
     }).then(imageAsset => {
       new ImageSource().fromAsset(imageAsset).then(imageSource => {
-        this.selectMLKitFeature(imageSource);
+        this.pickedImage = imageSource;
+        // give the user some to to see the picture
+        setTimeout(() => this.selectMLKitFeature(imageSource), 500);
       });
     });
   }
@@ -96,24 +101,28 @@ export class MLKitComponent {
         .authorize()
         .then(() => imagePicker.present())
         .then((selection: Array<ImageAsset>) => {
-          console.log(">>> selection: " + selection);
           if (selection.length === 0) return;
 
           const selected = selection[0];
-          console.log(">>> selected: " + selected);
           selected.options.height = 800;
           selected.options.width = 800;
           selected.options.keepAspectRatio = true;
           selected.getImageAsync((image: any, error: any) => {
-            console.log(">>> error: " + error);
-            console.log(">>> image: " + image);
             if (error) {
-              console.log(`Error getting image source from pickerI: ${error}`);
+              console.log(`Error getting image source from picker: ${error}`);
+              return;
+            }
+            if (!image) {
+              console.log(`This is probably an iCloud image - which won't work`);
               return;
             }
             const imageSource = new ImageSource();
             imageSource.setNativeSource(image);
-            this.selectMLKitFeature(imageSource);
+            this.zone.run(() => {
+              this.pickedImage = imageSource;
+            });
+            // give the user some to to see the picture
+            setTimeout(() => this.selectMLKitFeature(imageSource), 500);
           });
         })
         .catch(e => {

docs/ML_KIT.md

@@ -2,3 +2,6 @@ Just dumping a few things I should not forget to add to the doc:
 
 - see step 3 @ https://firebase.google.com/docs/ml-kit/android/recognize-text
 - For Cloud processing, enable the Vision API and upgrade your Firebase project to "Blaze" (pay as you go)
+
+Known issues:
+- Detecting faces from still images doesn't currently work on iOS (detection from the camera stream works though)

src/mlkit/barcodescanning/index.android.ts

@@ -25,11 +25,11 @@ export class MLKitBarcodeScanner extends MLKitBarcodeScannerBase {
 
         if (barcodes.size() === 0) return;
 
-        const imageSource = new ImageSource();
-        imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
+        // const imageSource = new ImageSource();
+        // imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
 
         const result = <MLKitScanBarcodesResult>{
-          imageSource: imageSource,
+          // imageSource: imageSource,
           barcodes: []
         };
 

src/mlkit/facedetection/index.android.ts

@@ -17,11 +17,11 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
 
         if (faces.size() === 0) return;
 
-        const imageSource = new ImageSource();
-        imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
+        // const imageSource = new ImageSource();
+        // imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
 
         const result = <MLKitDetectFacesResult>{
-          imageSource: imageSource,
+          // imageSource: imageSource,
           faces: []
         };
 

src/mlkit/facedetection/index.ios.ts

@@ -4,9 +4,9 @@ import { MLKitOptions } from "../index";
 import { MLKitFaceDetection as MLKitFaceDetectionBase } from "./facedetection-common";
 
 export class MLKitFaceDetection extends MLKitFaceDetectionBase {
+
   protected createDetector(): any {
-    const firVision: FIRVision = FIRVision.vision();
-    return firVision.faceDetector();
+    return getDetector();
   }
 
   protected createSuccessListener(): any {
@@ -21,9 +21,9 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
 
         for (let i = 0, l = faces.count; i < l; i++) {
           const face: FIRVisionFace = faces.objectAtIndex(i);
+          console.log(">> face: " + face);
           result.faces.push({
-            // smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
-            smilingProbability: face.smilingProbability,
+            smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
             leftEyeOpenProbability: face.hasLeftEyeOpenProbability ? face.leftEyeOpenProbability : undefined,
             rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined
           });
@@ -39,18 +39,27 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
     };
   }
 
-  // TODO only seems to work in landscape
   protected rotateRecording(): boolean {
     return false;
   }
 }
 
+function getDetector(): FIRVisionFaceDetector {
+  // TODO make configurable
+  const firVision: FIRVision = FIRVision.vision();
+  const options = FIRVisionFaceDetectorOptions.new();
+  options.modeType = FIRVisionFaceDetectorMode.Accurate;
+  options.landmarkType = FIRVisionFaceDetectorLandmark.All;
+  options.classificationType = FIRVisionFaceDetectorClassification.All;
+  options.minFaceSize = 0.1;
+  // options.isTrackingEnabled = true;
+  return firVision.faceDetectorWithOptions(options);
+}
+
 export function detectFaces(options: MLKitDetectFacesOptions): Promise<MLKitDetectFacesResult> {
   return new Promise((resolve, reject) => {
     try {
-      const firVision: FIRVision = FIRVision.vision();
-      const faceDetector: FIRVisionFaceDetector = firVision.faceDetector();
-
+      const faceDetector = getDetector();
       faceDetector.detectInImageCompletion(getImage(options), (faces: NSArray<FIRVisionFace>, error: NSError) => {
         if (error !== null) {
           reject(error.localizedDescription);
@@ -60,15 +69,16 @@ export function detectFaces(options: MLKitDetectFacesOptions): Promise<MLKitDete
             faces: []
           };
 
+          console.log(">>> faces.count: " + faces.count);
           for (let i = 0, l = faces.count; i < l; i++) {
             const face: FIRVisionFace = faces.objectAtIndex(i);
+            console.log(">> face: " + face);
             result.faces.push({
               smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
               leftEyeOpenProbability: face.hasLeftEyeOpenProbability ? face.leftEyeOpenProbability : undefined,
               rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined
             });
           }
-          console.log(">>> face result: " + JSON.stringify(result.faces));
           resolve(result);
         }
       });
@@ -79,8 +89,19 @@ export function detectFaces(options: MLKitDetectFacesOptions): Promise<MLKitDete
   });
 }
 
-// TODO move
+// TODO resize the image (here), like the example app?
 function getImage(options: MLKitOptions): FIRVisionImage {
   const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
-  return FIRVisionImage.alloc().initWithImage(image);
+  console.log(">> image.imageOrientation: " + image.imageOrientation);
+
+  const fIRVisionImageMetadata = FIRVisionImageMetadata.new();
+  // fIRVisionImageMetadata.orientation = FIRVisionDetectorImageOrientation.TopLeft;
+
+  // const randomOrientation = Math.floor(Math.random() * 8) + 1;
+  // console.log(">>> randomOrientation: " + randomOrientation);
+  fIRVisionImageMetadata.orientation = 1;
+
+  const fIRVisionImage = FIRVisionImage.alloc().initWithImage(image);
+  fIRVisionImage.metadata = fIRVisionImageMetadata;
+  return fIRVisionImage;
 }

src/mlkit/imagelabeling/index.android.ts

@@ -17,11 +17,11 @@ export class MLKitImageLabeling extends MLKitImageLabelingBase {
 
         if (labels.size() === 0) return;
 
-        const imageSource = new ImageSource();
-        imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
+        // const imageSource = new ImageSource();
+        // imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
 
         const result = <MLKitImageLabelingResult>{
-          imageSource: imageSource,
+          // imageSource: imageSource,
           labels: []
         };
 

src/mlkit/index.ts

@@ -20,7 +20,7 @@ export interface MLKitResult {
    * Only set when using a camera stream,
    * because when detecting from static images... you already have the image, right? :)
    */
-  imageSource?: ImageSource;
+  // imageSource?: ImageSource;
 }
 
 export declare class MLKitCameraView {