#699 Add ML Kit support (renamed all methods to *OnDevice/*Cloud, and started documentation)

Author: EddyVerbruggen

demo-ng/app/app.css

@@ -75,15 +75,6 @@ button {
   background-color: #1832d5;
 }
 
-.mlKitCamera {
-  /*border-radius: 20;*/
-  /*border-width: 8;*/
-  /*border-color: #ddd;*/
-  width: 280;
-  height: 280;
-  margin-top: 16;
-}
-
 Label.mlkit-result {
   margin: 5;
   text-align: left;

demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.html

@@ -8,7 +8,7 @@
 
   <MLKitBarcodeScanner
       row="1"
-      width="300"
+      width="260"
       height="340"
       formats="QR_CODE, EAN_8, EAN_13"
       processEveryNthFrame="5"

demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.ts

@@ -1,5 +1,5 @@
 import { Component } from "@angular/core";
-import { MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
+import { MLKitScanBarcodesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
 
 @Component({
   selector: "mlkit-barcodescanning",
@@ -13,8 +13,7 @@ export class BarcodeScanningComponent {
   }>;
 
   onBarcodeScanResult(event): void {
-    const result: MLKitScanBarcodesResult = event.value;
+    const result: MLKitScanBarcodesOnDeviceResult = event.value;
     this.barcodes = result.barcodes;
-    console.log(">>> onBarcodeScanResult, this.barcodes: " + JSON.stringify(this.barcodes));
   }
 }

demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html

@@ -8,7 +8,7 @@
 
   <MLKitFaceDetection
       row="1"
-      width="300"
+      width="260"
       height="340"
       processEveryNthFrame="30"
       (scanResult)="onFaceDetectionResult($event)">
@@ -25,9 +25,9 @@
   <ListView row="4" [items]="faces">
     <ng-template let-item="item">
       <GridLayout columns="*, *, *">
-        <Label col="0" class="mlkit-result" textWrap="true" [text]="item.smilingProbability | number"></Label>
-        <Label col="1" class="mlkit-result" textWrap="true" [text]="item.leftEyeOpenProbability | number"></Label>
-        <Label col="2" class="mlkit-result" textWrap="true" [text]="item.rightEyeOpenProbability | number"></Label>
+        <Label col="0" class="mlkit-result" [class.c-purple]="item.smilingProbability > 0.7" [text]="item.smilingProbability | number" textWrap="true"></Label>
+        <Label col="1" class="mlkit-result" [class.c-purple]="item.leftEyeOpenProbability > 0.7" [text]="item.leftEyeOpenProbability | number" textWrap="true"></Label>
+        <Label col="2" class="mlkit-result" [class.c-purple]="item.rightEyeOpenProbability > 0.7" [text]="item.rightEyeOpenProbability | number" textWrap="true"></Label>
       </GridLayout>
     </ng-template>
   </ListView>

demo-ng/app/tabs/mlkit/facedetection/facedetection.component.ts

@@ -1,6 +1,5 @@
 import { Component } from "@angular/core";
-import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
-import { ImageSource } from "tns-core-modules/image-source";
+import { MLKitDetectFacesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/facedetection";
 
 @Component({
   selector: "mlkit-facedetection",
@@ -18,7 +17,7 @@ export class FaceDetectionComponent {
   mlKitAllOK: string;
 
   onFaceDetectionResult(scanResult: any): any {
-    const value: MLKitDetectFacesResult = scanResult.value;
+    const value: MLKitDetectFacesOnDeviceResult = scanResult.value;
     if (value.faces.length > 0) {
       this.faces = value.faces;
 

demo-ng/app/tabs/mlkit/imagelabeling/imagelabeling.component.html

@@ -6,7 +6,7 @@
 
   <MLKitImageLabeling
       row="0"
-      width="300"
+      width="260"
       height="340"
       (scanResult)="onImageLabeledResult($event)">
   </MLKitImageLabeling>

demo-ng/app/tabs/mlkit/imagelabeling/imagelabeling.component.ts

@@ -1,5 +1,5 @@
 import { Component } from "@angular/core";
-import { MLKitImageLabelingResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";
+import { MLKitImageLabelingOnDeviceResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";
 
 @Component({
   selector: "mlkit-imagelabeling",
@@ -13,7 +13,7 @@ export class ImageLabelingComponent {
   }>;
 
   onImageLabeledResult(scanResult: any): void {
-    const value: MLKitImageLabelingResult = scanResult.value;
+    const value: MLKitImageLabelingOnDeviceResult = scanResult.value;
     this.labels = value.labels;
   }
 }

demo-ng/app/tabs/mlkit/mlkit.component.html

@@ -4,7 +4,10 @@
 
   <StackLayout row="1" orientation="horizontal" horizontalAlignment="center">
     <Button text="Cameraroll" (tap)="fromCameraroll()" class="button"></Button>
-    <Button text="Cam picture" (tap)="fromCameraPicture()" class="button"></Button>
+    <!-- the image may require rotation on Android, and permission may be required - not feeling like exposing this for now -->
+    <iOS>
+      <Button text="Cam picture" (tap)="fromCameraPicture()" class="button"></Button>
+    </iOS>
     <Button text="Cam feed" (tap)="fromCameraFeed()" class="button"></Button>
   </StackLayout>
 

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -1,17 +1,22 @@
 import { Component, NgZone } from "@angular/core";
+import { RouterExtensions } from "nativescript-angular";
 import { ImageSource } from "tns-core-modules/image-source";
-
-import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
-import { MLKitRecognizeTextLocalResult, MLKitRecognizeTextCloudResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
-import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
-import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
 import { action } from "tns-core-modules/ui/dialogs";
 import { ImageAsset } from "tns-core-modules/image-asset";
+import { isIOS } from "tns-core-modules/platform";
 import * as ImagePicker from "nativescript-imagepicker";
 import * as Camera from "nativescript-camera";
-import { RouterExtensions } from "nativescript-angular";
-import { isIOS } from "tns-core-modules/platform";
-import { MLKitImageLabelingResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";
+import { BarcodeFormat, MLKitScanBarcodesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
+import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
+import { MLKitDetectFacesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/facedetection";
+import {
+  MLKitRecognizeTextCloudResult,
+  MLKitRecognizeTextOnDeviceResult
+} from "nativescript-plugin-firebase/mlkit/textrecognition";
+import {
+  MLKitImageLabelingCloudResult,
+  MLKitImageLabelingOnDeviceResult
+} from "nativescript-plugin-firebase/mlkit/imagelabeling";
 
 const firebase = require("nativescript-plugin-firebase");
 
@@ -25,15 +30,16 @@ export class MLKitComponent {
   pickedImage: ImageSource;
 
   private mlkitFeatures: Array<string> = [
-    "Text recognition (local)",
+    "Text recognition (on device)",
     "Text recognition (cloud)",
-    "Barcode scanning",
-    "Face detection",
-    "Image labeling",
+    "Barcode scanning (on device)",
+    "Face detection (on device)",
+    "Image labeling (on device)",
+    "Image labeling (cloud)",
     "Landmark recognition (cloud)"
   ];
 
-  private mlkitLocalFeatures: Array<string> = [
+  private mlkitOnDeviceFeatures: Array<string> = [
     "Text recognition",
     "Barcode scanning",
     "Face detection",
@@ -46,9 +52,9 @@ export class MLKitComponent {
 
   fromCameraFeed(): void {
     action(
-        "Test which ML Kit feature? No cloud processing will be used.",
+        "Test which on-device ML Kit feature?",
         "Cancel",
-        this.mlkitLocalFeatures
+        this.mlkitOnDeviceFeatures
     ).then((pickedItem: string) => {
       let to;
       if (pickedItem === "Text recognition") {
@@ -83,7 +89,7 @@ export class MLKitComponent {
       width: 800,
       height: 800,
       keepAspectRatio: true,
-      saveToGallery: true,
+      saveToGallery: false,
       cameraFacing: "rear"
     }).then(imageAsset => {
       new ImageSource().fromAsset(imageAsset).then(imageSource => {
@@ -147,34 +153,34 @@ export class MLKitComponent {
         this.mlkitFeatures
     ).then((pickedItem: string) => {
       let pickedItemIndex = this.mlkitFeatures.indexOf(pickedItem);
-      if (pickedItem === "Text recognition (local)") {
-        this.recognizeTextLocal(imageSource);
+      if (pickedItem === "Text recognition (on device)") {
+        this.recognizeTextOnDevice(imageSource);
       } else if (pickedItem === "Text recognition (cloud)") {
         this.recognizeTextCloud(imageSource);
-      } else if (pickedItem === "Barcode scanning") {
-        this.scanBarcode(imageSource);
-      } else if (pickedItem === "Face detection") {
-        this.detectFaces(imageSource);
-      } else if (pickedItem === "Image labeling") {
-        this.labelImage(imageSource);
+      } else if (pickedItem === "Barcode scanning (on device)") {
+        this.scanBarcodeOnDevice(imageSource);
+      } else if (pickedItem === "Face detection (on device)") {
+        this.detectFacesOnDevice(imageSource);
+      } else if (pickedItem === "Image labeling (on device)") {
+        this.labelImageOnDevice(imageSource);
+      } else if (pickedItem === "Image labeling (cloud)") {
+        this.labelImageCloud(imageSource);
       } else if (pickedItem === "Landmark recognition (cloud)") {
         this.recognizeLandmarkCloud(imageSource);
       }
     });
   }
 
-  private recognizeTextLocal(imageSource: ImageSource): void {
-    firebase.mlkit.textrecognition.recognizeTextLocal({
+  private recognizeTextOnDevice(imageSource: ImageSource): void {
+    firebase.mlkit.textrecognition.recognizeTextOnDevice({
       image: imageSource
-    }).then(
-        (result: MLKitRecognizeTextLocalResult) => {
-          alert({
-            title: `Result`,
-            message: result.features.map(feature => feature.text).join(""),
-            okButtonText: "OK"
-          });
-        })
-        .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+    }).then((result: MLKitRecognizeTextOnDeviceResult) => {
+      alert({
+        title: `Result`,
+        message: result.features.map(feature => feature.text).join(""),
+        okButtonText: "OK"
+      });
+    }).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
   private recognizeTextCloud(imageSource: ImageSource): void {
@@ -207,12 +213,13 @@ export class MLKitComponent {
         .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
-  private scanBarcode(imageSource: ImageSource): void {
-    firebase.mlkit.barcodescanning.scanBarcodes({
+  private scanBarcodeOnDevice(imageSource: ImageSource): void {
+    console.log(">>> imageSource.rotationAngle: " + imageSource.rotationAngle);
+    firebase.mlkit.barcodescanning.scanBarcodesOnDevice({
       image: imageSource,
       formats: [BarcodeFormat.QR_CODE, BarcodeFormat.EAN_13]
     }).then(
-        (result: MLKitScanBarcodesResult) => {
+        (result: MLKitScanBarcodesOnDeviceResult) => {
           alert({
             title: `Result`,
             message: JSON.stringify(result.barcodes),
@@ -222,11 +229,11 @@ export class MLKitComponent {
         .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
-  private detectFaces(imageSource: ImageSource): void {
-    firebase.mlkit.facedetection.detectFaces({
+  private detectFacesOnDevice(imageSource: ImageSource): void {
+    firebase.mlkit.facedetection.detectFacesOnDevice({
       image: imageSource
     }).then(
-        (result: MLKitDetectFacesResult) => {
+        (result: MLKitDetectFacesOnDeviceResult) => {
           alert({
             title: `Result`,
             message: JSON.stringify(result.faces),
@@ -236,12 +243,27 @@ export class MLKitComponent {
         .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
-  private labelImage(imageSource: ImageSource): void {
-    firebase.mlkit.imagelabeling.labelImage({
+  private labelImageOnDevice(imageSource: ImageSource): void {
+    firebase.mlkit.imagelabeling.labelImageOnDevice({
+      image: imageSource,
+      confidenceThreshold: 0.3
+    }).then(
+        (result: MLKitImageLabelingOnDeviceResult) => {
+          alert({
+            title: `Result`,
+            message: JSON.stringify(result.labels),
+            okButtonText: "OK"
+          });
+        })
+        .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+  }
+
+  private labelImageCloud(imageSource: ImageSource): void {
+    firebase.mlkit.imagelabeling.labelImageCloud({
       image: imageSource,
       confidenceThreshold: 0.3
     }).then(
-        (result: MLKitImageLabelingResult) => {
+        (result: MLKitImageLabelingCloudResult) => {
           alert({
             title: `Result`,
             message: JSON.stringify(result.labels),

demo-ng/app/tabs/mlkit/textrecognition/textrecognition.component.html

@@ -6,7 +6,7 @@
 
   <MLKitTextRecognition
       row="0"
-      width="300"
+      width="260"
       height="340"
       (scanResult)="onTextRecognitionResult($event)">
   </MLKitTextRecognition>