#699 Add ML Kit support

Author: EddyVerbruggen

demo/app/app.css

@@ -82,8 +82,7 @@ button {
 }
 
 .barcodescanner {
-  width: 320;
-  height: 260;
+  height: 420;
   margin-top: 16;
 }
 

demo/app/main-page.ts

@@ -3,6 +3,7 @@ import * as pages from 'tns-core-modules/ui/page';
 import { HelloWorldModel } from './main-view-model';
 import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
 import { MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
+import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
 
 const model = new HelloWorldModel();
 
@@ -24,5 +25,30 @@ export function onBarcodesScanResult(scanResult: any) {
 
 export function onTextRecognitionResult(scanResult: any) {
   const value: MLKitRecognizeTextResult = scanResult.value;
-  model.set("textValue", value.features.map(feature => feature.text).join("\n"));
+  model.set("textValue", value.features.map(feature => feature.text).join("\n\n"));
+}
+
+export function onFaceDetectionResult(scanResult: any) {
+  const value: MLKitDetectFacesResult = scanResult.value;
+  if (value.faces.length > 0) {
+    let allSmilingAndEyesOpen = true;
+    value.faces.forEach(face => {
+      allSmilingAndEyesOpen = allSmilingAndEyesOpen && face.smilingProbability && face.leftEyeOpenProbability && face.rightEyeOpenProbability &&
+          face.smilingProbability > 0.7 && face.leftEyeOpenProbability > 0.7 && face.rightEyeOpenProbability > 0.7;
+    });
+    model.set("allOK", `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes, screen grabbed:' : 'Nope. Sad.'}`);
+    // model.set("textValue", value.faces.map(face => JSON.stringify(face)).join("\n"));
+    model.set("textValue", value.faces.map(face => `Smiling? ${round(face.smilingProbability)}%\nLeft eye open? ${round(face.leftEyeOpenProbability)}%\nRight eye open? ${round(face.rightEyeOpenProbability)}%`).join("\n\n"));
+
+    if (allSmilingAndEyesOpen && value.imageSource) {
+      model.set("lastMatch", value.imageSource);
+    }
+  }
+}
+
+function round (input) {
+  if (isNaN(input)) {
+    return 0;
+  }
+  return Math.round(input * 100);
 }

demo/app/main-page.xml

@@ -1,4 +1,8 @@
-<TabView loaded="pageLoaded" class="page" xmlns:FirebaseMLKitBarcodeScanning="nativescript-plugin-firebase/mlkit/barcodescanning">
+<TabView
+    loaded="pageLoaded"
+    class="page"
+    xmlns:FirebaseMLKitBarcodeScanning="nativescript-plugin-firebase/mlkit/barcodescanning"
+    xmlns:FirebaseMLKitFaceDetection="nativescript-plugin-firebase/mlkit/facedetection">
   <TabView.items>
 
     <TabViewItem title="ML Kit">
@@ -146,11 +150,24 @@
                 class="barcodescanner"
                 scanResult="onBarcodesScanResult" />
 
-            <!--<Label row="3" text="Barcode format:" />-->
-            <!--<Label row="3" col="1" text="{{ barcodeFormat }}" />-->
+            <!--FirebaseMLKitFaceDetection:MLKitFaceDetection
+                row="2"
+                colSpan="2"
+                class="barcodescanner"
+                scanResult="onFaceDetectionResult" /-->
+
+            <Label row="3" text="Barcode format:" />
+            <Label row="3" col="1" text="{{ barcodeFormat }}" />
+
+            <Label row="4" text="Barcode value:" />
+            <Label row="4" col="1" text="{{ barcodeValue }}" />
+
+            <!--<Label row="3" colSpan="2" text="{{ allOK }}" textWrap="true" />-->
+
+            <!--<Label row="4" text="{{ textValue }}" textWrap="true" />-->
+            <!--<Label row="4" col="1" text="If all above 70%, the image is shown here." textWrap="true" visibility="{{ lastMatch ? 'collapsed' : 'visible' }} " />-->
+            <!--<Image row="4" col="1" src="{{ lastMatch }}" visibility="{{ lastMatch ? 'visible' : 'collapsed' }} " />-->
 
-            <!--<Label row="4" text="Barcode value:" />-->
-            <Label row="3" colSpan="2" text="{{ textValue }}" textWrap="true" />
 
 
 

demo/app_resources/Android/src/main/AndroidManifest.xml

@@ -31,7 +31,7 @@
 
 		<meta-data
 				android:name="com.google.firebase.ml.vision.DEPENDENCIES"
-				android:value="text,barcode" />
+				android:value="text,barcode,face" />
 
 		<activity
 			android:name="com.tns.NativeScriptActivity"

src/mlkit/barcodescanning/barcodescanning-common.ts

@@ -1,6 +1,6 @@
-import { ContentView } from "tns-core-modules/ui/content-view";
 import { booleanConverter } from "tns-core-modules/ui/core/view-base";
 import { Property } from "tns-core-modules/ui/core/properties";
+import { MLKitCameraView } from "../mlkit-cameraview";
 
 export enum BarcodeFormat {
   // UNKNOWN = -1,
@@ -20,10 +20,6 @@ export enum BarcodeFormat {
   AZTEC = 4096,
 }
 
-export function getBarcodeFormatFromIndex(index: number): string {
-  return BarcodeFormat[index];
-}
-
 export const formatsProperty = new Property<MLKitBarcodeScanner, string>({
   name: "formats",
   defaultValue: null,
@@ -47,7 +43,7 @@ export const reportDuplicatesProperty = new Property<MLKitBarcodeScanner, boolea
   valueConverter: booleanConverter
 });
 
-export abstract class MLKitBarcodeScanner extends ContentView {
+export abstract class MLKitBarcodeScanner extends MLKitCameraView {
   static scanResultEvent: string = "scanResult";
 
   protected formats: string;