Expose more configuration and result properties to ML Kit's face detection #704

EddyVerbruggen · EddyVerbruggen · commit 5973326294ea · 2018-06-07T09:53:04.000+02:00
diff --git a/demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html b/demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html
@@ -8,6 +8,9 @@
         width="100%"
         height="100%"
         processEveryNthFrame="30"
+        enableFaceTracking="true"
+        minimumFaceSize="0.2"
+        modeType="accurate"
         (scanResult)="onFaceDetectionResult($event)">
     </MLKitFaceDetection>
 
@@ -37,20 +40,22 @@
       <Label row="0" col="1" class="text-center c-white" textWrap="true" color="white" verticalAlignment="center" text="The scanner has been configured to detect faces every 30th frame (default is 10). You can tweak this in your usage of the plugin."></Label>
       <StackLayout row="2" col="0" colSpan="3">
         <Label [text]="mlKitAllOK" textWrap="true" class="m-t-5 c-purple"></Label>
-        <GridLayout row="auto" columns="*, *, *" class="m-t-5">
-          <Label row="0" col="0" class="mlkit-result font-weight-bold" textWrap="true" text="Smiling"></Label>
-          <Label row="0" col="1" class="mlkit-result font-weight-bold" textWrap="true" text="Left 👁 open"></Label>
-          <Label row="0" col="2" class="mlkit-result font-weight-bold" textWrap="true" text="Right 👁 open"></Label>
+        <GridLayout row="auto" columns="60, *, *, *" class="m-t-5">
+          <Label row="0" col="0" class="mlkit-result font-weight-bold" textWrap="true" text="ID"></Label>
+          <Label row="0" col="1" class="mlkit-result font-weight-bold" textWrap="true" text="Smiling"></Label>
+          <Label row="0" col="2" class="mlkit-result font-weight-bold" textWrap="true" text="Left 👁 open"></Label>
+          <Label row="0" col="3" class="mlkit-result font-weight-bold" textWrap="true" text="Right 👁 open"></Label>
         </GridLayout>
       </StackLayout>
     </GridLayout>
   </GridLayout>
   <ListView backgroundColor="black" row="1" [items]="faces">
     <ng-template let-item="item">
-      <GridLayout columns="*, *, *">
-        <Label col="0" class="mlkit-result c-white" [class.c-purple]="item.smilingProbability > 0.7" [text]="item.smilingProbability | number" textWrap="true"></Label>
-        <Label col="1" class="mlkit-result c-white" [class.c-purple]="item.leftEyeOpenProbability > 0.7" [text]="item.leftEyeOpenProbability | number" textWrap="true"></Label>
-        <Label col="2" class="mlkit-result c-white" [class.c-purple]="item.rightEyeOpenProbability > 0.7" [text]="item.rightEyeOpenProbability | number" textWrap="true"></Label>
+      <GridLayout columns="50, *, *, *">
+        <Label col="0" class="mlkit-result c-white" [text]="item.trackingId" textWrap="true"></Label>
+        <Label col="1" class="mlkit-result c-white" [class.c-purple]="item.smilingProbability > 0.7" [text]="item.smilingProbability | number" textWrap="true"></Label>
+        <Label col="2" class="mlkit-result c-white" [class.c-purple]="item.leftEyeOpenProbability > 0.7" [text]="item.leftEyeOpenProbability | number" textWrap="true"></Label>
+        <Label col="3" class="mlkit-result c-white" [class.c-purple]="item.rightEyeOpenProbability > 0.7" [text]="item.rightEyeOpenProbability | number" textWrap="true"></Label>
       </GridLayout>
     </ng-template>
   </ListView>
diff --git a/demo-ng/app/tabs/mlkit/mlkit.component.ts b/demo-ng/app/tabs/mlkit/mlkit.component.ts
@@ -236,7 +236,10 @@ export class MLKitComponent {
 
   private detectFacesOnDevice(imageSource: ImageSource): void {
     firebase.mlkit.facedetection.detectFacesOnDevice({
-      image: imageSource
+      image: imageSource,
+      detectionMode: "accurate",
+      enableFaceTracking: false,
+      minimumFaceSize: 0.25
     }).then(
         (result: MLKitDetectFacesOnDeviceResult) => {
           alert({
diff --git a/docs/ML_KIT.md b/docs/ML_KIT.md
@@ -171,7 +171,10 @@ import { MLKitDetectFacesOnDeviceResult } from "nativescript-plugin-firebase/mlk
 const firebase = require("nativescript-plugin-firebase");
 
 firebase.mlkit.facedetection.detectFacesOnDevice({
-  image: imageSource // a NativeScript Image or ImageSource, see the demo for examples
+  image: imageSource, // a NativeScript Image or ImageSource, see the demo for examples
+  detectionMode: "accurate", // default "fast"
+  enableFaceTracking: true, // default false
+  minimumFaceSize: 0.25 // default 0.1 (which means the face must be at least 10% of the image)
 })
 .then((result: MLKitDetectFacesOnDeviceResult) => console.log(JSON.stringify(result.faces)))
 .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
@@ -189,6 +192,9 @@ registerElement("MLKitFaceDetection", () => require("nativescript-plugin-firebas
 <MLKitFaceDetection
     width="260"
     height="380"
+    detectionMode="accurate"
+    enableFaceTracking="true"
+    minimumFaceSize="0.2"
     (scanResult)="onFaceDetectionResult($event)">
 </MLKitFaceDetection>
 ```
diff --git a/src/mlkit/facedetection/facedetection-common.ts b/src/mlkit/facedetection/facedetection-common.ts
@@ -1,22 +1,46 @@
-import { booleanConverter } from "tns-core-modules/ui/core/view-base";
+import { booleanConverter, makeParser, makeValidator } from "tns-core-modules/ui/core/view-base";
 import { Property } from "tns-core-modules/ui/core/properties";
 import { MLKitCameraView } from "../mlkit-cameraview";
+import { MLKitFaceDetectionMode } from "./";
 
-export const preferFrontCameraProperty = new Property<MLKitFaceDetection, boolean>({
-  name: "preferFrontCamera",
+export const minimumFaceSizeProperty = new Property<MLKitFaceDetection, number>({
+  name: "minimumFaceSize",
+  defaultValue: 0.1
+});
+
+export const enableFaceTrackingProperty = new Property<MLKitFaceDetection, boolean>({
+  name: "enableFaceTracking",
   defaultValue: false,
   valueConverter: booleanConverter
 });
 
+const detectionModeConverter = makeParser<MLKitFaceDetectionMode>(makeValidator<MLKitFaceDetectionMode>("accurate", "fast"));
+export const detectionModeProperty = new Property<MLKitFaceDetection, MLKitFaceDetectionMode>({
+  name: "detectionMode",
+  defaultValue: "fast",
+  valueConverter: detectionModeConverter
+});
+
 export abstract class MLKitFaceDetection extends MLKitCameraView {
   static scanResultEvent: string = "scanResult";
 
-  protected formats: string;
-  protected preferFrontCamera: boolean;
+  protected enableFaceTracking: boolean;
+  protected detectionMode: MLKitFaceDetectionMode;
+  protected minimumFaceSize: number;
+
+  [minimumFaceSizeProperty.setNative](value: number) {
+    this.minimumFaceSize = value;
+  }
+
+  [enableFaceTrackingProperty.setNative](value: boolean) {
+    this.enableFaceTracking = value;
+  }
 
-  [preferFrontCameraProperty.setNative](value: boolean) {
-    this.preferFrontCamera = value;
+  [detectionModeProperty.setNative](value: MLKitFaceDetectionMode) {
+    this.detectionMode = value;
   }
 }
 
-preferFrontCameraProperty.register(MLKitFaceDetection);
+minimumFaceSizeProperty.register(MLKitFaceDetection);
+enableFaceTrackingProperty.register(MLKitFaceDetection);
+detectionModeProperty.register(MLKitFaceDetection);
diff --git a/src/mlkit/facedetection/index.android.ts b/src/mlkit/facedetection/index.android.ts
@@ -8,7 +8,11 @@ declare const com: any;
 export class MLKitFaceDetection extends MLKitFaceDetectionBase {
 
   protected createDetector(): any {
-    return getFaceDetector();
+    return getFaceDetector({
+      detectionMode: this.detectionMode,
+      enableFaceTracking: this.enableFaceTracking,
+      minimumFaceSize: this.minimumFaceSize
+    });
   }
 
   protected createSuccessListener(): any {
@@ -31,7 +35,8 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
           result.faces.push({
             smilingProbability: face.getSmilingProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getSmilingProbability() : undefined,
             leftEyeOpenProbability: face.getLeftEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getLeftEyeOpenProbability() : undefined,
-            rightEyeOpenProbability: face.getRightEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getRightEyeOpenProbability() : undefined
+            rightEyeOpenProbability: face.getRightEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getRightEyeOpenProbability() : undefined,
+            trackingId: face.getTrackingId() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.INVALID_ID ? face.getTrackingId() : undefined
           });
         }
 
@@ -45,14 +50,14 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
   }
 }
 
-function getFaceDetector(): any {
+function getFaceDetector(options: MLKitDetectFacesOnDeviceOptions): any {
   const faceDetectorOptions =
       new com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.Builder()
-          .setModeType(com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ACCURATE_MODE)
-          .setLandmarkType(com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS)
-          .setClassificationType(com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS)
-          .setMinFaceSize(0.15)
-          .setTrackingEnabled(false)
+          .setModeType(options.detectionMode === "accurate" ? com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ACCURATE_MODE : com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.FAST_MODE)
+          // .setLandmarkType(com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS) // TODO
+          // .setClassificationType(com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS) // TODO
+          .setMinFaceSize(options.minimumFaceSize)
+          .setTrackingEnabled(options.enableFaceTracking === true)
           .build();
 
   return com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionFaceDetector(faceDetectorOptions);
@@ -61,7 +66,7 @@ function getFaceDetector(): any {
 export function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): Promise<MLKitDetectFacesOnDeviceResult> {
   return new Promise((resolve, reject) => {
     try {
-      const firebaseVisionFaceDetector = getFaceDetector();
+      const firebaseVisionFaceDetector = getFaceDetector(options);
 
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: (faces) => {
@@ -76,7 +81,8 @@ export function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): P
             result.faces.push({
               smilingProbability: face.getSmilingProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getSmilingProbability() : undefined,
               leftEyeOpenProbability: face.getLeftEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getLeftEyeOpenProbability() : undefined,
-              rightEyeOpenProbability: face.getRightEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getRightEyeOpenProbability() : undefined
+              rightEyeOpenProbability: face.getRightEyeOpenProbability() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.UNCOMPUTED_PROBABILITY ? face.getRightEyeOpenProbability() : undefined,
+              trackingId: face.getTrackingId() !== com.google.firebase.ml.vision.face.FirebaseVisionFace.INVALID_ID ? face.getTrackingId() : undefined
             });
           }
 
diff --git a/src/mlkit/facedetection/index.d.ts b/src/mlkit/facedetection/index.d.ts
@@ -2,10 +2,10 @@ import { MLKitOptions } from "../";
 import { MLKitCameraView, MLKitResult } from "../index";
 
 export interface MLKitDetectFacesResultFace {
-  // TODO there are more properties we can return, see https://firebase.google.com/docs/ml-kit/android/detect-faces
   smilingProbability?: number;
   leftEyeOpenProbability?: number;
   rightEyeOpenProbability?: number;
+  trackingId?: number;
   ios?: any;
   android?: any;
 }
@@ -14,8 +14,33 @@ export interface MLKitDetectFacesOnDeviceResult extends MLKitResult {
   faces: Array<MLKitDetectFacesResultFace>;
 }
 
+export type MLKitFaceDetectionMode = "fast" | "accurate";
+
 export interface MLKitDetectFacesOnDeviceOptions extends MLKitOptions {
-  // TODO there are a few options here
+  /**
+   * Favor speed or accuracy when detecting faces.
+   * Default "fast".
+   */
+  detectionMode?: MLKitFaceDetectionMode;
+
+  /**
+   * Whether or not to attempt to identify facial "landmarks": eyes, ears, nose, cheeks, mouth.
+   * Default "none".
+   *
+  // detectLandmarks?: "none" | "all";
+  */
+
+  /**
+   * The minimum size, relative to the image, of faces to detect.
+   * Default 0.1.
+   */
+  minimumFaceSize?: number;
+
+  /**
+   * Whether or not to assign faces an ID, which can be used to track faces across images.
+   * Default false.
+   */
+  enableFaceTracking?: boolean;
 }
 
 export declare function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): Promise<MLKitDetectFacesOnDeviceResult>;
diff --git a/src/mlkit/facedetection/index.ios.ts b/src/mlkit/facedetection/index.ios.ts
@@ -2,11 +2,16 @@ import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitDetectFacesOnDeviceOptions, MLKitDetectFacesOnDeviceResult } from "./";
 import { MLKitOptions } from "../index";
 import { MLKitFaceDetection as MLKitFaceDetectionBase } from "./facedetection-common";
+import { ios as iosUtils } from "tns-core-modules/utils/utils";
 
 export class MLKitFaceDetection extends MLKitFaceDetectionBase {
 
   protected createDetector(): any {
-    return getDetector();
+    return getDetector({
+      detectionMode: this.detectionMode,
+      enableFaceTracking: this.enableFaceTracking,
+      minimumFaceSize: this.minimumFaceSize
+    });
   }
 
   protected createSuccessListener(): any {
@@ -25,7 +30,8 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
           result.faces.push({
             smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
             leftEyeOpenProbability: face.hasLeftEyeOpenProbability ? face.leftEyeOpenProbability : undefined,
-            rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined
+            rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined,
+            trackingId: face.hasTrackingID ? face.trackingID : undefined
           });
         }
 
@@ -42,25 +48,32 @@ export class MLKitFaceDetection extends MLKitFaceDetectionBase {
   protected rotateRecording(): boolean {
     return false;
   }
+
+  getVisionOrientation(imageOrientation: UIImageOrientation): FIRVisionDetectorImageOrientation {
+    if (imageOrientation === UIImageOrientation.Up && !iosUtils.isLandscape()) {
+      return FIRVisionDetectorImageOrientation.RightTop;
+    } else {
+      return super.getVisionOrientation(imageOrientation);
+    }
+  }
 }
 
-function getDetector(): FIRVisionFaceDetector {
-  // TODO make configurable (see #704)
+function getDetector(options: MLKitDetectFacesOnDeviceOptions): FIRVisionFaceDetector {
   const firVision: FIRVision = FIRVision.vision();
-  const options = FIRVisionFaceDetectorOptions.new();
-  options.modeType = FIRVisionFaceDetectorMode.Accurate;
-  options.landmarkType = FIRVisionFaceDetectorLandmark.All;
-  options.classificationType = FIRVisionFaceDetectorClassification.All;
-  options.minFaceSize = 0.1;
-  // options.isTrackingEnabled = true;
-  return firVision.faceDetectorWithOptions(options);
+  const firOptions = FIRVisionFaceDetectorOptions.new();
+  firOptions.modeType = options.detectionMode === "accurate" ? FIRVisionFaceDetectorMode.Accurate : FIRVisionFaceDetectorMode.Fast;
+  // firOptions.landmarkType = FIRVisionFaceDetectorLandmark.All; // TODO
+  // firOptions.classificationType = FIRVisionFaceDetectorClassification.All; // TODO
+  firOptions.minFaceSize = options.minimumFaceSize;
+  firOptions.isTrackingEnabled = options.enableFaceTracking === true;
+  return firVision.faceDetectorWithOptions(firOptions);
 }
 
 // TODO somehow this function doesn't work.. probably because of the passed image, but I can't find the cause.. the live camera version works great tho
 export function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): Promise<MLKitDetectFacesOnDeviceResult> {
   return new Promise((resolve, reject) => {
     try {
-      const faceDetector = getDetector();
+      const faceDetector = getDetector(options);
       faceDetector.detectInImageCompletion(getImage(options), (faces: NSArray<FIRVisionFace>, error: NSError) => {
         if (error !== null) {
           reject(error.localizedDescription);
@@ -75,7 +88,8 @@ export function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): P
             result.faces.push({
               smilingProbability: face.hasSmilingProbability ? face.smilingProbability : undefined,
               leftEyeOpenProbability: face.hasLeftEyeOpenProbability ? face.leftEyeOpenProbability : undefined,
-              rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined
+              rightEyeOpenProbability: face.hasRightEyeOpenProbability ? face.rightEyeOpenProbability : undefined,
+              trackingId: face.hasTrackingID ? face.trackingID : undefined
             });
           }
           resolve(result);
@@ -88,19 +102,7 @@ export function detectFacesOnDevice(options: MLKitDetectFacesOnDeviceOptions): P
   });
 }
 
-// TODO resize the image (here), like the example app?
 function getImage(options: MLKitOptions): FIRVisionImage {
   const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
-  console.log(">> image.imageOrientation: " + image.imageOrientation);
-
-  const fIRVisionImageMetadata = FIRVisionImageMetadata.new();
-  // fIRVisionImageMetadata.orientation = FIRVisionDetectorImageOrientation.TopLeft;
-
-  // const randomOrientation = Math.floor(Math.random() * 8) + 1;
-  // console.log(">>> randomOrientation: " + randomOrientation);
-  fIRVisionImageMetadata.orientation = 1;
-
-  const fIRVisionImage = FIRVisionImage.alloc().initWithImage(image);
-  fIRVisionImage.metadata = fIRVisionImageMetadata;
-  return fIRVisionImage;
+  return FIRVisionImage.alloc().initWithImage(image);
 }
diff --git a/src/mlkit/index.ts b/src/mlkit/index.ts
@@ -8,7 +8,7 @@ import { ImageSource } from "tns-core-modules/image-source";
 import { Image } from "tns-core-modules/ui/image";
 
 export interface MLKitOptions {
-  image: Image | ImageSource;
+  image?: Image | ImageSource;
 }
 
 export type MLKitCloudModelType = "stable" | "latest";
diff --git a/src/mlkit/mlkit-cameraview.d.ts b/src/mlkit/mlkit-cameraview.d.ts
@@ -6,5 +6,7 @@ export declare abstract class MLKitCameraView extends MLKitCameraViewBase {
   protected abstract createDetector(): any;
 
   protected abstract createSuccessListener(): any;
+
+  getVisionOrientation(imageOrientation: any): any;
 }
 
diff --git a/src/mlkit/mlkit-cameraview.ios.ts b/src/mlkit/mlkit-cameraview.ios.ts

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ import { ImageSource } from "tns-core-modules/image-source";`
`8`	`8`	`import { Image } from "tns-core-modules/ui/image";`
`9`	`9`
`10`	`10`	`export interface MLKitOptions {`
`11`		`- image: Image \| ImageSource;`
	`11`	`+ image?: Image \| ImageSource;`
`12`	`12`	`}`
`13`	`13`
`14`	`14`	`export type MLKitCloudModelType = "stable" \| "latest";`
Original file line number	Diff line number	Diff line change
`@@ -6,5 +6,7 @@ export declare abstract class MLKitCameraView extends MLKitCameraViewBase {`
`6`	`6`	`protected abstract createDetector(): any;`
`7`	`7`
`8`	`8`	`protected abstract createSuccessListener(): any;`
	`9`	`+`
	`10`	`+ getVisionOrientation(imageOrientation: any): any;`
`9`	`11`	`}`
`10`	`12`