#699 Add ML Kit support (refactored landmark recognition so it's more clear it's always Cloud based)

Author: EddyVerbruggen

demo-ng/app/tabs/mlkit/mlkit.component.html

@@ -10,5 +10,6 @@
 
   <Image row="2" [src]="pickedImage" width="240" horizontalAlignment="center" (tap)="reusePickedImage()"></Image>
 
-  <Label row="3" text="⬆️ Tap the image to use it again" textWrap="true" horizontalAlignment="center" *ngIf="pickedImage"></Label>
+  <Label row="3" [text]="pickedImage ? '⬆️ Tap the image to use it again' : ''" textWrap="true" horizontalAlignment="center" class="m-b-20" width="100%"></Label>
+
 </GridLayout>

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -3,7 +3,7 @@ import { ImageSource } from "tns-core-modules/image-source";
 
 import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
 import { MLKitRecognizeTextLocalResult, MLKitRecognizeTextCloudResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
-import { MLKitLandmarkRecognitionResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
+import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
 import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
 import { action } from "tns-core-modules/ui/dialogs";
 import { ImageAsset } from "tns-core-modules/image-asset";
@@ -158,7 +158,7 @@ export class MLKitComponent {
       } else if (pickedItem === "Image labeling") {
         this.labelImage(imageSource);
       } else if (pickedItem === "Landmark recognition (cloud)") {
-        this.recognizeLandmark(imageSource);
+        this.recognizeLandmarkCloud(imageSource);
       }
     });
   }
@@ -193,11 +193,11 @@ export class MLKitComponent {
         .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
-  private recognizeLandmark(imageSource: ImageSource): void {
-    firebase.mlkit.landmarkrecognition.recognizeLandmark({
+  private recognizeLandmarkCloud(imageSource: ImageSource): void {
+    firebase.mlkit.landmarkrecognition.recognizeLandmarksCloud({
       image: imageSource
     }).then(
-        (result: MLKitLandmarkRecognitionResult) => {
+        (result: MLKitLandmarkRecognitionCloudResult) => {
           alert({
             title: `Result`,
             message: JSON.stringify(result.landmarks),

src/mlkit/index.ts

@@ -11,11 +11,13 @@ export interface MLKitOptions {
   image: Image | ImageSource;
 }
 
+export type MLKitCloudModelType = "stable" | "latest";
+
 export interface MLKitCloudOptions extends MLKitOptions {
   /**
    * Defaults to "stable".
    */
-  modelType?: "stable" | "latest";
+  modelType?: MLKitCloudModelType;
   /**
    * Defaults to 10.
    */

src/mlkit/landmarkrecognition/index.android.ts

@@ -1,65 +1,28 @@
 import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitOptions, } from "../";
-import { MLKitLandmarkRecognitionOptions, MLKitLandmarkRecognitionResult } from "./";
-import { MLKitLandmarkRecognition as MLKitLandmarkRecognitionBase } from "./landmarkrecognition-common";
+import { MLKitLandmarkRecognitionCloudOptions, MLKitLandmarkRecognitionCloudResult } from "./index";
+import { MLKitCloudModelType } from "../index";
 
 declare const com: any;
 
-export class MLKitLandmarkRecognition extends MLKitLandmarkRecognitionBase {
-
-  protected createDetector(): any {
-    return getDetector(this.maxResults);
-  }
-
-  protected createSuccessListener(): any {
-    return new com.google.android.gms.tasks.OnSuccessListener({
-      onSuccess: landmarks => {
-
-        if (landmarks.size() === 0) return;
-
-        // const imageSource = new ImageSource();
-        // imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
-
-        const result = <MLKitLandmarkRecognitionResult>{
-          // imageSource: imageSource,
-          landmarks: []
-        };
-
-        for (let i = 0; i < landmarks.size(); i++) {
-          const landmark = landmarks.get(i);
-          result.landmarks.push({
-            name: landmark.getLandmark(),
-            confidence: landmark.getConfidence()
-          });
-        }
-
-        this.notify({
-          eventName: MLKitLandmarkRecognition.scanResultEvent,
-          object: this,
-          value: result
-        });
-      }
-    });
-  }
-}
-
-function getDetector(confidenceThreshold: number): any {
+function getDetector(modelType: MLKitCloudModelType, confidenceThreshold: number): any {
   const landmarkDetectorOptions =
       new com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.Builder()
-          .setMaxResults(confidenceThreshold)
+          .setModelType(modelType === "latest" ? com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.LATEST_MODEL : com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.STABLE_MODEL)
+          .setMaxResults(confidenceThreshold || 10)
           .build();
 
   return com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionCloudLandmarkDetector(landmarkDetectorOptions);
 }
 
-export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult> {
+export function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult> {
   return new Promise((resolve, reject) => {
     try {
-      const firebaseVisionLandmarkDetector = getDetector(options.maxResults || 10);
+      const firebaseVisionLandmarkDetector = getDetector(options.modelType, options.maxResults);
 
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: landmarks => {
-          const result = <MLKitLandmarkRecognitionResult>{
+          const result = <MLKitLandmarkRecognitionCloudResult>{
             landmarks: []
           };
 
@@ -86,7 +49,7 @@ export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Pro
           .addOnFailureListener(onFailureListener);
 
     } catch (ex) {
-      console.log("Error in firebase.mlkit.recognizeLandmark: " + ex);
+      console.log("Error in firebase.mlkit.recognizeLandmarksCloud: " + ex);
       reject(ex);
     }
   });

src/mlkit/landmarkrecognition/index.d.ts

@@ -1,22 +1,14 @@
-import { MLKitCameraView, MLKitOptions, MLKitResult } from "../index";
+import { MLKitCloudOptions, MLKitResult } from "../index";
 
-export interface MLKitLandmarkRecognitionResult extends MLKitResult {
+export interface MLKitLandmarkRecognitionCloudResult extends MLKitResult {
   // TODO add locations
   landmarks: Array<{
     name: string;
     confidence: number;
   }>;
 }
 
-export interface MLKitLandmarkRecognitionOptions extends MLKitOptions {
-  /**
-   * 10 by default
-   */
-  maxResults?: number;
+export interface MLKitLandmarkRecognitionCloudOptions extends MLKitCloudOptions {
 }
 
-export declare function recognizeLandmarks(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult>;
-
-// TODO not sure this works.. but we need quite a processing threshold as this is always in the cloud..
-// .. and what about other features.. the camera should prolly always use a local model
-export declare class MLKitLandmarkRecognition extends MLKitCameraView {}
+export declare function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult>;

src/mlkit/landmarkrecognition/index.ios.ts

@@ -1,66 +1,27 @@
 import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitOptions } from "../";
-import { MLKitLandmarkRecognitionOptions, MLKitLandmarkRecognitionResult } from "./";
-import { MLKitLandmarkRecognition as MLKitLandmarkRecognitionBase } from "./landmarkrecognition-common";
+import { MLKitLandmarkRecognitionCloudOptions, MLKitLandmarkRecognitionCloudResult } from "./";
+import { MLKitCloudModelType } from "../index";
 
-export class MLKitLandmarkRecognition extends MLKitLandmarkRecognitionBase {
-
-  protected createDetector(): any {
-    return getDetector(this.maxResults);
-  }
-
-  protected createSuccessListener(): any {
-    return (landmarks: NSArray<FIRVisionCloudLandmark>, error: NSError) => {
-      if (error !== null) {
-        console.log(error.localizedDescription);
-
-      } else if (landmarks !== null && landmarks.count > 0) {
-        const result = <MLKitLandmarkRecognitionResult>{
-          landmarks: []
-        };
-
-        for (let i = 0, l = landmarks.count; i < l; i++) {
-          const landmark: FIRVisionCloudLandmark = landmarks.objectAtIndex(i);
-          console.log(">> detected landmark: " + landmark);
-          result.landmarks.push({
-            name: landmark.landmark,
-            confidence: landmark.confidence
-          });
-        }
-
-        console.log(">>> notify " + MLKitLandmarkRecognition.scanResultEvent + " with " + JSON.stringify(result));
-        this.notify({
-          eventName: MLKitLandmarkRecognition.scanResultEvent,
-          object: this,
-          value: result
-        });
-      }
-    }
-  }
-
-  protected rotateRecording(): boolean {
-    return false;
-  }
-}
-
-function getDetector(maxResults: number): FIRVisionCloudLandmarkDetector {
+function getDetector(modelType: MLKitCloudModelType, maxResults: number): FIRVisionCloudLandmarkDetector {
   const firVision: FIRVision = FIRVision.vision();
   const fIRVisionCloudDetectorOptions = FIRVisionCloudDetectorOptions.alloc();
-  fIRVisionCloudDetectorOptions.maxResults = maxResults;
+  fIRVisionCloudDetectorOptions.modelType = modelType === "latest" ? FIRVisionCloudModelType.Latest : FIRVisionCloudModelType.Stable;
+  fIRVisionCloudDetectorOptions.maxResults = maxResults || 10;
   return firVision.cloudLandmarkDetectorWithOptions(fIRVisionCloudDetectorOptions);
 }
 
-export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult> {
+export function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult> {
   return new Promise((resolve, reject) => {
     try {
-      const landmarkDetector = getDetector(options.maxResults || 10);
+      const landmarkDetector = getDetector(options.modelType, options.maxResults);
 
       landmarkDetector.detectInImageCompletion(getImage(options), (landmarks: NSArray<FIRVisionCloudLandmark>, error: NSError) => {
         if (error !== null) {
           reject(error.localizedDescription);
 
         } else if (landmarks !== null) {
-          const result = <MLKitLandmarkRecognitionResult>{
+          const result = <MLKitLandmarkRecognitionCloudResult>{
             landmarks: []
           };
 
@@ -77,7 +38,7 @@ export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Pro
         }
       });
     } catch (ex) {
-      console.log("Error in firebase.mlkit.recognizeLandmark: " + ex);
+      console.log("Error in firebase.mlkit.recognizeLandmarksCloud: " + ex);
       reject(ex);
     }
   });