#699 Add ML Kit support (cloud text recognition)

Author: EddyVerbruggen

demo-ng/app/tabs/mlkit/mlkit.component.html

@@ -1,4 +1,4 @@
-<GridLayout rows="auto, auto, *" class="tab-content">
+<GridLayout rows="auto, auto, *, auto" class="tab-content">
 
   <Label row="0" text="Pick the image processing source:" textWrap="true"></Label>
 
@@ -8,5 +8,7 @@
     <Button text="Cam feed" (tap)="fromCameraFeed()" class="button"></Button>
   </StackLayout>
 
-  <Image row="2" [src]="pickedImage" width="240" horizontalAlignment="center"></Image>
+  <Image row="2" [src]="pickedImage" width="240" horizontalAlignment="center" (tap)="reusePickedImage()"></Image>
+
+  <Label row="3" text="⬆️ Tap the image to use it again" textWrap="true" horizontalAlignment="center" *ngIf="pickedImage"></Label>
 </GridLayout>

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -2,7 +2,7 @@ import { Component, NgZone } from "@angular/core";
 import { ImageSource } from "tns-core-modules/image-source";
 
 import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
-import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
+import { MLKitRecognizeTextLocalResult, MLKitRecognizeTextCloudResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
 import { MLKitLandmarkRecognitionResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
 import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
 import { action } from "tns-core-modules/ui/dialogs";
@@ -24,24 +24,31 @@ export class MLKitComponent {
 
   pickedImage: ImageSource;
 
-  // TODO once more ML plugin features support cloud, add those as (cloud) options to this list
   private mlkitFeatures: Array<string> = [
-    "Text recognition",
+    "Text recognition (local)",
+    "Text recognition (cloud)",
     "Barcode scanning",
     "Face detection",
     "Image labeling",
     "Landmark recognition (cloud)"
   ];
 
+  private mlkitLocalFeatures: Array<string> = [
+    "Text recognition",
+    "Barcode scanning",
+    "Face detection",
+    "Image labeling"
+  ];
+
   constructor(private routerExtensions: RouterExtensions,
               private zone: NgZone) {
   }
 
   fromCameraFeed(): void {
     action(
-        "Test which ML Kit feature?",
+        "Test which ML Kit feature? No cloud processing will be used.",
         "Cancel",
-        this.mlkitFeatures
+        this.mlkitLocalFeatures
     ).then((pickedItem: string) => {
       let to;
       if (pickedItem === "Text recognition") {
@@ -52,14 +59,8 @@ export class MLKitComponent {
         to = "/tabs/mlkit/facedetection";
       } else if (pickedItem === "Image labeling") {
         to = "/tabs/mlkit/imagelabeling";
-      } else if (pickedItem === "Landmark recognition (cloud)") {
-        alert({
-          title: `Not available`,
-          message: `Landmark recognition is currently cloud-only, so that would be a bit too taxing on your dataplan.`,
-          okButtonText: "Gotcha!"
-        });
-        return;
       }
+
       if (to !== undefined) {
         this.routerExtensions.navigate([to],
             {
@@ -135,15 +136,21 @@ export class MLKitComponent {
         });
   }
 
-  selectMLKitFeature(imageSource: ImageSource): void {
+  reusePickedImage(): void {
+    this.selectMLKitFeature(this.pickedImage);
+  }
+
+  private selectMLKitFeature(imageSource: ImageSource): void {
     action(
         "Use which ML Kit feature?",
         "Cancel",
         this.mlkitFeatures
     ).then((pickedItem: string) => {
       let pickedItemIndex = this.mlkitFeatures.indexOf(pickedItem);
-      if (pickedItem === "Text recognition") {
-        this.recognizeText(imageSource);
+      if (pickedItem === "Text recognition (local)") {
+        this.recognizeTextLocal(imageSource);
+      } else if (pickedItem === "Text recognition (cloud)") {
+        this.recognizeTextCloud(imageSource);
       } else if (pickedItem === "Barcode scanning") {
         this.scanBarcode(imageSource);
       } else if (pickedItem === "Face detection") {
@@ -156,11 +163,11 @@ export class MLKitComponent {
     });
   }
 
-  private recognizeText(imageSource: ImageSource): void {
-    firebase.mlkit.textrecognition.recognizeText({
+  private recognizeTextLocal(imageSource: ImageSource): void {
+    firebase.mlkit.textrecognition.recognizeTextLocal({
       image: imageSource
     }).then(
-        (result: MLKitRecognizeTextResult) => {
+        (result: MLKitRecognizeTextLocalResult) => {
           alert({
             title: `Result`,
             message: result.features.map(feature => feature.text).join(""),
@@ -170,6 +177,22 @@ export class MLKitComponent {
         .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
+  private recognizeTextCloud(imageSource: ImageSource): void {
+    firebase.mlkit.textrecognition.recognizeTextCloud({
+      image: imageSource,
+      modelType: "latest",
+      maxResults: 15
+    }).then(
+        (result: MLKitRecognizeTextCloudResult) => {
+          alert({
+            title: `Result`,
+            message: result.text,
+            okButtonText: "OK"
+          });
+        })
+        .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+  }
+
   private recognizeLandmark(imageSource: ImageSource): void {
     firebase.mlkit.landmarkrecognition.recognizeLandmark({
       image: imageSource

demo-ng/app/tabs/mlkit/textrecognition/textrecognition.component.ts

@@ -1,5 +1,5 @@
 import { Component } from "@angular/core";
-import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
+import { MLKitRecognizeTextLocalResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
 
 @Component({
   selector: "mlkit-textrecognition",
@@ -12,7 +12,7 @@ export class TextRecognitionComponent {
   }>;
 
   onTextRecognitionResult(scanResult: any): void {
-    const value: MLKitRecognizeTextResult = scanResult.value;
+    const value: MLKitRecognizeTextLocalResult = scanResult.value;
     this.features = value.features;
   }
 }

demo-ng/package.json

@@ -25,7 +25,7 @@
     "nativescript-angular": "~5.3.0",
     "nativescript-camera": "^4.0.2",
     "nativescript-imagepicker": "^6.0.1",
-    "nativescript-plugin-firebase": "^5.0.0",
+    "nativescript-plugin-firebase": "^6.0.0",
     "nativescript-theme-core": "~1.0.4",
     "reflect-metadata": "~0.1.10",
     "rxjs": "~5.5.0",

src/mlkit/imagelabeling/index.d.ts

@@ -1,4 +1,4 @@
-import { MLKitMultiEngineOptions } from "../";
+import { MLKitOptions } from "../";
 import { MLKitCameraView, MLKitResult } from "../index";
 
 export interface MLKitImageLabelingResult extends MLKitResult {
@@ -9,7 +9,7 @@ export interface MLKitImageLabelingResult extends MLKitResult {
   }>;
 }
 
-export interface MLKitImageLabelingOptions extends MLKitMultiEngineOptions {
+export interface MLKitImageLabelingOptions extends MLKitOptions {
   /**
    * 0.5 by default
    */

src/mlkit/index.ts

@@ -11,8 +11,15 @@ export interface MLKitOptions {
   image: Image | ImageSource;
 }
 
-export interface MLKitMultiEngineOptions extends MLKitOptions {
-  engine?: "device" | "cloud"; // defaults to local, as cloud requires an additional plan
+export interface MLKitCloudOptions extends MLKitOptions {
+  /**
+   * Defaults to "stable".
+   */
+  modelType?: "stable" | "latest";
+  /**
+   * Defaults to 10.
+   */
+  maxResults?: number
 }
 
 export interface MLKitResult {
@@ -26,8 +33,8 @@ export interface MLKitResult {
 export declare class MLKitCameraView {
 }
 
-export declare class MLKitBarcodeScanner {
-}
+// export declare class MLKitBarcodeScanner {
+// }
 
 export {
   textrecognition,

src/mlkit/textrecognition/index.android.ts

@@ -1,8 +1,12 @@
 import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitOptions, } from "../";
-import { MLKitRecognizeTextOptions, MLKitRecognizeTextResult } from "./";
+import { MLKitRecognizeTextLocalOptions, MLKitRecognizeTextLocalResult } from "./";
 import { MLKitTextRecognition as MLKitTextRecognitionBase } from "./textrecognition-common";
-import { MLKitRecognizeTextResultFeature } from "./index";
+import {
+  MLKitRecognizeTextCloudOptions,
+  MLKitRecognizeTextCloudResult,
+  MLKitRecognizeTextResultFeature
+} from "./index";
 
 declare const com: any;
 
@@ -19,16 +23,16 @@ export class MLKitTextRecognition extends MLKitTextRecognitionBase {
           this.notify({
             eventName: MLKitTextRecognition.scanResultEvent,
             object: this,
-            value: getResult(textBlocks.getBlocks())
+            value: getLocalResult(textBlocks.getBlocks())
           });
         }
       }
     });
   }
 }
 
-function getResult(blocks: any): MLKitRecognizeTextResult {
-  const result = <MLKitRecognizeTextResult>{
+function getLocalResult(blocks: any): MLKitRecognizeTextLocalResult {
+  const result = <MLKitRecognizeTextLocalResult>{
     features: []
   };
 
@@ -68,14 +72,14 @@ function getResult(blocks: any): MLKitRecognizeTextResult {
   return result;
 }
 
-export function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKitRecognizeTextResult> {
+export function recognizeTextLocal(options: MLKitRecognizeTextLocalOptions): Promise<MLKitRecognizeTextLocalResult> {
   return new Promise((resolve, reject) => {
     try {
       const firebaseVisionTextDetector = com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionTextDetector();
 
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: textBlocks => {
-          resolve(getResult(textBlocks.getBlocks()));
+          resolve(getLocalResult(textBlocks.getBlocks()));
           firebaseVisionTextDetector.close();
         }
       });
@@ -90,13 +94,48 @@ export function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKit
           .addOnFailureListener(onFailureListener);
 
     } catch (ex) {
-      console.log("Error in firebase.mlkit.recognizeText: " + ex);
+      console.log("Error in firebase.mlkit.recognizeTextLocal: " + ex);
+      reject(ex);
+    }
+  });
+}
+
+export function recognizeTextCloud(options: MLKitRecognizeTextCloudOptions): Promise<MLKitRecognizeTextCloudResult> {
+  return new Promise((resolve, reject) => {
+    try {
+      const cloudDetectorOptions =
+          new com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.Builder()
+              .setModelType(options.modelType === "latest" ? com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.LATEST_MODEL : com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.STABLE_MODEL)
+              .setMaxResults(options.maxResults || 10)
+              .build();
+
+      const firebaseVisionCloudTextDetector = com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionCloudTextDetector(cloudDetectorOptions);
+
+      const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
+        onSuccess: firebaseVisionCloudText => {
+          resolve({
+            text: firebaseVisionCloudText.getText()
+          });
+          firebaseVisionCloudTextDetector.close();
+        }
+      });
+
+      const onFailureListener = new com.google.android.gms.tasks.OnFailureListener({
+        onFailure: exception => reject(exception.getMessage())
+      });
+
+      firebaseVisionCloudTextDetector
+          .detectInImage(getImage(options))
+          .addOnSuccessListener(onSuccessListener)
+          .addOnFailureListener(onFailureListener);
+
+    } catch (ex) {
+      console.log("Error in firebase.mlkit.recognizeTextCloud: " + ex);
       reject(ex);
     }
   });
 }
 
-// TODO move
 function getImage(options: MLKitOptions): any /* com.google.firebase.ml.vision.common.FirebaseVisionImage */ {
   const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
   return com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);

src/mlkit/textrecognition/index.d.ts

@@ -1,5 +1,4 @@
-import { MLKitMultiEngineOptions } from "../";
-import { MLKitCameraView, MLKitResult } from "../index";
+import { MLKitCameraView, MLKitCloudOptions, MLKitOptions, MLKitResult } from "../index";
 
 export interface MLKitRecognizeTextResultFeature {
   text: string;
@@ -18,13 +17,23 @@ export interface MLKitRecognizeTextResultFeature {
   }>
 }
 
-export interface MLKitRecognizeTextResult extends MLKitResult {
+export interface MLKitRecognizeTextLocalResult extends MLKitResult {
   features: Array<MLKitRecognizeTextResultFeature>;
 }
 
-export interface MLKitRecognizeTextOptions extends MLKitMultiEngineOptions {
+export interface MLKitRecognizeTextCloudResult extends MLKitResult {
+  text: string;
+}
+
+export interface MLKitRecognizeTextLocalOptions extends MLKitOptions {
 }
 
-export declare function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKitRecognizeTextResult>;
+export interface MLKitRecognizeTextCloudOptions extends MLKitCloudOptions {
+}
 
-export declare class MLKitTextRecognition extends MLKitCameraView {}
+export declare function recognizeTextLocal(options: MLKitRecognizeTextLocalOptions): Promise<MLKitRecognizeTextLocalResult>;
+
+export declare function recognizeTextCloud(options: MLKitRecognizeTextCloudOptions): Promise<MLKitRecognizeTextCloudResult>;
+
+export declare class MLKitTextRecognition extends MLKitCameraView {
+}