#699 Add ML Kit support (Add a 'bounds' property so you know where in the image text was found)

EddyVerbruggen · EddyVerbruggen · commit 5d9133d918de · 2018-05-15T16:03:18.000+02:00
diff --git a/demo-ng/app/tabs/mlkit/mlkit.component.ts b/demo-ng/app/tabs/mlkit/mlkit.component.ts
@@ -24,6 +24,7 @@ export class MLKitComponent {
 
   pickedImage: ImageSource;
 
+  // TODO once more ML plugin features support cloud, add those as (cloud) options to this list
   private mlkitFeatures: Array<string> = [
     "Text recognition",
     "Barcode scanning",
@@ -113,7 +114,11 @@ export class MLKitComponent {
               return;
             }
             if (!image) {
-              console.log(`This is probably an iCloud image - which won't work`);
+              alert({
+                title: `Invalid image`,
+                message: `Perhaps this is an image from cloud storage?`,
+                okButtonText: "Hmm.."
+              });
               return;
             }
             const imageSource = new ImageSource();
@@ -158,7 +163,7 @@ export class MLKitComponent {
         (result: MLKitRecognizeTextResult) => {
           alert({
             title: `Result`,
-            message: JSON.stringify(result.features),
+            message: result.features.map(feature => feature.text).join(""),
             okButtonText: "OK"
           });
         })
diff --git a/publish/publish.sh b/publish/publish.sh
@@ -5,7 +5,8 @@ PACK_DIR=package;
 publish() {
     cd $PACK_DIR
     echo 'Publishing to npm...'
+#    npm publish *.tgz --tag next
     npm publish *.tgz
 }
 
-./pack.sh && publish
+./pack.sh && publish
diff --git a/src/mlkit/textrecognition/index.android.ts b/src/mlkit/textrecognition/index.android.ts
@@ -2,6 +2,7 @@ import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitOptions, } from "../";
 import { MLKitRecognizeTextOptions, MLKitRecognizeTextResult } from "./";
 import { MLKitTextRecognition as MLKitTextRecognitionBase } from "./textrecognition-common";
+import { MLKitRecognizeTextResultFeature } from "./index";
 
 declare const com: any;
 
@@ -14,58 +15,67 @@ export class MLKitTextRecognition extends MLKitTextRecognitionBase {
   protected createSuccessListener(): any {
     return new com.google.android.gms.tasks.OnSuccessListener({
       onSuccess: textBlocks => {
-
-        if (textBlocks.getBlocks().size() === 0) return;
-
-        const blocks = textBlocks.getBlocks();
-
-        // const imageSource = new ImageSource();
-        // imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
-
-        const result = <MLKitRecognizeTextResult>{
-          features: []
-        };
-
-        // see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
-        for (let i = 0; i < blocks.size(); i++) {
-          const textBlock = blocks.get(i);
-          result.features.push({
-            text: textBlock.getText()
+        if (textBlocks.getBlocks().size() > 0) {
+          this.notify({
+            eventName: MLKitTextRecognition.scanResultEvent,
+            object: this,
+            value: getResult(textBlocks.getBlocks())
           });
         }
-
-        this.notify({
-          eventName: MLKitTextRecognition.scanResultEvent,
-          object: this,
-          value: result
-        });
       }
     });
   }
 }
 
+function getResult(blocks: any): MLKitRecognizeTextResult {
+  const result = <MLKitRecognizeTextResult>{
+    features: []
+  };
+
+  // see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
+  for (let i = 0; i < blocks.size(); i++) {
+    const textBlock = blocks.get(i);
+    const blockResult = <MLKitRecognizeTextResultFeature>{
+      text: textBlock.getText(),
+      elements: []
+    };
+
+    const lines = textBlock.getLines();
+    for (let j = 0; j < lines.size(); j++) {
+      const elements = lines.get(j).getElements();
+      for (let k = 0; k < elements.size(); k++) {
+        const element = elements.get(k);
+        const rect = element.getBoundingBox();
+        const blockElement = {
+          text: element.getText(),
+          bounds: {
+            origin: {
+              x: rect.left,
+              y: rect.top
+            },
+            size: {
+              width: rect.width(),
+              height: rect.height()
+            }
+          }
+        };
+        blockResult.elements.push(blockElement);
+      }
+    }
+
+    result.features.push(blockResult);
+  }
+  return result;
+}
+
 export function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKitRecognizeTextResult> {
   return new Promise((resolve, reject) => {
     try {
       const firebaseVisionTextDetector = com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionTextDetector();
 
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: textBlocks => {
-          const blocks = textBlocks.getBlocks();
-
-          const result = <MLKitRecognizeTextResult>{
-            features: []
-          };
-
-          // see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
-          for (let i = 0; i < blocks.size(); i++) {
-            const textBlock = blocks.get(i);
-            result.features.push({
-              text: textBlock.getText()
-            });
-          }
-
-          resolve(result);
+          resolve(getResult(textBlocks.getBlocks()));
           firebaseVisionTextDetector.close();
         }
       });
diff --git a/src/mlkit/textrecognition/index.d.ts b/src/mlkit/textrecognition/index.d.ts
@@ -1,11 +1,25 @@
 import { MLKitMultiEngineOptions } from "../";
 import { MLKitCameraView, MLKitResult } from "../index";
 
-export interface MLKitRecognizeTextResult extends MLKitResult {
-  features: Array<{
+export interface MLKitRecognizeTextResultFeature {
+  text: string;
+  elements: Array<{
     text: string;
-    // corners: any;
-  }>;
+    bounds: {
+      origin: {
+        x: number;
+        y: number;
+      },
+      size: {
+        width: number;
+        height: number;
+      }
+    };
+  }>
+}
+
+export interface MLKitRecognizeTextResult extends MLKitResult {
+  features: Array<MLKitRecognizeTextResultFeature>;
 }
 
 export interface MLKitRecognizeTextOptions extends MLKitMultiEngineOptions {
diff --git a/src/mlkit/textrecognition/index.ios.ts b/src/mlkit/textrecognition/index.ios.ts
@@ -2,6 +2,7 @@ import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitOptions } from "../";
 import { MLKitRecognizeTextOptions, MLKitRecognizeTextResult } from "./";
 import { MLKitTextRecognition as MLKitTextRecognitionBase } from "./textrecognition-common";
+import { MLKitRecognizeTextResultFeature } from "./index";
 
 export class MLKitTextRecognition extends MLKitTextRecognitionBase {
   protected createDetector(): any {
@@ -13,46 +14,14 @@ export class MLKitTextRecognition extends MLKitTextRecognitionBase {
     return (features: NSArray<FIRVisionText>, error: NSError) => {
       if (error !== null) {
         console.log(error.localizedDescription);
-
       } else if (features !== null && features.count > 0) {
-        const result = <MLKitRecognizeTextResult>{
-          features: []
-        };
-
-        for (let i = 0, l = features.count; i < l; i++) {
-          const feature: FIRVisionText = features.objectAtIndex(i);
-          // Note that fetching these details works, but there's currently no added value
-          /*
-            if (feature instanceof FIRVisionTextBlock) {
-              const textBlock = <FIRVisionTextBlock>feature;
-              for (let j = 0, k = textBlock.lines.count; j < k; j++) {
-                const textBlockLine: FIRVisionTextLine = textBlock.lines.objectAtIndex(j);
-                for (let a = 0, m = textBlockLine.elements.count; a < m; a++) {
-                  const element: FIRVisionTextElement = textBlockLine.elements.objectAtIndex(a);
-                  console.log("FIRVisionTextBlock text: " + element.text);
-                }
-              }
-            }
-            if (feature instanceof FIRVisionTextLine) {
-              for (let a = 0, m = feature.elements.count; a < m; a++) {
-                const element: FIRVisionTextElement = feature.elements.objectAtIndex(a);
-                console.log("FIRVisionTextLine text: " + element.text);
-              }
-            }
-          */
-          result.features.push({
-            text: feature.text,
-            // corners: this.getCorners(<any>feature.cornerPoints)
-          });
-        }
-
         this.notify({
           eventName: MLKitTextRecognition.scanResultEvent,
           object: this,
-          value: result
+          value: getResult(features)
         });
       }
-    }
+    };
   }
 
   protected rotateRecording(): boolean {
@@ -70,6 +39,46 @@ export class MLKitTextRecognition extends MLKitTextRecognitionBase {
   */
 }
 
+function getResult(features: NSArray<FIRVisionText>): MLKitRecognizeTextResult {
+  const result = <MLKitRecognizeTextResult>{
+    features: []
+  };
+
+  for (let i = 0, l = features.count; i < l; i++) {
+    const feature = features.objectAtIndex(i);
+    const resultFeature = <MLKitRecognizeTextResultFeature>{
+      text: feature.text,
+      elements: []
+    };
+
+    const addLineToResult = (line: FIRVisionTextLine): void => {
+      for (let a = 0, m = line.elements.count; a < m; a++) {
+        const element: FIRVisionTextElement = line.elements.objectAtIndex(a);
+        const bounds = element.frame;
+        resultFeature.elements.push({
+          text: element.text,
+          bounds: bounds,
+        });
+      }
+    };
+
+    if (feature instanceof FIRVisionTextBlock) {
+      const textBlock = <FIRVisionTextBlock>feature;
+      for (let j = 0, k = textBlock.lines.count; j < k; j++) {
+        addLineToResult(textBlock.lines.objectAtIndex(j));
+      }
+    }
+
+    if (feature instanceof FIRVisionTextLine) {
+      addLineToResult(feature);
+    }
+
+    console.log(">>> resulting resultFeature: " + JSON.stringify(resultFeature));
+    result.features.push(resultFeature);
+  }
+  return result;
+}
+
 export function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKitRecognizeTextResult> {
   return new Promise((resolve, reject) => {
     try {
@@ -79,20 +88,8 @@ export function recognizeText(options: MLKitRecognizeTextOptions): Promise<MLKit
       textDetector.detectInImageCompletion(getImage(options), (features: NSArray<FIRVisionText>, error: NSError) => {
         if (error !== null) {
           reject(error.localizedDescription);
-
         } else if (features !== null) {
-          const result = <MLKitRecognizeTextResult>{
-            features: []
-          };
-
-          for (let i = 0, l = features.count; i < l; i++) {
-            const feature: FIRVisionText = features.objectAtIndex(i);
-            result.features.push({
-              text: feature.text,
-              // corners: <any>feature.cornerPoints
-            });
-          }
-          resolve(result);
+          resolve(getResult(features));
         }
       });
     } catch (ex) {
diff --git a/src/package.json b/src/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nativescript-plugin-firebase",
-  "version": "5.3.1",
+  "version": "6.0.0",
   "description": "Fire. Base. Firebase!",
   "main": "firebase",
   "typings": "index.d.ts",

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,8 @@ PACK_DIR=package;`
`5`	`5`	`publish() {`
`6`	`6`	`cd $PACK_DIR`
`7`	`7`	`echo 'Publishing to npm...'`
	`8`	`+# npm publish *.tgz --tag next`
`8`	`9`	`npm publish *.tgz`
`9`	`10`	`}`
`10`	`11`
`11`		`-./pack.sh && publish`
	`12`	`+./pack.sh && publish`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "nativescript-plugin-firebase",`
`3`		`- "version": "5.3.1",`
	`3`	`+ "version": "6.0.0",`
`4`	`4`	`"description": "Fire. Base. Firebase!",`
`5`	`5`	`"main": "firebase",`
`6`	`6`	`"typings": "index.d.ts",`