Add support for ML Kit Object Detection and Tracking #1274

Author: EddyVerbruggen

docs/ML_KIT.md

@@ -81,6 +81,7 @@ To be able to use Cloud features you need to do two things:
 |---|---|---
 |[Text recognition](#text-recognition)|✅|✅
 |[Face detection](#face-detection)|✅|
+|[Object detection and tracking](#object-detection-and-tracking)|✅|
 |[Barcode scanning](#barcode-scanning)|✅|
 |[Image labeling](#image-labeling)|✅|✅
 |[Landmark recognition](#landmark-recognition)||✅
@@ -269,6 +270,46 @@ registerElement("MLKitFaceDetection", () => require("nativescript-plugin-firebas
 </MLKitFaceDetection>
 ```
 
+### Object detection and tracking
+<img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_object_detection.png" height="124px" alt="ML Kit - Object detection and tracking"/>
+
+[Firebase documentation 🌎](https://firebase.google.com/docs/ml-kit/object-detection)
+
+#### Still image
+
+```typescript
+import { MLKitObjectDetectionResult } from "nativescript-plugin-firebase/mlkit/objectdetection";
+const firebase = require("nativescript-plugin-firebase");
+
+firebase.mlkit.objectdetection.detectObjects({
+  image: imageSource, // a NativeScript Image or ImageSource, see the demo for examples
+  classify: true, // default false, attempts to classify the object(s) if true
+  multiple: true // default false, attempts to detect multiple objects (instead of only the most prominent one) when true
+})
+.then((result: MLKitObjectDetectionResult) => console.log(JSON.stringify(result.objects)))
+.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+```
+
+#### Live camera feed
+The basics are explained above for 'Text recognition', so we're only showing the differences here.
+
+```typescript
+import { registerElement } from "nativescript-angular/element-registry";
+registerElement("MLKitObjectDetection", () => require("nativescript-plugin-firebase/mlkit/objectdetection").MLKitObjectDetection);
+```
+
+```html
+<MLKitObjectDetection
+    width="260"
+    height="380"
+    classify="true"
+    multiple="false"
+    processEveryNthFrame="20"
+    [torchOn]="torchOn"
+    (scanResult)="onObjectDetectionResult($event)">
+</MLKitObjectDetection>
+```
+
 ### Barcode scanning
 <img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_text_barcode_scanning.png" height="153px" alt="ML Kit - Barcode scanning"/>
 

docs/images/features/mlkit_object_detection.png

@@ -1,5 +1,6 @@
 import { ImageSource } from "tns-core-modules/image-source";
 import { MLKitVisionOptions, } from "../";
+import { MLKitScanBarcodesResultBounds } from "../barcodescanning";
 import { MLKitObjectDetectionOptions, MLKitObjectDetectionResult, MLKitObjectDetectionResultItem } from "./";
 import { MLKitObjectDetection as MLKitObjectDetectionBase, ObjectDetectionCategory } from "./objectdetection-common";
 
@@ -22,9 +23,11 @@ export class MLKitObjectDetection extends MLKitObjectDetectionBase {
           objects: []
         };
 
+        const image: android.graphics.Bitmap = this.lastVisionImage && this.lastVisionImage.getBitmap ? this.lastVisionImage.getBitmap() : null;
+
         // see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
         for (let i = 0; i < objects.size(); i++) {
-          result.objects.push(getMLKitObjectDetectionResultItem(objects.get(i)));
+          result.objects.push(getMLKitObjectDetectionResultItem(objects.get(i), image));
         }
 
         this.notify({
@@ -57,6 +60,9 @@ export function detectObjects(options: MLKitObjectDetectionOptions): Promise<MLK
     try {
       const firebaseObjectDetector = getDetector(options.classify, options.multiple);
 
+      const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
+      const firImage = com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
+
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: objects => {
           const result = <MLKitObjectDetectionResult>{
@@ -65,7 +71,7 @@ export function detectObjects(options: MLKitObjectDetectionOptions): Promise<MLK
 
           if (objects) {
             for (let i = 0; i < objects.size(); i++) {
-              result.objects.push(getMLKitObjectDetectionResultItem(objects.get(i)));
+              result.objects.push(getMLKitObjectDetectionResultItem(objects.get(i), image));
             }
           }
 
@@ -79,7 +85,7 @@ export function detectObjects(options: MLKitObjectDetectionOptions): Promise<MLK
       });
 
       firebaseObjectDetector
-          .processImage(getImage(options))
+          .processImage(firImage)
           .addOnSuccessListener(onSuccessListener)
           .addOnFailureListener(onFailureListener);
 
@@ -90,18 +96,33 @@ export function detectObjects(options: MLKitObjectDetectionOptions): Promise<MLK
   });
 }
 
-function getMLKitObjectDetectionResultItem(obj: com.google.firebase.ml.vision.objects.FirebaseVisionObject): MLKitObjectDetectionResultItem {
+function getMLKitObjectDetectionResultItem(obj: com.google.firebase.ml.vision.objects.FirebaseVisionObject, image: android.graphics.Bitmap): MLKitObjectDetectionResultItem {
   return {
     id: obj.getTrackingId() ? obj.getTrackingId().intValue() : undefined,
     confidence: obj.getClassificationConfidence() ? obj.getClassificationConfidence().doubleValue() : undefined,
     category: ObjectDetectionCategory[obj.getClassificationCategory()],
-    // TODO
-    image: undefined,
-    bounds: undefined
+    bounds: boundingBoxToBounds(obj.getBoundingBox()),
+    image: !image ? null : {
+      width: image.getWidth(),
+      height: image.getHeight()
+    }
   };
 }
 
 function getImage(options: MLKitVisionOptions): any /* com.google.firebase.ml.vision.common.FirebaseVisionImage */ {
   const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
   return com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
 }
+
+function boundingBoxToBounds(rect: any): MLKitScanBarcodesResultBounds {
+  return {
+    origin: {
+      x: rect.left,
+      y: rect.top
+    },
+    size: {
+      width: rect.width(),
+      height: rect.height()
+    }
+  };
+}