AutoMLAdd support for MLKit's AutoML #1500

Author: EddyVerbruggen

.travis.yml

@@ -42,8 +42,8 @@ android:
     - sys-img-armeabi-v7a-android-21
 
 before_install:
-#  - sudo pip install --upgrade pip
-#  - sudo pip install six
+  - sudo pip install --upgrade pip
+  - sudo pip install six
 
 install:
   - echo no | npm install -g nativescript

demo-ng/app/App_Resources/Android/src/main/assets/leftright/dict.txt

@@ -0,0 +1,2 @@
+Left
+Right

demo-ng/app/App_Resources/Android/src/main/assets/leftright/manifest.json

@@ -0,0 +1,5 @@
+{
+  "modelFile": "model.tflite",
+  "labelsFile": "dict.txt",
+  "modelType": "IMAGE_LABELING"
+}

demo-ng/app/App_Resources/Android/src/main/assets/leftright/model.tflite

@@ -8,6 +8,7 @@
       width="100%"
       height="100%"
       opacity="0.8"
+      localModelResourceFolder="leftright"
       processEveryNthFrame="60"
       confidenceThreshold="0.4"
       (scanResult)="onAutoMLResult($event)">

demo-ng/app/tabs/mlkit/automl/automl.component.html

@@ -459,6 +459,7 @@ export class MLKitComponent {
   private autoML(imageSource: ImageSource): void {
     firebase.mlkit.automl.labelImage({
       image: imageSource,
+      localModelResourceFolder: "leftright",
       confidenceThreshold: 0.3
     }).then(
         (result: MLKitAutoMLResult) => {

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -90,6 +90,7 @@ To be able to use Cloud features you need to do two things:
 |[Natural language identification](#natural-language-identification)|✅|
 |[Translate text](#translate-text)|✅|
 |[Smart reply](#smart-reply)|✅|
+|[AutoML Vision Edge](#automl-vision-edge)|✅|✅
 |[Custom model inference](#custom-model-inference)|✅|✅
 
 ### Text recognition
@@ -513,6 +514,46 @@ firebase.mlkit.smartreply.suggestReplies({
 .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
 ```
 
+### AutoML Vision Edge
+<img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_automl.png" height="153px" alt="ML Kit - AutoML Vision Edge"/>
+
+[Firebase documentation 🌎](https://firebase.google.com/docs/ml-kit/automl-image-labeling)
+
+> NOTE: currently only local models are supported (not cloud models), but it's fairly easy to add those so open an issue if you need it. See the demo-ng folder for an example.
+
+#### Still image (on-device)
+
+```typescript
+import { MLKitAutoMLResult } from "nativescript-plugin-firebase/mlkit/automl";
+const firebase = require("nativescript-plugin-firebase");
+
+firebase.mlkit.automl.labelImage({
+  localModelResourceFolder: "leftright", 
+  image: imageSource,
+  confidenceThreshold: 0.6 // this will only return labels with at least 0.6 (60%) confidence. Default 0.5.
+})
+.then((result: MLKitAutoMLResult) => console.log(JSON.stringify(result.labels)))
+.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+```
+
+#### Live camera feed
+The basics are explained above for 'Text recognition', so we're only showing the differences here.
+
+```typescript
+import { registerElement } from "nativescript-angular/element-registry";
+registerElement("MLKitAutoML", () => require("nativescript-plugin-firebase/mlkit/automl").MLKitAutoML);
+```
+
+```html
+<MLKitAutoML
+    width="260"
+    height="380"
+    localModelResourceFolder="leftright"
+    confidenceThreshold="0.6"
+    (scanResult)="onAutoMLResult($event)">
+</MLKitImageLabeling>
+```
+
 ### Custom model inference
 <img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_custom_model_tflite.png" height="153px" alt="ML Kit - Custom Model (TensorFlow Lite)"/>
 

docs/ML_KIT.md

@@ -1,6 +1,11 @@
 import { Property } from "tns-core-modules/ui/core/properties";
 import { MLKitCameraView } from "../mlkit-cameraview";
 
+export const localModelResourceFolderProperty = new Property<MLKitAutoML, string>({
+  name: "localModelResourceFolder",
+  defaultValue: null,
+});
+
 export const confidenceThresholdProperty = new Property<MLKitAutoML, number>({
   name: "confidenceThreshold",
   defaultValue: 0.5,
@@ -9,11 +14,18 @@ export const confidenceThresholdProperty = new Property<MLKitAutoML, number>({
 export abstract class MLKitAutoML extends MLKitCameraView {
   static scanResultEvent: string = "scanResult";
 
+  protected localModelResourceFolder: string;
   protected confidenceThreshold: number;
 
+
+  [localModelResourceFolderProperty.setNative](value: string) {
+    this.localModelResourceFolder = value;
+  }
+
   [confidenceThresholdProperty.setNative](value: any) {
     this.confidenceThreshold = parseFloat(value);
   }
 }
 
+localModelResourceFolderProperty.register(MLKitAutoML);
 confidenceThresholdProperty.register(MLKitAutoML);

docs/images/features/mlkit_automl.png

@@ -8,7 +8,7 @@ declare const com: any;
 export class MLKitAutoML extends MLKitAutoMLBase {
 
   protected createDetector(): any {
-    return getDetector(this.confidenceThreshold);
+    return getDetector(this.localModelResourceFolder, this.confidenceThreshold);
   }
 
   protected createSuccessListener(): any {
@@ -44,19 +44,26 @@ export class MLKitAutoML extends MLKitAutoMLBase {
   }
 }
 
-function getDetector(confidenceThreshold: number): com.google.firebase.ml.vision.label.FirebaseVisionImageLabeler {
+function getDetector(localModelResourceFolder: string, confidenceThreshold: number): com.google.firebase.ml.vision.label.FirebaseVisionImageLabeler {
+  // TODO also support cloud hosted models
+  const model = new com.google.firebase.ml.vision.automl.FirebaseAutoMLLocalModel.Builder()
+      .setAssetFilePath(localModelResourceFolder + "/manifest.json") // TODO this..
+      // .setFilePath() // .. or this
+      .build();
+
   const labelDetectorOptions =
-      new com.google.firebase.ml.vision.label.FirebaseVisionOnDeviceAutoMLImageLabelerOptions.Builder()
+      new com.google.firebase.ml.vision.label.FirebaseVisionOnDeviceAutoMLImageLabelerOptions.Builder(model)
           .setConfidenceThreshold(confidenceThreshold)
           .build();
 
-  return com.google.firebase.ml.vision.FirebaseVision.getInstance().getOnDeviceAutoMLImageLabeler(labelDetectorOptions);
+  return com.google.firebase.ml.vision.FirebaseVision.getInstance()
+      .getOnDeviceAutoMLImageLabeler(labelDetectorOptions);
 }
 
 export function labelImage(options: MLKitAutoMLOptions): Promise<MLKitAutoMLResult> {
   return new Promise((resolve, reject) => {
     try {
-      const firebaseVisionAutoMLImageLabeler = getDetector(options.confidenceThreshold || 0.5);
+      const firebaseVisionAutoMLImageLabeler = getDetector(options.localModelResourceFolder, options.confidenceThreshold || 0.5);
 
       const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
         onSuccess: labels => {