Add custom (TensorFlow Lite) models support to the ML Kit feature #702

Author: EddyVerbruggen

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -1,21 +1,21 @@
 import { Component, NgZone } from "@angular/core";
 import { RouterExtensions } from "nativescript-angular";
-import { fromFile, ImageSource } from "tns-core-modules/image-source";
-import * as fileSystemModule from "tns-core-modules/file-system";
-import { action } from "tns-core-modules/ui/dialogs";
-import { ImageAsset } from "tns-core-modules/image-asset";
-import { isIOS } from "tns-core-modules/platform";
-import * as ImagePicker from "nativescript-imagepicker";
 import * as Camera from "nativescript-camera";
+import * as ImagePicker from "nativescript-imagepicker";
 import { BarcodeFormat, MLKitScanBarcodesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
-import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
+import { MLKitCustomModelResult } from "nativescript-plugin-firebase/mlkit/custommodel";
 import { MLKitDetectFacesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/facedetection";
-import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
 import {
   MLKitImageLabelingCloudResult,
   MLKitImageLabelingOnDeviceResult
 } from "nativescript-plugin-firebase/mlkit/imagelabeling";
-import { MLKitCustomModelResult } from "nativescript-plugin-firebase/mlkit/custommodel";
+import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
+import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
+import * as fileSystemModule from "tns-core-modules/file-system";
+import { ImageAsset } from "tns-core-modules/image-asset";
+import { fromFile, ImageSource } from "tns-core-modules/image-source";
+import { isIOS } from "tns-core-modules/platform";
+import { action } from "tns-core-modules/ui/dialogs";
 
 const firebase = require("nativescript-plugin-firebase");
 

docs/ML_KIT.md

@@ -84,7 +84,7 @@ To be able to use Cloud features you need to do two things:
 |[Barcode scanning](#barcode-scanning)|✅|
 |[Image labeling](#image-labeling)|✅|✅
 |[Landmark recognition](#landmark-recognition)||✅
-|[Custom model inference](#custom-model-inference)||
+|[Custom model inference](#custom-model-inference)|✅|✅
 
 ### Text recognition
 <img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_text_recognition.png" height="153px" alt="ML Kit - Text recognition"/>
@@ -363,6 +363,60 @@ firebase.mlkit.landmarkrecognition.recognizeLandmarksCloud({
 ```
 
 ### Custom model inference
+<img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_custom_model_tflit.png" height="153px" alt="ML Kit - Custom Model (TensorFlow Lite)"/>
+
 [Firebase documentation 🌎](https://firebase.google.com/docs/ml-kit/use-custom-models)
 
-Coming soon. See issue #702.
+⚠️ Please take note of the following:
+
+- Currently only models bundled with your app can be used (not ones hosted on Firebase). That may change in the future.
+- Prefix the `localModelFile` and `labelsFile` below with `~/` so they point to your `app/` folder. This is for future compatibility, because I'd like to support loading models from the native bundle as well.
+- On Android, make sure the model is not compressed by adding [your model's file extension to app.gradle](https://github.com/EddyVerbruggen/nativescript-plugin-firebase/blob/57969d0a62d761bffb98b19db85af88bfae858dd/demo-ng/app/App_Resources/Android/app.gradle#L22).
+- Only "Quantized" models can be used. Not "Float" models, so `modelInput.type` below must be set to `QUANT`.
+- The `modelInput.shape` parameter below must specify your model's dimensions. If you're not sure, use the script in the paragraph "Specify the model's input and output" at [the Firebase docs](https://firebase.google.com/docs/ml-kit/ios/use-custom-models).
+
+#### Still image (on-device)
+
+```typescript
+import { MLKitCustomModelResult } from "nativescript-plugin-firebase/mlkit/custommodel";
+const firebase = require("nativescript-plugin-firebase");
+
+firebase.mlkit.custommodel.useCustomModel({
+  image: imageSource, // a NativeScript Image or ImageSource, see the demo for examples
+  maxResults: 10, // default 5 (limit numbers to this amount of results)
+  localModelFile: "~/custommodel/inception/inception_v3_quant.tflite", // see the demo, where the model lives in app/custommodel/etc..
+  labelsFile: "~/custommodel/inception/inception_labels.txt",
+  modelInput: [{ // Array<TNSCustomModelInput>
+    shape: [1, 299, 299, 3], // see the tips above
+    type: "QUANT" // for now, must be "QUANT" (and you should use a 'quantized' model (not 'float'))
+  }]
+})
+.then((result: MLKitCustomModelResult) => console.log(JSON.stringify(result.result)))
+.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+```
+
+#### Live camera feed
+The basics are explained above for 'Text recognition'.
+
+Make sure to specify `modelInputShape` without the `[` and `]` characters.
+
+```typescript
+import { registerElement } from "nativescript-angular/element-registry";
+registerElement("MLKitCustomModel", () => require("nativescript-plugin-firebase/mlkit/custommodel").MLKitCustomModel);
+```
+
+```html
+<MLKitCustomModel
+    width="100%"
+    height="100%"
+    localModelFile="~/custommodel/inception/inception_v3_quant.tflite"
+    labelsFile="~/custommodel/inception/inception_labels.txt"
+    modelInputShape="1, 299, 299, 3"
+    modelInputType="QUANT"
+    processEveryNthFrame="30"
+    maxResults="5"
+    [torchOn]="torchOn"
+    (scanResult)="onCustomModelResult($event)">
+</MLKitCustomModel>
+```
+