load model in preload() ; updated ObjectDetector Class to return object (instance.model), added mediaReady utils so that detection waits for video to be ready before detecting, updated tf.backend to webgl to avoid webgpu warning

Author: yiyujin

examples/objectDetection/index.html

@@ -4,7 +4,6 @@
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script>
     <!-- <script src="https://unpkg.com/[email protected]/dist/ml5.min.js"></script> -->
     <script src="../../dist/ml5.js"></script>
-    <link rel="stylesheet" type="text/css" href="style.css">
     <meta charset="utf-8" />
 
   </head>

examples/objectDetection/sketch.js

@@ -13,35 +13,38 @@ let video;
 let detector;
 let detections = [];
 
+function preload(){
+  detector = ml5.objectDetector("cocossd");
+}
+
 function setup() {
   createCanvas(640, 480);
-  video = createCapture(VIDEO, videoReady);
+
+  video = createCapture(VIDEO);
   video.size(640, 480);
   video.hide();
-}
 
-function videoReady() {
-  detector = ml5.objectDetector('cocossd', modelReady);
+  detector.detect(video, gotDetections);
 }
 
 function gotDetections(results) {
   detections = results;
   detector.detect(video, gotDetections);
 }
 
-function modelReady() {
-  detector.detect(video, gotDetections);
-}
-
 function draw() {
   image(video, 0, 0);
 
   for (let i = 0; i < detections.length; i += 1) {
     const object = detections[i];
+
+    // draw bounding box
     stroke(0, 255, 0);
     strokeWeight(4);
     noFill();
     rect(object.x, object.y, object.width, object.height);
+
+    // draw label
     noStroke();
     fill(255);
     textSize(24);

src/ObjectDetector/cocossd.js

@@ -11,6 +11,7 @@ import * as tf from "@tensorflow/tfjs";
 import * as cocoSsd from "@tensorflow-models/coco-ssd";
 import callCallback from "../utils/callcallback";
 import handleArguments from "../utils/handleArguments";
+import { mediaReady } from "../utils/imageUtilities";
 
 const DEFAULTS = {
   base: "lite_mobilenet_v2",
@@ -40,6 +41,7 @@ export class CocoSsdBase {
    * load model
    */
   async loadModel() {
+    await tf.setBackend("webgl"); // this line resolves warning : performance is poor on webgpu backend
     await tf.ready();
 
     this.model = await cocoSsd.load(this.config);
@@ -73,6 +75,9 @@ export class CocoSsdBase {
    */
   async detectInternal(imgToPredict) {
     this.isPredicting = true;
+    mediaReady(imgToPredict, true)
+    await tf.nextFrame();
+
     const predictions = await this.model.detect(imgToPredict);
     const formattedPredictions = predictions.map(prediction => {
       return {
@@ -108,6 +113,8 @@ export class CocoSsdBase {
     const args = handleArguments(this.video, inputOrCallback, cb);
     args.require("image", "Detection subject not supported");
 
+    await mediaReady(args.image, true);
+
     return callCallback(this.detectInternal(args.image), args.callback);
   }
 }

src/ObjectDetector/index.js

@@ -7,8 +7,6 @@
   ObjectDetection
 */
 
-console.log("running ObjectDetection/index.js");
-
 import handleArguments from "../utils/handleArguments";
 import {CocoSsd} from "./cocossd.js";
 
@@ -27,7 +25,7 @@ class ObjectDetector {
    * @param {function} callback - Optional. A callback function that is called once the model has loaded.
    */
   constructor(modelNameOrUrl, video, options, callback) {
-    this.video = video;
+    this.video = video || null;
     this.modelNameOrUrl = modelNameOrUrl;
     this.options = options || {};
     this.callback = callback;
@@ -56,9 +54,8 @@ const objectDetector = (...inputs) => {
 
   const instance = new ObjectDetector(model, video, options, callback);
 
-  console.log("model", instance.model, instance.model.ready);
-
-  return instance.model.callback ? instance.model : instance.model.ready;
+  // return instance.model.callback ? instance.model : instance.model.ready;
+  return instance.model;
 };
 
 export default objectDetector;