Skip to content

Commit 57a91db

Browse files
yiyujinpearmini
yiyujin
and
pearmini
authored
Add legacy object detection cocossd code and example from p5js (#257)
* add legacy object detection cocossd code and example from p5js * add code to build objectDetector, change example code callback to return result not error * load model in preload() ; updated ObjectDetector Class to return object (instance.model), added mediaReady utils so that detection waits for video to be ready before detecting, updated tf.backend to webgl to avoid webgpu warning * added detectStart and detectStop to cocossd.js and updated example code to call detectStart in setup() * refactor ObjectDetector Class to be main interface for models * update objectDetection-webcam example * add examples : single image, video(resizing canvas to video and showing transparent canvas on top of video) * update video example - play video on loop and detect * update video file example to work with a fixed sized video 640 x 480 * remove reference to yolo for now * change video file name * change video file name in code * reduced video file size * Remove useless comments and clean up ObjectDetector documentation Removed unused typedefs for options and predictions. * Update comment to reflect objectDetector usage * Update example to use ml5.objectDetector --------- Co-authored-by: Bairui Su <[email protected]>
1 parent 3616de0 commit 57a91db

File tree

13 files changed

+438
-1
lines changed

13 files changed

+438
-1
lines changed

‎examples/objectDetection-image/dog_cat.jpg‎

323 KB
Loading

‎examples/objectDetection-image/index.html‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!--
2+
👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
Learn more about the ml5.js project: https://ml5js.org/
4+
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
6+
This example demonstrates object detection on a single image through ml5.objectDetector.
7+
-->
8+
9+
<!DOCTYPE html>
10+
<html lang="en">
11+
<head>
12+
<meta charset="UTF-8" />
13+
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
14+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
15+
<title>ml5.js objectDetector Example - Single Image</title>
16+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script>
17+
<script src="../../dist/ml5.js"></script>
18+
</head>
19+
<body>
20+
<main>
21+
</main>
22+
<script src="sketch.js"></script>
23+
</body>
24+
</html>

‎examples/objectDetection-image/sketch.js‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
* Learn more about the ml5.js project: https://ml5js.org/
4+
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
*
6+
* This example demonstrates detecting objects in a live video through ml5.objectDetector.
7+
*/
8+
9+
let img;
10+
let detector;
11+
let detections = [];
12+
13+
function preload(){
14+
detector = ml5.objectDetector("cocossd");
15+
img = loadImage('dog_cat.jpg');
16+
}
17+
18+
function setup() {
19+
createCanvas(640, 480);
20+
image(img, 0, 0);
21+
detector.detectStart(img, gotDetections);
22+
}
23+
24+
// Callback function is called each time the object detector finishes processing a frame.
25+
function gotDetections(results) {
26+
// Update detections array with the new results
27+
detections = results;
28+
29+
for (let i = 0; i < detections.length; i += 1) {
30+
let detection = detections[i];
31+
32+
// Draw bounding box
33+
stroke(0, 255, 0);
34+
strokeWeight(4);
35+
noFill();
36+
rect(detection.x, detection.y, detection.width, detection.height);
37+
38+
// Draw label
39+
noStroke();
40+
fill(255);
41+
textSize(24);
42+
text(detection.label, detection.x + 10, detection.y + 24);
43+
}
44+
}

‎examples/objectDetection-video/ball_lifting.mp4‎

1.29 MB
Binary file not shown.

‎examples/objectDetection-video/index.html‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!--
2+
👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
Learn more about the ml5.js project: https://ml5js.org/
4+
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
6+
This example demonstrates object detection on a video through ml5.objectDetector.
7+
-->
8+
9+
<!DOCTYPE html>
10+
<html lang="en">
11+
<head>
12+
<meta charset="UTF-8" />
13+
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
14+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
15+
<title>ml5.js objectDetector Example - Video</title>
16+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script>
17+
<script src="../../dist/ml5.js"></script>
18+
</head>
19+
<body>
20+
<main>
21+
</main>
22+
<script src="sketch.js"></script>
23+
</body>
24+
</html>

‎examples/objectDetection-video/sketch.js‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
* Learn more about the ml5.js project: https://ml5js.org
4+
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
*
6+
* This example demonstrates detecting objects in a video file through ml5.objectDetector.
7+
*/
8+
9+
let video;
10+
let detector;
11+
let detections = [];
12+
13+
function preload(){
14+
detector = ml5.objectDetector("cocossd");
15+
}
16+
17+
// Callback function is called each time the object detector finishes processing a frame.
18+
function gotDetections(results) {
19+
// Update detections array with the new results
20+
detections = results;
21+
}
22+
23+
function setup() {
24+
createCanvas(640, 480);
25+
26+
// Load and loop the video for object detection
27+
video = createVideo('ball_lifting.mp4'); // video sized 640 x 480
28+
video.hide();
29+
video.loop();
30+
31+
detector.detectStart(video, gotDetections);
32+
}
33+
34+
function draw(){
35+
image(video, 0, 0); // draw video frame
36+
37+
for (let i = 0; i < detections.length; i++) {
38+
let detection = detections[i];
39+
40+
let x = detection.x;
41+
let y = detection.y;
42+
let w = detection.width;
43+
let h = detection.height;
44+
45+
stroke(0, 255, 0);
46+
strokeWeight(4);
47+
noFill();
48+
rect(x, y, w, h);
49+
50+
noStroke();
51+
fill(255);
52+
textSize(18);
53+
text(detection.label, x + 5, y + 20);
54+
}
55+
}

‎examples/objectDetection-webcam/index.html‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!--
2+
👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
Learn more about the ml5.js project: https://ml5js.org/
4+
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
6+
This example demonstrates object detection on live video through ml5.objectDetector.
7+
-->
8+
9+
<!DOCTYPE html>
10+
<html lang="en">
11+
<head>
12+
<meta charset="UTF-8" />
13+
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
14+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
15+
<title>ml5.js objectDetector Example - Webcam</title>
16+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script>
17+
<script src="../../dist/ml5.js"></script>
18+
</head>
19+
<body>
20+
<main>
21+
</main>
22+
<script src="sketch.js"></script>
23+
</body>
24+
</html>

‎examples/objectDetection-webcam/sketch.js‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
3+
* Learn more about the ml5.js project: https://ml5js.org/
4+
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
5+
*
6+
* This example demonstrates detecting objects in a live video through ml5.objectDetector.
7+
*/
8+
9+
let video;
10+
let detector;
11+
let detections = [];
12+
13+
function preload(){
14+
detector = ml5.objectDetector("cocossd");
15+
}
16+
17+
function setup() {
18+
createCanvas(640, 480);
19+
20+
// Using webcam feed as video input, hiding html element to avoid duplicate with canvas
21+
video = createCapture(VIDEO);
22+
video.size(width, height);
23+
video.hide();
24+
25+
detector.detectStart(video, gotDetections);
26+
}
27+
28+
// Callback function is called each time the object detector finishes processing a frame.
29+
function gotDetections(results) {
30+
// Update detections array with the new results
31+
detections = results;
32+
}
33+
34+
function draw() {
35+
// Draw the current video frame onto the canvas.
36+
image(video, 0, 0);
37+
38+
for (let i = 0; i < detections.length; i += 1) {
39+
let detection = detections[i];
40+
41+
// Draw bounding box
42+
stroke(0, 255, 0);
43+
strokeWeight(4);
44+
noFill();
45+
rect(detection.x, detection.y, detection.width, detection.height);
46+
47+
// Draw label
48+
noStroke();
49+
fill(255);
50+
textSize(24);
51+
text(detection.label, detection.x + 10, detection.y + 24);
52+
}
53+
}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
"@mediapipe/pose": "^0.5.1675469404",
5858
"@mediapipe/selfie_segmentation": "~0.1.0",
5959
"@tensorflow-models/body-segmentation": "^1.0.1",
60+
"@tensorflow-models/coco-ssd": "^2.2.3",
6061
"@tensorflow-models/depth-estimation": "^0.0.4",
6162
"@tensorflow-models/face-landmarks-detection": "1.0.5",
6263
"@tensorflow-models/hand-pose-detection": "^2.0.0",

‎src/ObjectDetector/cocossd.js‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) 2019 ml5
2+
//
3+
// This software is released under the MIT License.
4+
// https://opensource.org/licenses/MIT
5+
6+
/*
7+
COCO-SSD Object detection model
8+
Wraps the coco-ssd model in tfjs to be used in ml5
9+
*/
10+
import * as tf from "@tensorflow/tfjs";
11+
import * as cocoSsd from "@tensorflow-models/coco-ssd";
12+
import { mediaReady } from "../utils/imageUtilities";
13+
14+
const DEFAULTS = {
15+
base: "lite_mobilenet_v2",
16+
modelUrl: undefined,
17+
};
18+
19+
export class CocoSsd {
20+
constructor(options = {}) {
21+
this.model = null;
22+
this.config = {
23+
base: options.base || DEFAULTS.base,
24+
modelUrl: options.modelUrl || DEFAULTS.modelUrl,
25+
};
26+
}
27+
28+
async load() {
29+
await tf.setBackend("webgl"); // this line resolves warning : performance is poor on webgpu backend
30+
await tf.ready();
31+
32+
this.model = await cocoSsd.load(this.config);
33+
return this;
34+
}
35+
36+
/**
37+
* Detect objects that are in the image/video/canvas
38+
* @param {HTMLVideoElement|HTMLImageElement|HTMLCanvasElement|ImageData} imgToPredict - Subject of the detection.
39+
* @returns {Array} Array of detection detections
40+
*/
41+
async detect(imgToPredict) {
42+
mediaReady(imgToPredict, true);
43+
44+
await tf.nextFrame();
45+
46+
const detections = await this.model.detect(imgToPredict);
47+
const formattedDetections = detections.map(prediction => {
48+
return {
49+
label: prediction.class,
50+
confidence: prediction.score,
51+
x: prediction.bbox[0],
52+
y: prediction.bbox[1],
53+
width: prediction.bbox[2],
54+
height: prediction.bbox[3],
55+
normalized: {
56+
x: prediction.bbox[0] / imgToPredict.width,
57+
y: prediction.bbox[1] / imgToPredict.height,
58+
width: prediction.bbox[2] / imgToPredict.width,
59+
height: prediction.bbox[3] / imgToPredict.height,
60+
},
61+
};
62+
});
63+
64+
return formattedDetections;
65+
}
66+
}
67+
68+
export async function load(modelConfig = {}) {
69+
const cocoSsdInstance = new CocoSsd(modelConfig);
70+
await cocoSsdInstance.load();
71+
return cocoSsdInstance;
72+
}

0 commit comments

Comments
 (0)