add legacy object detection cocossd code and example from p5js #257
There are no files selected for viewing
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| <!-- | ||
| 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| Learn more about the ml5.js project: https://ml5js.org/ | ||
| ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
|
|
||
| This example demonstrates object detection on a single image through ml5.objectDetector. | ||
| --> | ||
|
|
||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js objectDetector Example - Single Image</title> | ||
| <script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
| <body> | ||
| <main> | ||
| </main> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* | ||
| * 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| * Learn more about the ml5.js project: https://ml5js.org/ | ||
| * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
| * | ||
| * This example demonstrates detecting objects in a live video through ml5.imageClassifier. | ||
| */ | ||
|
|
||
| let img; | ||
| let detector; | ||
| let detections = []; | ||
|
|
||
| function preload(){ | ||
| detector = ml5.objectDetector("cocossd"); | ||
| img = loadImage('dog_cat.jpg'); | ||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
| image(img, 0, 0); | ||
| detector.detectStart(img, gotDetections); | ||
| } | ||
|
|
||
| // Callback function is called each time the object detector finishes processing a frame. | ||
| function gotDetections(results) { | ||
| // Update detections array with the new results | ||
| detections = results; | ||
|
|
||
| for (let i = 0; i < detections.length; i += 1) { | ||
| let detection = detections[i]; | ||
|
|
||
| // Draw bounding box | ||
| stroke(0, 255, 0); | ||
| strokeWeight(4); | ||
| noFill(); | ||
| rect(detection.x, detection.y, detection.width, detection.height); | ||
|
|
||
| // Draw label | ||
| noStroke(); | ||
| fill(255); | ||
| textSize(24); | ||
| text(detection.label, detection.x + 10, detection.y + 24); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| <!-- | ||
| 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| Learn more about the ml5.js project: https://ml5js.org/ | ||
| ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
|
|
||
| This example demonstrates object detection on a video through ml5.objectDetector. | ||
| --> | ||
|
|
||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js objectDetector Example - Video</title> | ||
| <script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
| <body> | ||
| <main> | ||
| </main> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| /* | ||
| * 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| * Learn more about the ml5.js project: https://ml5js.org | ||
| * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
| * | ||
| * This example demonstrates detecting objects in a video file through ml5.objectDetector. | ||
| */ | ||
|
|
||
| let video; | ||
| let detector; | ||
| let detections = []; | ||
|
|
||
| function preload(){ | ||
| detector = ml5.objectDetector("cocossd"); | ||
| } | ||
|
|
||
| // Callback function is called each time the object detector finishes processing a frame. | ||
| function gotDetections(results) { | ||
| // Update detections array with the new results | ||
| detections = results; | ||
| } | ||
|
|
||
| function setup() { | ||
| // Create canvas with initial size - will be resized later | ||
| createCanvas(640, 480); | ||
|
|
||
| // Load and loop the video for object detection | ||
| video = createVideo('test.mov'); | ||
| video.size(width, height); | ||
| video.hide(); | ||
| video.loop(); | ||
|
|
||
| detector.detectStart(video, gotDetections); | ||
| } | ||
|
|
||
| function draw(){ | ||
| image(video, 0, 0, width, height); // draw video frame | ||
|
|
||
| // Scale factors from original video to canvas size | ||
| let scaleX = width / video.elt.videoWidth; | ||
| let scaleY = height / video.elt.videoHeight; | ||
|
|
||
| for (let i = 0; i < detections.length; i++) { | ||
| let detection = detections[i]; | ||
|
|
||
| let x = detection.x * scaleX; | ||
| let y = detection.y * scaleY; | ||
| let w = detection.width * scaleX; | ||
| let h = detection.height * scaleY; | ||
|
||
|
|
||
| stroke(0, 255, 0); | ||
| strokeWeight(4); | ||
| noFill(); | ||
| rect(x, y, w, h); | ||
|
|
||
| noStroke(); | ||
| fill(255); | ||
| textSize(18); | ||
| text(detection.label, x + 5, y + 20); | ||
| } | ||
| } | ||
Binary file not shown.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| <!-- | ||
| 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| Learn more about the ml5.js project: https://ml5js.org/ | ||
| ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
|
|
||
| This example demonstrates object detection on live video through ml5.objectDetector. | ||
| --> | ||
|
|
||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js objectDetector Example - Webcam</title> | ||
| <script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/p5.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
| <body> | ||
| <main> | ||
| </main> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| /* | ||
| * 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| * Learn more about the ml5.js project: https://ml5js.org/ | ||
| * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
| * | ||
| * This example demonstrates detecting objects in a live video through ml5.imageClassifier. | ||
| */ | ||
|
|
||
| let video; | ||
| let detector; | ||
| let detections = []; | ||
|
|
||
| function preload(){ | ||
| detector = ml5.objectDetector("cocossd"); | ||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
|
|
||
| // Using webcam feed as video input, hiding html element to avoid duplicate with canvas | ||
| video = createCapture(VIDEO); | ||
| video.size(width, height); | ||
|
Comment on lines
+18
to
+22
There was a problem hiding this comment. Ah, I just tested and confirm that the #264 applies here too! If I change the webcam to |
||
| video.hide(); | ||
|
|
||
| detector.detectStart(video, gotDetections); | ||
| } | ||
|
|
||
| // Callback function is called each time the object detector finishes processing a frame. | ||
| function gotDetections(results) { | ||
| // Update detections array with the new results | ||
| detections = results; | ||
| } | ||
|
|
||
| function draw() { | ||
| // Draw the current video frame onto the canvas. | ||
| image(video, 0, 0); | ||
|
|
||
| for (let i = 0; i < detections.length; i += 1) { | ||
| let detection = detections[i]; | ||
|
|
||
| // Draw bounding box | ||
| stroke(0, 255, 0); | ||
| strokeWeight(4); | ||
| noFill(); | ||
| rect(detection.x, detection.y, detection.width, detection.height); | ||
|
|
||
| // Draw label | ||
| noStroke(); | ||
| fill(255); | ||
| textSize(24); | ||
| text(detection.label, detection.x + 10, detection.y + 24); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| // Copyright (c) 2019 ml5 | ||
| // | ||
| // This software is released under the MIT License. | ||
| // https://opensource.org/licenses/MIT | ||
|
|
||
| /* | ||
| COCO-SSD Object detection model | ||
| Wraps the coco-ssd model in tfjs to be used in ml5 | ||
| */ | ||
| import * as tf from "@tensorflow/tfjs"; | ||
| import * as cocoSsd from "@tensorflow-models/coco-ssd"; | ||
| import { mediaReady } from "../utils/imageUtilities"; | ||
|
|
||
| const DEFAULTS = { | ||
| base: "lite_mobilenet_v2", | ||
| modelUrl: undefined, | ||
| }; | ||
|
|
||
| export class CocoSsd { | ||
| constructor(options = {}) { | ||
| this.model = null; | ||
| this.config = { | ||
| base: options.base || DEFAULTS.base, | ||
| modelUrl: options.modelUrl || DEFAULTS.modelUrl, | ||
| }; | ||
| } | ||
|
|
||
| async load() { | ||
| await tf.setBackend("webgl"); // this line resolves warning : performance is poor on webgpu backend | ||
| await tf.ready(); | ||
|
|
||
| this.model = await cocoSsd.load(this.config); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Detect objects that are in the image/video/canvas | ||
| * @param {HTMLVideoElement|HTMLImageElement|HTMLCanvasElement|ImageData} imgToPredict - Subject of the detection. | ||
| * @returns {Array} Array of detection detections | ||
| */ | ||
| async detect(imgToPredict) { | ||
| mediaReady(imgToPredict, true); | ||
|
|
||
| await tf.nextFrame(); | ||
|
|
||
| const detections = await this.model.detect(imgToPredict); | ||
| const formattedDetections = detections.map(prediction => { | ||
| return { | ||
| label: prediction.class, | ||
| confidence: prediction.score, | ||
| x: prediction.bbox[0], | ||
| y: prediction.bbox[1], | ||
| width: prediction.bbox[2], | ||
| height: prediction.bbox[3], | ||
| normalized: { | ||
| x: prediction.bbox[0] / imgToPredict.width, | ||
| y: prediction.bbox[1] / imgToPredict.height, | ||
| width: prediction.bbox[2] / imgToPredict.width, | ||
| height: prediction.bbox[3] / imgToPredict.height, | ||
| }, | ||
| }; | ||
| }); | ||
|
|
||
| return formattedDetections; | ||
| } | ||
| } | ||
|
|
||
| export async function load(modelConfig = {}) { | ||
| const cocoSsdInstance = new CocoSsd(modelConfig); | ||
| await cocoSsdInstance.load(); | ||
| return cocoSsdInstance; | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just noting our discussion to use a video we have permission for. Also, I would suggest sizing the video to 640x480 so it doesn't appear squashed or skewed. Or just make the canvas size identical to the video. You can add a code comment that explains that if the video does not match the canvas size, you may need to adjust the code to scale.