HandPose connections (#226)

ziyuan-linn · web-flow · commit 308b786f0f0f · 2024-10-31T22:06:38.000-04:00
* implement getConnections

* add example

* rename example
diff --git a/examples/handPose-skeletal-connections/index.html b/examples/handPose-skeletal-connections/index.html
@@ -0,0 +1,22 @@
+<!--
+  👋 Hello! This is an ml5.js example made and shared with ❤️.
+  Learn more about the ml5.js project: https://ml5js.org/
+  ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+
+  This example demonstrates drawing skeletal connections through ml5.handPose.
+-->
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ml5.js handPose Skeletal Connection Example</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
+    <script src="../../dist/ml5.js"></script>
+  </head>
+  <body>
+    <script src="sketch.js"></script>
+  </body>
+</html>
diff --git a/examples/handPose-skeletal-connections/sketch.js b/examples/handPose-skeletal-connections/sketch.js
@@ -0,0 +1,65 @@
+/*
+ * 👋 Hello! This is an ml5.js example made and shared with ❤️.
+ * Learn more about the ml5.js project: https://ml5js.org/
+ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+ *
+ * This example demonstrates drawing skeletal connections through ml5.handPose.
+ */
+
+let handPose;
+let video;
+let hands = [];
+let connections;
+
+function preload() {
+  // Load the handPose model
+  handPose = ml5.handPose();
+}
+
+function setup() {
+  createCanvas(640, 480);
+  // Create the webcam video and hide it
+  video = createCapture(VIDEO);
+  video.size(640, 480);
+  video.hide();
+  // start detecting hands from the webcam video
+  handPose.detectStart(video, gotHands);
+  // Get the skeletal connection information
+  connections = handPose.getConnections();
+}
+
+function draw() {
+  // Draw the webcam video
+  image(video, 0, 0, width, height);
+
+  // Draw the skeletal connections
+  for (let i = 0; i < hands.length; i++) {
+    let hand = hands[i];
+    for (let j = 0; j < connections.length; j++) {
+      let pointAIndex = connections[j][0];
+      let pointBIndex = connections[j][1];
+      let pointA = hand.keypoints[pointAIndex];
+      let pointB = hand.keypoints[pointBIndex];
+      stroke(255, 0, 0);
+      strokeWeight(2);
+      line(pointA.x, pointA.y, pointB.x, pointB.y);
+    }
+  }
+
+  // Draw all the tracked hand points
+  for (let i = 0; i < hands.length; i++) {
+    let hand = hands[i];
+    for (let j = 0; j < hand.keypoints.length; j++) {
+      let keypoint = hand.keypoints[j];
+      fill(0, 255, 0);
+      noStroke();
+      circle(keypoint.x, keypoint.y, 10);
+    }
+  }
+}
+
+// Callback function for when handPose outputs data
+function gotHands(results) {
+  // save the output to the hands variable
+  hands = results;
+}
diff --git a/src/HandPose/index.js b/src/HandPose/index.js
@@ -20,6 +20,7 @@
 
 import * as tf from "@tensorflow/tfjs";
 import * as handPoseDetection from "@tensorflow-models/hand-pose-detection";
+import { MEDIAPIPE_CONNECTED_KEYPOINTS_PAIRS } from "@tensorflow-models/hand-pose-detection/dist/constants";
 import callCallback from "../utils/callcallback";
 import handleArguments from "../utils/handleArguments";
 import handleOptions from "../utils/handleOptions";
@@ -292,6 +293,15 @@ class HandPose {
       return hand;
     });
   }
+
+  /**
+   * Returns the pairs of keypoint indices that are connected by a skeletal representation.
+   * @returns {number[][]} The connected keypoint pairs.
+   * @public
+   */
+  getConnections() {
+    return MEDIAPIPE_CONNECTED_KEYPOINTS_PAIRS;
+  }
 }
 
 /**
