ml5js · mop9047 · Apr 1, 2025 · Apr 5, 2025 · Apr 5, 2025 · May 7, 2025
diff --git a/examples/neuralNetwork-sequence-hand-gestures/index.html b/examples/neuralNetwork-sequence-hand-gestures/index.html
@@ -0,0 +1,42 @@
+<!--
+  👋 Hello! This is an ml5.js example made and shared with ❤️.
+  Learn more about the ml5.js project: https://ml5js.org/
+  ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+
+  This example demonstrates training a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task.
+-->
+
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ml5.js Neural Network Hand Gesture Train and Save</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
+    <script src="../../dist/ml5.js"></script>
+  </head>
+
+  <body>
+    <script src="sketch.js"></script>
+    <div id="canvasDiv"></div>
+
+    <p>
+      Instructions: <br />
+      1.) To add data, press the "Add Gesture #1 Data" button <br />
+      2.) Use one hand to do a gesture in front of the camera <br />
+      3.) Once done with the gesture, take your hand out of the frame<br />
+      4.) Repeat until there is a good amount of data<br />
+      5.) Do the same for gesture #2 by pressing the "Add Gesture #2 Data"
+      button before doing the hand gesture in the frame<br />
+      6.) Press the "Train and Save Model" button to train the model and save it
+      for future use.<br />
+    </p>
+  </body>
+
+  <style>
+    button {
+      font-family: Georgia, "Times New Roman", Times, serif;
+      font-size: 20px;
+    }
+  </style>
+</html>
diff --git a/examples/neuralNetwork-sequence-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-hand-gestures/sketch.js
@@ -0,0 +1,180 @@
+/*
+ * 👋 Hello! This is an ml5.js example made and shared with ❤️.
+ * Learn more about the ml5.js project: https://ml5js.org/
+ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+ *
+ * This example demonstrates training a Hand Gesture classifier through ml5.neuralNetwork with sequeceClassification Task.
+ */
+
+let video;
+let handPose;
+let hands = [];
+
+let sequence = [];
+let targetLength = 30;
+
+let gestures = ["Gesture #1", "Gesture #2"];
+let counts = { "Gesture #1": 0, "Gesture #2": 0 };
+
+let state = "collection";
+let currGesture = gestures[0]; //set currGesture to gesture 1 by default
+
+function preload() {
+  // Load the handPose model
+  handPose = ml5.handPose();
+
+  // setup the timeseries neural network
+  let options = {
+    outputs: ["label"],
+    task: "sequenceClassificationConv",
+    debug: "true",
+    learningRate: 0.001, // the default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update)
+  };
+  model = ml5.neuralNetwork(options);
+}
+
+function setup() {
+  let canvas = createCanvas(640, 480);
+  canvas.parent("canvasDiv");
+
+  // setup video capture
+  video = createCapture(VIDEO);
+  video.size(640, 480);
+  video.hide();
+
+  // Setup the UI buttons for training
+  UI();
+
+  // use handpose model on video
+  handPose.detectStart(video, gotHands);
+}
+
+function draw() {
+  // draw video on frame
+  image(video, 0, 0, width, height);
+
+  // helpful tooltip to include
+  textSize(20);
+  stroke(255);
+  fill(0);
+  text(state + " : " + currGesture, 50, 50);
+
+  // if hand is detected in the frame, start recording gesture
+  if (hands.length > 0) {
+    handpoints = drawPoints();
+    sequence.push(handpoints);
+
+    // add collected data to model once the hand is gone and state is collection
+  } else if (hands.length <= 0 && sequence.length > 0) {
+    if (state == "collection") {
+      // pad the length of the coordinates to targetLength
+      let inputData = model.padCoordinates(sequence, targetLength);
+      let outputData = { label: currGesture };
+
+      // add data to the model
+      model.addData(inputData, outputData);
+
+      // Update the counts for the UI
+      counts[currGesture]++;
+      updateDataCountUI();
+
+      // pad the data and use for prediction if state is prediction
+    } else if (state == "prediction") {
+      let predictData = model.padCoordinates(sequence, targetLength);
+      model.classify(predictData, gotResults);
+    }
+
+    // reset the sequence
+    sequence = [];
+  }
+}
+
+// Train the data when 'Train abd Save Model' button is pressed
+function train() {
+  // The data should be normalized before training
+  model.normalizeData();
+
+  currGesture = "";
+
+  // Train the model
+  let trainingOptions = {
+    epochs: 50,
+  };
+  model.train(trainingOptions, finishedTraining);
+}
+
+// When the model is trained, save the model
+function finishedTraining() {
+  state = "prediction";
+  model.save();
+}
+
+// callback for predict
+function gotResults(results) {
+  currGesture = results[0].label;
+}
+
+// Callback function for when handPose outputs data
+function gotHands(results) {
+  hands = results;
+}
+
+// draw visuals for hand points and flatten values into an array
+function drawPoints() {
+  let handpoints = [];
+  // iterate through both hands
+  for (let i = 0; i < hands.length; i++) {
+    let hand = hands[i];
+    for (let j = 0; j < hand.keypoints.length; j++) {
+      // access the keypoints in the hand
+      let keypoint = hand.keypoints[j];
+      handpoints.push(keypoint.x, keypoint.y);
+
+      fill(0, 255, 0);
+      noStroke();
+      circle(keypoint.x, keypoint.y, 5);
+    }
+  }
+
+  // assign to a different variable before clearing
+  let output = handpoints;
+  handpoints = [];
+
+  return output;
+}
+
+////////////// UI Elements ////////////
+function UI() {
+  dataCountsP = createP(
+    "Gesture 1 data: " +
+      counts[gestures[0]] +
+      "<br>Gesture 2 data: " +
+      counts[gestures[0]]
+  );
+  rockButton = createButton("Add Gesture #1 Data");
+  rockButton.mousePressed(addGesture1);
+  paperButton = createButton("Add Gesture #2 Data");
+  paperButton.mousePressed(addGesture2);
+  trainButton = createButton("Train and Save Model");
+  trainButton.mousePressed(train);
+}
+
+// Set the current handPose data to the model as "Gesture #1"
+function addGesture1() {
+  currGesture = gestures[0];
+}
+
+// Set the current handPose data to the model as "Gesture #2"
+function addGesture2() {
+  currGesture = gestures[1];
+}
+
+// Update the HTML UI with the current data counts
+function updateDataCountUI() {
+  dataCountsP.html(
+    "Gesture 1 data: " +
+      counts[gestures[0]] +
+      "<br>Gesture 2 data: " +
+      counts[gestures[1]]
+  );
+}
diff --git a/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html b/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html
@@ -0,0 +1,45 @@
+<!--
+  👋 Hello! This is an ml5.js example made and shared with ❤️.
+  Learn more about the ml5.js project: https://ml5js.org/
+  ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+
+  This example demonstrates loading a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task.
+-->
+
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ml5.js Time Series Hand Gesture load model</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
+    <script src="../../dist/ml5.js"></script>
+  </head>
+
+  <body>
+    <script src="sketch.js"></script>
+    <div id="canvasDiv"></div>
+    <p>
+      This example loads a model that is trained with ASL hand gestures for
+      Hello and Goodbye. <br />
+      <br />
+
+      Instructions: <br />
+      1.) Use one hand to do a gesture in front of the camera <br />
+      2.) Wait for the points to disappear or the prediction appears on
+      screen<br />
+      3.) To predict again, remove your hands in the frame and do the gesture
+      again<br /><br />
+
+      How to do gestures for Hello and Goodbye in ASL: <br />
+      Hello:
+      <a href="https://babysignlanguage.com/dictionary/hello/"
+        >https://babysignlanguage.com/dictionary/hello/ </a
+      ><br />
+      Goodbye:
+      <a href="https://babysignlanguage.com/dictionary/goodbye/"
+        >https://babysignlanguage.com/dictionary/goodbye/ </a
+      ><br />
+    </p>
+  </body>
+</html>