ml5js · mop9047 · Apr 1, 2025 · Apr 5, 2025 · Apr 5, 2025 · May 7, 2025
diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html
@@ -0,0 +1,28 @@
+<!--
+  👋 Hello! This is an ml5.js example made and shared with ❤️.
+  Learn more about the ml5.js project: https://ml5js.org/
+  ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+
+  This example demonstrates loading a hand gesture classifier through
+  ml5.neuralNetwork with the sequenceClassificationWithCNN task.
+  This example has been trained with the ASL gestures for Hello and Goodbye.
+-->
+
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ml5.js neuralNetwork Hand Gesture Loading Pre-trained Model Example</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
+    <script src="../../dist/ml5.js"></script>
+  </head>
+
+  <body>
+    <script src="sketch.js"></script>
+    <div id="canvasDiv"></div>
+    <p>
+      How to sign: <a href="https://www.signasl.org/sign/hello">Hello</a> &amp; <a href="https://www.signasl.org/sign/goodbye">Goodbye</a> in ASL.
+    </p>
+  </body>
+</html>
diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json
@@ -0,0 +1 @@
+{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,30,84],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.22.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,84,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[96,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]}
diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin
diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json
@@ -0,0 +1,105 @@
+{
+  "inputUnits": [84],
+  "outputUnits": 2,
+  "inputs": {
+    "label_0": { "dtype": "number", "min": 0, "max": 479.19925350185474 },
+    "label_1": { "dtype": "number", "min": 0, "max": 428.55538772969044 },
+    "label_2": { "dtype": "number", "min": 0, "max": 493.1563637363407 },
+    "label_3": { "dtype": "number", "min": 0, "max": 427.56692302194773 },
+    "label_4": { "dtype": "number", "min": 0, "max": 499.91689098126307 },
+    "label_5": { "dtype": "number", "min": 0, "max": 430.02650947613336 },
+    "label_6": { "dtype": "number", "min": 0, "max": 496.3208721418956 },
+    "label_7": { "dtype": "number", "min": 0, "max": 441.4487064271474 },
+    "label_8": { "dtype": "number", "min": 0, "max": 491.1078639783216 },
+    "label_9": { "dtype": "number", "min": 0, "max": 449.2829368661615 },
+    "label_10": { "dtype": "number", "min": 0, "max": 509.81763121840737 },
+    "label_11": { "dtype": "number", "min": 0, "max": 427.9468311472154 },
+    "label_12": { "dtype": "number", "min": 0, "max": 519.7167764450397 },
+    "label_13": { "dtype": "number", "min": 0, "max": 449.2054737819238 },
+    "label_14": { "dtype": "number", "min": 0, "max": 518.5199748098399 },
+    "label_15": { "dtype": "number", "min": 0, "max": 463.8811725137893 },
+    "label_16": { "dtype": "number", "min": 0, "max": 514.2732060805772 },
+    "label_17": { "dtype": "number", "min": 0, "max": 475.8006861523057 },
+    "label_18": { "dtype": "number", "min": 0, "max": 499.0736746691703 },
+    "label_19": { "dtype": "number", "min": 0, "max": 435.18154671948156 },
+    "label_20": { "dtype": "number", "min": 0, "max": 503.95906901514456 },
+    "label_21": { "dtype": "number", "min": 0, "max": 453.42442120690896 },
+    "label_22": { "dtype": "number", "min": 0, "max": 500.483162019238 },
+    "label_23": { "dtype": "number", "min": 0, "max": 468.18940999515536 },
+    "label_24": { "dtype": "number", "min": 0, "max": 495.75003969620434 },
+    "label_25": { "dtype": "number", "min": 0, "max": 480.72524952103413 },
+    "label_26": { "dtype": "number", "min": 0, "max": 483.45096761731435 },
+    "label_27": { "dtype": "number", "min": 0, "max": 440.95834852166774 },
+    "label_28": { "dtype": "number", "min": 0, "max": 473.19789907159645 },
+    "label_29": { "dtype": "number", "min": 0, "max": 458.5087523020914 },
+    "label_30": { "dtype": "number", "min": 0, "max": 467.93109224233444 },
+    "label_31": { "dtype": "number", "min": 0, "max": 471.2808751940178 },
+    "label_32": { "dtype": "number", "min": 0, "max": 465.7011008815248 },
+    "label_33": { "dtype": "number", "min": 0, "max": 481.9448468282673 },
+    "label_34": { "dtype": "number", "min": 0, "max": 464.5309672255035 },
+    "label_35": { "dtype": "number", "min": 0, "max": 444.791971246973 },
+    "label_36": { "dtype": "number", "min": 0, "max": 451.3378977170124 },
+    "label_37": { "dtype": "number", "min": 0, "max": 459.57942119350326 },
+    "label_38": { "dtype": "number", "min": 0, "max": 451.17016603053224 },
+    "label_39": { "dtype": "number", "min": 0, "max": 470.1975592491152 },
+    "label_40": {
+      "dtype": "number",
+      "min": -2.250544608200542,
+      "max": 455.08975496996834
+    },
+    "label_41": { "dtype": "number", "min": 0, "max": 478.7628390581024 },
+    "label_42": { "dtype": "number", "min": 0, "max": 586.6608405854927 },
+    "label_43": { "dtype": "number", "min": 0, "max": 443.59360893451384 },
+    "label_44": { "dtype": "number", "min": 0, "max": 575.2940255551233 },
+    "label_45": { "dtype": "number", "min": 0, "max": 439.4650023423489 },
+    "label_46": { "dtype": "number", "min": 0, "max": 572.6978570905426 },
+    "label_47": { "dtype": "number", "min": 0, "max": 445.77737640752645 },
+    "label_48": { "dtype": "number", "min": 0, "max": 577.5610386619726 },
+    "label_49": { "dtype": "number", "min": 0, "max": 455.3685328800494 },
+    "label_50": { "dtype": "number", "min": 0, "max": 587.5799889243797 },
+    "label_51": { "dtype": "number", "min": 0, "max": 460.6591159738472 },
+    "label_52": { "dtype": "number", "min": 0, "max": 595.3738279672924 },
+    "label_53": { "dtype": "number", "min": 0, "max": 435.3837420626638 },
+    "label_54": { "dtype": "number", "min": 0, "max": 603.0984481232397 },
+    "label_55": { "dtype": "number", "min": 0, "max": 458.47111655979 },
+    "label_56": { "dtype": "number", "min": 0, "max": 610.7793182445314 },
+    "label_57": { "dtype": "number", "min": 0, "max": 475.95085629845164 },
+    "label_58": { "dtype": "number", "min": 0, "max": 618.5882608713695 },
+    "label_59": { "dtype": "number", "min": 0, "max": 489.3053477098461 },
+    "label_60": { "dtype": "number", "min": 0, "max": 610.2477703388294 },
+    "label_61": { "dtype": "number", "min": 0, "max": 439.0911629675826 },
+    "label_62": { "dtype": "number", "min": 0, "max": 620.111633978391 },
+    "label_63": { "dtype": "number", "min": 0, "max": 465.4726266037314 },
+    "label_64": { "dtype": "number", "min": 0, "max": 626.9405786817175 },
+    "label_65": { "dtype": "number", "min": 0, "max": 484.92308751687654 },
+    "label_66": { "dtype": "number", "min": 0, "max": 633.3462004686493 },
+    "label_67": { "dtype": "number", "min": 0, "max": 497.7592120455144 },
+    "label_68": { "dtype": "number", "min": 0, "max": 622.5964250264554 },
+    "label_69": { "dtype": "number", "min": 0, "max": 443.22402036828703 },
+    "label_70": { "dtype": "number", "min": 0, "max": 634.5536612109757 },
+    "label_71": { "dtype": "number", "min": 0, "max": 470.2350331722456 },
+    "label_72": { "dtype": "number", "min": 0, "max": 642.2073345193608 },
+    "label_73": { "dtype": "number", "min": 0, "max": 487.5833155606672 },
+    "label_74": { "dtype": "number", "min": 0, "max": 648.6144713693823 },
+    "label_75": { "dtype": "number", "min": 0, "max": 498.5309450346654 },
+    "label_76": { "dtype": "number", "min": 0, "max": 632.6189459192858 },
+    "label_77": { "dtype": "number", "min": 0, "max": 447.8460008763578 },
+    "label_78": { "dtype": "number", "min": 0, "max": 644.54236495545 },
+    "label_79": { "dtype": "number", "min": 0, "max": 472.5026761828674 },
+    "label_80": { "dtype": "number", "min": 0, "max": 651.8202599792932 },
+    "label_81": { "dtype": "number", "min": 0, "max": 486.6449571135753 },
+    "label_82": { "dtype": "number", "min": 0, "max": 656.9313671078949 },
+    "label_83": { "dtype": "number", "min": 0, "max": 497.0423817040591 }
+  },
+  "outputs": {
+    "label": {
+      "dtype": "string",
+      "min": 0,
+      "max": 1,
+      "uniqueValues": ["Hello", "Good Bye"],
+      "legend": { "Hello": [1, 0], "Good Bye": [0, 1] }
+    }
+  },
+  "isNormalized": true,
+  "seriesShape": [30, 84]
+}
diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js
@@ -0,0 +1,158 @@
+/*
+ * 👋 Hello! This is an ml5.js example made and shared with ❤️.
+ * Learn more about the ml5.js project: https://ml5js.org/
+ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+ *
+ * This example demonstrates loading a hand gesture classifier through
+ * ml5.neuralNetwork with the sequenceClassificationWithCNN task.
+ * This example has been trained with the ASL gestures for Hello and Goodbye.
+ *
+ * Reference to sign hello and goodbye in ASL:
+ * Hello: https://www.signasl.org/sign/hello
+ * Goodbye: https://www.signasl.org/sign/goodbye
+ */
+
+let video;
+let handPose;
+let hands = [];
+let model;
+let isModelLoaded = false;
+
+let sequence = [];
+let sequenceLength = 30;
+let curGesture;
+
+function preload() {
+  // load the handPose model
+  handPose = ml5.handPose({ flipHorizontal: true });
+}
+
+function setup() {
+  let canvas = createCanvas(640, 480);
+  canvas.parent("canvasDiv");
+
+  video = createCapture(VIDEO, { flipped: true });
+  video.size(640, 480);
+  video.hide();
+  handPose.detectStart(video, gotHands);
+
+  let options = {
+    task: "sequenceClassificationWithCNN",
+  };
+  model = ml5.neuralNetwork(options);
+
+  // setup the model files to load
+  let modelDetails = {
+    model: "model/model.json",
+    metadata: "model/model_meta.json",
+    weights: "model/model.weights.bin",
+  };
+
+  // load the model and call modelLoaded once finished
+  model.load(modelDetails, modelLoaded);
+}
+
+function modelLoaded() {
+  console.log("Model loaded");
+  isModelLoaded = true;
+}
+
+function draw() {
+  image(video, 0, 0, width, height);
+  drawHands();
+  textSize(16);
+  stroke(0);
+  fill(255);
+
+  if (hands.length > 0) {
+    // hands in frame, add their keypoints to the sequence (input)
+    let handpoints = getKeypoints(["Left", "Right"]);
+    sequence.push(handpoints);
+    text(
+      "Move your hand(s) out of the frame after finishing the gesture",
+      50,
+      50
+    );
+  } else if (sequence.length > 0) {
+    // hands moved out of the frame, end of sequence
+
+    // Sequence will have varying length at this point, depending on
+    // how long the hands were in frame - a line simplification algorithm
+    // (RDP) turns it into the fixed length the NN can work with.
+    // For more information about RDP, see:
+    // https://www.youtube.com/watch?v=ZCXkvwLxBrA
+    let inputs = model.setFixedLength(sequence, sequenceLength);
+
+    // start the classification
+    if (isModelLoaded) {
+      model.classify(inputs, gotResults);
+    }
+    // reset the sequence
+    sequence = [];
+    text("Classifying...", 50, 50);
+  } else if (curGesture == null) {
+    // on program start
+    text("Move your hand(s) into the frame to sign a gesture", 50, 50);
+  } else {
+    // after receiving a classification
+    text('Saw "' + curGesture + '"', 50, 50);
+  }
+}
+
+// callback function for when the classification fininished
+function gotResults(results, error) {
+  if (error) {
+    console.error(error);
+    return;
+  }
+  curGesture = results[0].label;
+}
+
+// callback function for when handPose outputs data
+function gotHands(results) {
+  hands = results;
+}
+
+function drawHands() {
+  for (let i = 0; i < hands.length; i++) {
+    let hand = hands[i];
+    for (let j = 0; j < hand.keypoints.length; j++) {
+      let keypoint = hand.keypoints[j];
+      fill(0, 255, 0);
+      noStroke();
+      circle(keypoint.x, keypoint.y, 5);
+    }
+  }
+}
+
+// Return the tracked hand points as flattened array of 84 numbers
+// for use as input to the neural network
+
+function getKeypoints(whichHands = ["Left", "Right"]) {
+  let keypoints = [];
+  // look for the left and right hand
+  for (let whichHand of whichHands) {
+    let found = false;
+    for (let i = 0; i < hands.length; i++) {
+      let hand = hands[i];
+      if (hand.handedness == whichHand) {
+        // and add the x and y numbers of each tracked keypoint
+        // to the array
+        for (let j = 0; j < hand.keypoints.length; j++) {
+          let keypoint = hand.keypoints[j];
+          keypoints.push(keypoint.x, keypoint.y);
+        }
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      // if we don't find a right or a left hand, add 42 zeros
+      // to the keypoints array instead
+      for (let j = 0; j < 42; j++) {
+        keypoints.push(0);
+      }
+    }
+  }
+  return keypoints;
+}
diff --git a/examples/neuralNetwork-sequence-hand-gesture/index.html b/examples/neuralNetwork-sequence-hand-gesture/index.html
@@ -0,0 +1,24 @@
+<!--
+  👋 Hello! This is an ml5.js example made and shared with ❤️.
+  Learn more about the ml5.js project: https://ml5js.org/
+  ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
+
+  This example demonstrates training a hand gesture classifier through
+  ml5.neuralNetwork with the sequenceClassificationWithCNN task.
+-->
+
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ml5.js neuralNetwork Hand Gesture Training and Saving Example</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
+    <script src="../../dist/ml5.js"></script>
+  </head>
+
+  <body>
+    <script src="sketch.js"></script>
+    <div id="canvasDiv"></div>
+  </body>
+</html>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,30,84],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.22.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,84,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[96,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]}