From 0355d383a0b515b617c0cbcdc9220777643ce6d8 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 1 Apr 2025 11:47:34 -0400 Subject: [PATCH 01/47] added_instantiation_files --- .../timeSeries-instantiate-example/index.html | 22 + .../timeSeries-instantiate-example/sketch.js | 94 ++ .../index.html | 37 + .../sketch.js | 140 +++ .../timeSeries-train-mouse-gesture/index.html | 43 + .../timeSeries-train-mouse-gesture/sketch.js | 204 ++++ src/NeuralNetwork/index.js | 3 +- src/TimeSeries/index.js | 361 +++++++ src/TimeSeries/timeSeries.js | 251 +++++ src/TimeSeries/timeSeriesData.js | 943 ++++++++++++++++++ src/TimeSeries/timeSeriesUtils.js | 401 ++++++++ src/index.js | 2 + 12 files changed, 2500 insertions(+), 1 deletion(-) create mode 100644 examples/timeSeries-instantiate-example/index.html create mode 100644 examples/timeSeries-instantiate-example/sketch.js create mode 100644 examples/timeSeries-train-mouse-gesture RDP/index.html create mode 100644 examples/timeSeries-train-mouse-gesture RDP/sketch.js create mode 100644 examples/timeSeries-train-mouse-gesture/index.html create mode 100644 examples/timeSeries-train-mouse-gesture/sketch.js create mode 100644 src/TimeSeries/index.js create mode 100644 src/TimeSeries/timeSeries.js create mode 100644 src/TimeSeries/timeSeriesData.js create mode 100644 src/TimeSeries/timeSeriesUtils.js diff --git a/examples/timeSeries-instantiate-example/index.html b/examples/timeSeries-instantiate-example/index.html new file mode 100644 index 00000000..c406a501 --- /dev/null +++ b/examples/timeSeries-instantiate-example/index.html @@ -0,0 +1,22 @@ + + + + + + + + + ml5.js neuralNetwork Gesture Classifier Example + + + + + + + diff --git a/examples/timeSeries-instantiate-example/sketch.js b/examples/timeSeries-instantiate-example/sketch.js new file mode 100644 index 00000000..fcd0e8d9 --- /dev/null +++ b/examples/timeSeries-instantiate-example/sketch.js @@ -0,0 +1,94 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates training a mouse gesture classifier with ml5.neuralNetwork. + */ + +// Step 1: load data or create some data +let data = [ + { x: 0.99, y: 0.02, label: "right" }, + { x: 0.76, y: -0.1, label: "right" }, + { x: -1.0, y: 0.12, label: "left" }, + { x: -0.9, y: -0.1, label: "left" }, + { x: 0.02, y: 0.98, label: "down" }, + { x: -0.2, y: 0.75, label: "down" }, + { x: 0.01, y: -0.9, label: "up" }, + { x: -0.1, y: -0.8, label: "up" }, +]; + +let classifer; +let label = "training"; + +let start, end; + +function setup() { + createCanvas(640, 240); + // For this example to work across all browsers + // "webgl" or "cpu" needs to be set as the backend + ml5.setBackend("webgl"); + + // Step 2: set your neural network options + let options = { + task: "classification", + debug: true, + }; + + // Step 3: initialize your neural network + classifier = ml5.timeSeries(options); + + // Step 4: add data to the neural network + for (let i = 0; i < data.length; i++) { + let item = data[i]; + let inputs = [item.x, item.y]; + let outputs = [item.label]; + classifier.addData(inputs, outputs); + } + + // Step 5: normalize your data; + classifier.normalizeData(); + + // Step 6: train your neural network + classifier.train({ epochs: 100 }, finishedTraining); +} +// Step 7: use the trained model +function finishedTraining() { + label = "ready"; +} + +// Step 8: make a classification + +function draw() { + background(200); + textAlign(CENTER, CENTER); + textSize(64); + text(label, width / 2, height / 2); + if (start && end) { + strokeWeight(8); + line(start.x, start.y, end.x, end.y); + } +} + +function mousePressed() { + start = createVector(mouseX, mouseY); + end = createVector(mouseX, mouseY); +} + +function mouseDragged() { + end = createVector(mouseX, mouseY); +} + +function mouseReleased() { + let dir = p5.Vector.sub(end, start); + dir.normalize(); + let inputs = [dir.x, dir.y]; + console.log(inputs); + classifier.classify(inputs, gotResults); +} + +// Step 9: define a function to handle the results of your classification +function gotResults(results) { + label = results[0].label; + console.log(results); +} diff --git a/examples/timeSeries-train-mouse-gesture RDP/index.html b/examples/timeSeries-train-mouse-gesture RDP/index.html new file mode 100644 index 00000000..6407c4eb --- /dev/null +++ b/examples/timeSeries-train-mouse-gesture RDP/index.html @@ -0,0 +1,37 @@ + + + + + + + + ml5.js Time Series Train Mouse Gesture classifier Example + + + + + +
+ +
+ +
+ + + +
+ + + + diff --git a/examples/timeSeries-train-mouse-gesture RDP/sketch.js b/examples/timeSeries-train-mouse-gesture RDP/sketch.js new file mode 100644 index 00000000..31b54d27 --- /dev/null +++ b/examples/timeSeries-train-mouse-gesture RDP/sketch.js @@ -0,0 +1,140 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates How to train your own mouse gesture classifier through ml5.TimeSeries. + */ + +let model; + +let currShape = "circle"; +let state = "collection"; + +let datapoints; +let sequence = []; +let targetSequence = 30; +let recCircle, recSquare, trainBut; + +function preload() { + let options = { + inputs: ["x", "y"], + outputs: ["label"], + task: "classification", + dataMode: "spatial", + debug: "true", + learningRate: 0.005, + }; + + model = ml5.timeSeries(options); +} + +function setup() { + // p5 js elements + let canvas = createCanvas(600, 400); + canvas.parent("canvasDiv"); + background(220); + UI(); +} + +function draw() { + // record data when the mouse is pressed inside the canvas + if (mouseIsPressed && mouseY < height && mouseX < width) { + // draw lines through coordinates + line(pmouseX, pmouseY, mouseX, mouseY); + let inputs = { x: mouseX, y: mouseY }; + sequence.push(inputs); + } +} + +// code to signify drawing can be done again +function mouseReleased() { + if (mouseY < height && mouseX < width) { + // if state is collection, add whole sequence as X, and shape as Y + if (state == "collection") { + let target = { label: currShape }; + let paddedCoordinates = model.padCoordinates(sequence, targetSequence); + model.addData(paddedCoordinates, target); + clearScreen(); + } else if (state == "prediction") { + let paddedCoordinates = model.padCoordinates(sequence, targetSequence); + model.classify(paddedCoordinates, gotResults); + clearScreen(); + } + } + // reset the sequence + sequence = []; +} + +// cleanup screen and removed drawn elements, add helpful text +function clearScreen() { + background(220); + textSize(20); + fill(0); + text(state + " : " + currShape, 50, 50); +} + +function trainModel() { + // normalize Data first before Training + model.normalizeData(); + + // set the number of epochs for training + let options = { + epochs: 40, + }; + model.train(options, whileTraining, finishedTraining); + + background(220); + state = "training"; + text("Training...", 50, 50); + + recCircle.attribute("disabled", true); + recSquare.attribute("disabled", true); + trainBut.attribute("disabled", true); +} + +function whileTraining(epoch, loss) { + console.log(epoch); +} + +function finishedTraining() { + background(220); + text("Training Finished, Draw again to predict", 50, 50); + state = "prediction"; +} + +function gotResults(results) { + // console.log("results", results); + let label = results[0].label; + + currShape = label; +} + +////////////// UI Elements //////////// +function UI() { + textSize(20); + + recCircle = select("#recCircle"); + recSquare = select("#recSquare"); + trainBut = select("#trainBut"); + + recCircle.mouseClicked(recordCircle); + recSquare.mouseClicked(recordSquare); + trainBut.mouseClicked(trainModel); + + function recordCircle() { + state = "collection"; + currShape = "circle"; + + background(220); + text(state + " : " + currShape, 50, 50); + } + + function recordSquare() { + state = "collection"; + currShape = "square"; + + background(220); + text(state + " : " + currShape, 50, 50); + } +} diff --git a/examples/timeSeries-train-mouse-gesture/index.html b/examples/timeSeries-train-mouse-gesture/index.html new file mode 100644 index 00000000..acdfde38 --- /dev/null +++ b/examples/timeSeries-train-mouse-gesture/index.html @@ -0,0 +1,43 @@ + + + + + + + + ml5.js Time Series Train Mouse Gesture classifier Example + + + + + + +
+ + + + +

+ Instructions:
+ 1.) Press the "Record Circle" or "Record Square" and start drawing until + the ink runs out
+ 2.) Draw multiple times for each shape
2.) Press "Train" and wait for + training to finish
+ 3.) Draw again to predict drawn shape

+ Tip: Collect at least 5 drawings for each: +

+ + + + diff --git a/examples/timeSeries-train-mouse-gesture/sketch.js b/examples/timeSeries-train-mouse-gesture/sketch.js new file mode 100644 index 00000000..c0139c65 --- /dev/null +++ b/examples/timeSeries-train-mouse-gesture/sketch.js @@ -0,0 +1,204 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates How to train your own mouse gesture classifier through ml5.TimeSeries. + */ + +let model; +let counts = { + circleDataCount: 0, + squareDataCount: 0, +}; +let currShape = "circle"; +let state = "collection"; + +let pressedOnce = true; +let frameCount = 0; +let datapoints; +let sequence = []; +let recCircle, recSquare, trainBut; + +// Training Data lenghts +let ink_multiplier = 3; +let num_seq = 20; + +function preload() { + let options = { + inputs: ["x", "y"], + outputs: ["label"], + task: "classification", + spatialData: "true", + debug: "true", + learningRate: 0.005, + }; + + model = ml5.timeSeries(options); +} + +function setup() { + // p5 js elements + let canvas = createCanvas(600, 400); + canvas.parent("canvasDiv"); + background(220); + UI(); + + // set framerate to constant rate for constant data collection + frameRate(60); +} + +function draw() { + // record data when the mouse is pressed inside the canvas + if (mouseIsPressed && pressedOnce && mouseY < 400 && mouseX < 600) { + // draw lines through coordinates + line(pmouseX, pmouseY, mouseX, mouseY); + + frameCount++; + + let inputs = { x: mouseX, y: mouseY }; + + sequence.push(inputs); + + if (sequence.length == num_seq * ink_multiplier) { + pressedOnce = false; + frameCount = 0; + + // if state is collection, add whole sequence as X, and shape as Y + if (state == "collection") { + let target = { label: currShape }; + model.addData(sequence, target); + + // add to the count for each + counts[currShape + "DataCount"] += 1; + console.log(counts); + updateDataCountUI(); + + // reset the screen + background(220); + textSize(20); + fill(0); + text("Recording: " + currShape, 50, 50); + // if prediction, classify using the whole sequence + } else if (state == "prediction") { + model.classify(sequence, gotResults); + + background(220); + } + + // reset the sequence + sequence = []; + } + } + inkBar(); +} + +function trainModel() { + // normalize Data first before Training + model.normalizeData(); + + // set the number of epochs for training + let options = { + epochs: 40, + }; + model.train(options, whileTraining, finishedTraining); + + background(220); + state = "training"; + text("Training...", 50, 50); + recCircle.style("background-color", ""); + recSquare.style("background-color", ""); + trainBut.style("background-color", "#f0f0f0"); +} + +function whileTraining(epoch, loss) { + console.log(epoch); +} + +function finishedTraining() { + background(220); + text("Training Finished, Draw again to predict", 50, 50); + state = "prediction"; +} + +function gotResults(results) { + let label = results[0].label; + + fill(0); + text("Prediction: " + label, 50, 50); +} + +// code to signify drawing can be done again +function mouseReleased() { + pressedOnce = true; +} + +////////////// UI Elements //////////// + +// code to visualize how much ink left +function inkBar() { + datapoints = map(frameCount, 0, ink_multiplier * num_seq, 0, num_seq); + + bar_height = 250; + height_miltiplier = bar_height / num_seq; + push(); + fill(0); + textSize(15); + text("Ink:", 550, 90); + rect(550, 100, 25, num_seq * height_miltiplier); + fill(255); + rect(550, 100, 25, datapoints * height_miltiplier); + pop(); +} + +// code for UI elements such as buttons +function UI() { + textSize(20); + + recCircle = select("#recCircle"); + recSquare = select("#recSquare"); + trainBut = select("#trainBut"); + + recCircle.mouseClicked(recordCircle); + recCircle.style("background-color", "#f0f0f0"); + recSquare.mouseClicked(recordSquare); + trainBut.mouseClicked(trainModel); + + function recordCircle() { + state = "collection"; + currShape = "circle"; + + background(220); + text("Recording: circle", 50, 50); + recCircle.style("background-color", "#f0f0f0"); + recSquare.style("background-color", ""); + trainBut.style("background-color", ""); + } + + function recordSquare() { + state = "collection"; + currShape = "square"; + + background(220); + text("Recording: square", 50, 50); + recCircle.style("background-color", ""); + recSquare.style("background-color", "#f0f0f0"); + trainBut.style("background-color", ""); + } + dataCountsP = createP( + "circle data: " + + counts.circleDataCount + + "
square data: " + + counts.squareDataCount + ); +} + +// Update the HTML UI with the current data counts +function updateDataCountUI() { + dataCountsP.html( + "circle data: " + + counts.circleDataCount + + "
square data: " + + counts.squareDataCount + ); +} diff --git a/src/NeuralNetwork/index.js b/src/NeuralNetwork/index.js index fa439452..e3972c15 100644 --- a/src/NeuralNetwork/index.js +++ b/src/NeuralNetwork/index.js @@ -1268,4 +1268,5 @@ const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { return instance; }; -export default neuralNetwork; +export { DiyNeuralNetwork }; // Named export for extending +export default neuralNetwork; // Default export remains diff --git a/src/TimeSeries/index.js b/src/TimeSeries/index.js new file mode 100644 index 00000000..1e931192 --- /dev/null +++ b/src/TimeSeries/index.js @@ -0,0 +1,361 @@ +import * as tf from "@tensorflow/tfjs"; + +import { DiyNeuralNetwork } from "../NeuralNetwork"; + +import callCallback from "../utils/callcallback"; +import setBackend from "../utils/setBackend"; + +import tsUtils from "./timeSeriesUtils"; + +// import TimeSeries from "./timeSeries"; +import TimeSeriesData from "./timeSeriesData"; + +// call an extension of DIY Neural Network as a new class, override select methods +// which are seen below: +class DIYTimesSeries extends DiyNeuralNetwork { + constructor(options, callback) { + super( + { + ...options, + // neuralNetwork: null, + neuralNetworkData: null, + }, + callback + ); + // call all options set in the this class which is the default + this.options = { ...this.options, dataMode: "linear", ...(options || {}) }; + + // this.neuralNetwork = this.options.neuralNetwork || new TimeSeries(); + this.neuralNetworkData = + this.options.neuralNetworkData || new TimeSeriesData(); + + this.init = this.init.bind(this); + this.ready = callCallback(this.init(), callback); + } + + async init() { + // workaround for Error + setBackend("webgl"); + + await tf.ready(); + if (this.options.dataUrl) { + await this.loadDataFromUrl(this.options.dataUrl); + } else if (this.options.modelUrl) { + await this.load(this.options.modelUrl); + } + return this; + } + + addData(xInputs, yInputs, options = null) { + // 1. verify format between the three possible types of xinputs + const xs = tsUtils.verifyAndFormatInputs(xInputs, options, this.options); + + // 2. format the yInput - same logic as NN class + const ys = tsUtils.verifyAndFormatOutputs(yInputs, options, this.options); + console.log(xs, ys); + // 3. add data to raw + this.neuralNetworkData.addData(xs, ys); + } + + createMetaData() { + // this method does not get shape for images but instead for timesteps + const { inputs } = this.options; + + let inputShape; + if (typeof inputs === "number") { + inputShape = inputs; + } else if (Array.isArray(inputs) && inputs.length > 0) { + inputShape = inputs.length; //will be fed into the tensors later + } + + this.neuralNetworkData.createMetadata(inputShape); + } + + addDefaultLayers() { + let layers; + const task = this.options.task; + const dataMode = this.options.dataMode; + let taskConditions = `${task}_${dataMode}`; + switch (taskConditions.toLowerCase()) { + // if the task is classification and spatial modality + case "classification_spatial": + layers = [ + { + type: "conv1d", + filters: 8, + kernelSize: 3, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "conv1d", + filters: 16, + kernelSize: 3, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "flatten", + }, + { + type: "dense", + units: this.options.hiddenUnits, + activation: "relu", + }, + { + type: "dense", + activation: "softmax", + }, + ]; + + return this.createNetworkLayers(layers); + // if the task is classification and sequential modality + case "classification_linear": + layers = [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + returnSequences: true, + }, + { + type: "lstm", + units: 8, + activation: "relu", + returnSequences: false, + }, + { + type: "dense", + units: this.options.hiddenUnits, + activation: "relu", + }, + { + type: "dense", + activation: "softmax", + }, + ]; + + return this.createNetworkLayers(layers); + + // if the task is regression + case "regression_spatial": + layers = [ + { + type: "conv1d", + filters: 8, + kernelSize: 3, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "conv1d", + filters: 16, + kernelSize: 3, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "flatten", + }, + { + type: "dense", + units: this.options.hiddenUnits, + activation: "relu", + }, + { + type: "dense", + activation: "sigmoid", + }, + ]; + + return this.createNetworkLayers(layers); + + case "regression_linear": + layers = [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + returnSequences: true, + }, + { + type: "lstm", + units: 8, + activation: "relu", + }, + { + type: "dense", + units: this.options.hiddenUnits, + activation: "relu", + }, + { + type: "dense", + activation: "sigmoid", + }, + ]; + + return this.createNetworkLayers(layers); + + default: + console.log("no inputUnits or outputUnits defined"); + layers = [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: this.neuralNetworkData.meta.seriesShape, + }, + { + type: "lstm", + units: 8, + activation: "relu", + }, + { + type: "dense", + units: this.options.hiddenUnits, + activation: "relu", + }, + { + type: "dense", + activation: "sigmoid", + }, + ]; + return this.createNetworkLayers(layers); + } + } + + async loadDataFromUrl(dataUrl, inputs, outputs) { + let json; + let dataFromUrl; + try { + if (dataUrl.endsWith(".csv")) { + dataFromUrl = await this.neuralNetworkData.loadCSV( + dataUrl, + inputs, + outputs + ); + } else if (dataUrl.endsWith(".json")) { + dataFromUrl = await this.neuralNetworkData.loadJSON( + dataUrl, + inputs, + outputs + ); + } else if (dataUrl.includes("blob")) { + dataFromUrl = await this.loadBlob(dataUrl, inputs, outputs); + } else { + throw new Error("Not a valid data format. Must be csv or json"); + } + } catch (error) { + console.error(error); + throw new Error(error); + } + + dataFromUrl.map((item) => { + this.addData(item.xs, item.ys); + }); + + this.createMetaData(); + + this.prepareForTraining(); + } + + formatInputsForPredictionAll(_input) { + const { meta } = this.neuralNetworkData; + const inputHeaders = Object.keys(meta.inputs); + + const formatted_inputs = tsUtils.verifyAndFormatInputs( + _input, + null, + this.options + ); + const normalized_inputs = this.neuralNetworkData.normalizePredictData( + formatted_inputs, + meta.inputs + ); + const output = tf.tensor(normalized_inputs); + + return output; + } + + async classifyInternal(_input) { + const { meta } = this.neuralNetworkData; + const headers = Object.keys(meta.inputs); + + let inputData; + + inputData = this.formatInputsForPredictionAll(_input); + + const unformattedResults = await this.neuralNetwork.classify(inputData); + inputData.dispose(); + + if (meta !== null) { + const label = Object.keys(meta.outputs)[0]; + const vals = Object.entries(meta.outputs[label].legend); + + const formattedResults = unformattedResults.map((unformattedResult) => { + return vals + .map((item, idx) => { + return { + [item[0]]: unformattedResult[idx], + label: item[0], + confidence: unformattedResult[idx], + }; + }) + .sort((a, b) => b.confidence - a.confidence); + }); + + // return single array if the length is less than 2, + // otherwise return array of arrays + if (formattedResults.length < 2) { + return formattedResults[0]; + } + return formattedResults; + } + + return unformattedResults; + } + + padCoordinates(coordinates, targetPointCount) { + const maxEpsilon = int(coordinates.length / 2); + return tsUtils.padCoordinates(coordinates, targetPointCount, maxEpsilon); + } +} + +const timeSeries = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + let cb; + + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + cb = outputsOrCallback; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + cb = callback; + } + + const instance = new DIYTimesSeries(options, cb); + return instance; +}; + +export default timeSeries; diff --git a/src/TimeSeries/timeSeries.js b/src/TimeSeries/timeSeries.js new file mode 100644 index 00000000..685e2831 --- /dev/null +++ b/src/TimeSeries/timeSeries.js @@ -0,0 +1,251 @@ +import * as tf from "@tensorflow/tfjs"; +import { saveBlob } from "../utils/io"; +import timeSeries from "."; + +/* + +Things changed from neural network class: + +1. No neuro evolution + +*/ + +class TimeSeries { + constructor() { + // flags + this.isTrained = false; + this.isCompiled = false; + this.isLayered = false; + /** + * @type {tf.Sequential | null} - the TensorFlow model + */ + this.model = null; + + // methods + this.init = this.init.bind(this); + this.createModel = this.createModel.bind(this); + this.addLayer = this.addLayer.bind(this); + this.compile = this.compile.bind(this); + this.setOptimizerFunction = this.setOptimizerFunction.bind(this); + this.train = this.train.bind(this); + this.predict = this.predict.bind(this); + this.classify = this.classify.bind(this); + this.save = this.save.bind(this); + this.load = this.load.bind(this); + + // initialize + this.init(); + } + + /** + * initialize with create model + */ + init() { + this.createModel(); + } + + /** + * creates a sequential model + * uses switch/case for potential future where different formats are supported + * @param {*} _type + */ + createModel(_type = "sequential") { + switch (_type.toLowerCase()) { + case "sequential": + this.model = tf.sequential(); + return this.model; + default: + this.model = tf.sequential(); + return this.model; + } + } + + /** + * add layer to the model + * if the model has 2 or more layers switch the isLayered flag + * @param {tf.Layer} layer + * @void + */ + addLayer(layer) { + this.model.add(layer); + + // check if it has at least an input and output layer + if (this.model.layers.length >= 2) { + this.isLayered = true; + } + } + + /** + * Compile the model + * if the model is compiled, set the isCompiled flag to true + * @param {*} _modelOptions + */ + compile(_modelOptions) { + this.model.compile(_modelOptions); + this.isCompiled = true; + } + + /** + * Set the optimizer function given the learning rate + * as a parameter + * @param {*} learningRate + * @param {*} optimizer + */ + setOptimizerFunction(learningRate, optimizer) { + return optimizer.call(this, learningRate); + } + + /** + * Train the model + * @param {Object} _options + */ + async train(_options) { + const TRAINING_OPTIONS = _options; + + const xs = TRAINING_OPTIONS.inputs; + const ys = TRAINING_OPTIONS.outputs; + console.log("train", xs, ys); + const { batchSize, epochs, shuffle, validationSplit, whileTraining } = + TRAINING_OPTIONS; + + await this.model.fit(xs, ys, { + batchSize, + epochs, + shuffle, + validationSplit, + callbacks: whileTraining, + }); + + xs.dispose(); + ys.dispose(); + + this.isTrained = true; + } + + /** + * returns the prediction as an array synchronously + * @param {*} _inputs + */ + predictSync(_inputs) { + const output = tf.tidy(() => { + return this.model.predict(_inputs); + }); + const result = output.arraySync(); + + output.dispose(); + _inputs.dispose(); + + return result; + } + + /** + * returns the prediction as an array + * @param {*} _inputs + */ + async predict(_inputs) { + const output = tf.tidy(() => { + return this.model.predict(_inputs); + }); + const result = await output.array(); + + output.dispose(); + _inputs.dispose(); + + return result; + } + + /** + * classify is the same as .predict() + * @param {*} _inputs + */ + async classify(_inputs) { + return this.predict(_inputs); + } + + /** + * classify is the same as .predict() + * @param {*} _inputs + */ + classifySync(_inputs) { + return this.predictSync(_inputs); + } + + // predictMultiple + // classifyMultiple + // are the same as .predict() + + /** + * save the model.json and the weights.bin files + * @param {string} modelName + * @return {Promise} + */ + async save(modelName = "model") { + await this.model.save( + tf.io.withSaveHandler(async (data) => { + this.weightsManifest = { + modelTopology: data.modelTopology, + weightsManifest: [ + { + paths: [`./${modelName}.weights.bin`], + weights: data.weightSpecs, + }, + ], + }; + console.log("data.weightData", data.weightData); + await saveBlob( + data.weightData, + `${modelName}.weights.bin`, + "application/octet-stream" + ); + console.log("this.weightsManifest", this.weightsManifest); + await saveBlob( + JSON.stringify(this.weightsManifest), + `${modelName}.json`, + "text/plain" + ); + }) + ); + } + + /** + * loads the model and weights + * @param {string | FileList | Object} filesOrPath + */ + async load(filesOrPath) { + if (filesOrPath instanceof FileList) { + const files = Array.from(filesOrPath); + // find the correct files + const model = files.find( + (file) => file.name.includes(".json") && !file.name.includes("_meta") + ); + const weights = files.find((file) => file.name.includes(".bin")); + // load the model + this.model = await tf.loadLayersModel( + tf.io.browserFiles([model, weights]) + ); + } else if (filesOrPath instanceof Object) { + this.model = await tf.loadLayersModel( + tf.io.http(filesOrPath.model, { + // Override the weights path from the JSON weightsManifest + weightUrlConverter: (weightFileName) => { + return filesOrPath.weights || weightFileName; + }, + }) + ); + } else { + this.model = await tf.loadLayersModel(filesOrPath); + } + + this.isCompiled = true; + this.isLayered = true; + this.isTrained = true; + } + + /** + * dispose and release the memory for the model + */ + dispose() { + this.model.dispose(); + } +} +export default TimeSeries; diff --git a/src/TimeSeries/timeSeriesData.js b/src/TimeSeries/timeSeriesData.js new file mode 100644 index 00000000..f9b106c3 --- /dev/null +++ b/src/TimeSeries/timeSeriesData.js @@ -0,0 +1,943 @@ +import * as tf from "@tensorflow/tfjs"; +import axios from "axios"; +import { saveBlob } from "../utils/io"; +import modelLoader from "../utils/modelLoader"; +import nnUtils from "../NeuralNetwork/NeuralNetworkUtils"; + +import tsUtils from "./timeSeriesUtils"; + +class TimeSeriesData { + constructor() { + this.meta = { + inputUnits: null, // Number + outputUnits: null, // Number + // objects describing input/output data by property name + inputs: {}, // { name1: {dtype}, name2: {dtype} } + outputs: {}, // { name1: {dtype} } + isNormalized: false, // Boolean - keep this in meta for model saving/loading + }; + + this.isMetadataReady = false; + this.isWarmedUp = false; + + this.data = { + raw: [], // array of {xs:[{},{}], ys:{}} + }; + } + + /** + * //////////////////////////////////////////////////////// + * Add Data + * //////////////////////////////////////////////////////// + */ + + /** + * Add Data + * @param {object} xInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array + * @param {*} yInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array + * @void - updates this.data + */ + addData(xInputObj, yInputObj) { + this.data.raw.push({ + xs: xInputObj, + ys: yInputObj, + }); + } + + /** + * //////////////////////////////////////////////////////// + * Summarize Data + * //////////////////////////////////////////////////////// + */ + + /** + * create the metadata from the data + * this covers: + * 1. getting the datatype from the data + * 2. getting the min and max from the data + * 3. getting the oneHot encoded values + * 4. getting the inputShape and outputUnits from the data + * @param {Array} [inputShape] + * @void + */ + createMetadata(inputShape = null) { + // get the data type for each property + this.getDTypesFromSeriesData(); + // get the stats - min, max + this.getDataStats(); + // onehot encode + this.getDataOneHot(); + // calculate the input units from the data + this.getDataUnits(inputShape); + // get the shape of batch + + this.isMetadataReady = true; + } + + /** + * getDTypesFromData + * gets the data types of the data we're using + * important for handling oneHot + * @private + * @void - updates this.meta + */ + getDTypesFromSeriesData() { + const meta = { + ...this.meta, + inputs: {}, + outputs: {}, + }; + + const sample = this.data.raw[0]; + + //consistent dTypes have already been checked at add data + const xs = Object.keys(sample.xs[0]); //since time series data is in form of array + const ys = Object.keys(sample.ys); + xs.forEach((prop) => { + meta.inputs[prop] = { + dtype: nnUtils.getDataType(sample.xs[0][prop]), + }; + }); + + ys.forEach((prop) => { + meta.outputs[prop] = { + dtype: nnUtils.getDataType(sample.ys[prop]), + }; + }); + + this.meta = meta; + } + + /** + * get stats about the data + * @private + * @void + */ + getDataStats() { + this.meta.inputs = this.getInputMetaStats(this.meta.inputs, "xs"); + this.meta.outputs = this.getInputMetaStats(this.meta.outputs, "ys"); + } + + /** + * get back the min and max of each label + * @private + * @param {Object} inputOrOutputMeta + * @param {"xs" | "ys"} xsOrYs + * @return {Object} + */ + getInputMetaStats(inputOrOutputMeta, xsOrYs) { + const inputMeta = Object.assign({}, inputOrOutputMeta); + + Object.keys(inputMeta).forEach((k) => { + if (inputMeta[k].dtype === "string") { + inputMeta[k].min = 0; + inputMeta[k].max = 1; + } else if (inputMeta[k].dtype === "number") { + let dataAsArray; + if (xsOrYs === "ys") { + dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]); + } else if (xsOrYs === "xs") { + dataAsArray = this.data.raw.flatMap((item) => + item[xsOrYs].map((obj) => obj[k]) + ); + } + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } else if (inputMeta[k].dtype === "array") { + const dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]).flat(); + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } + }); + + return inputMeta; + } + + /** + * getDataOneHot + * creates onehot encodings for the input and outputs + * and adds them to the meta info + * @private + * @void + */ + getDataOneHot() { + this.meta.inputs = this.getInputMetaOneHot(this.meta.inputs, "xs"); + this.meta.outputs = this.getInputMetaOneHot(this.meta.outputs, "ys"); + } + + /** + * getOneHotMeta + * @param {Object} _inputsMeta + * @param {"xs" | "ys"} xsOrYs + * @return {Object} + */ + getInputMetaOneHot(_inputsMeta, xsOrYs) { + const inputsMeta = Object.assign({}, _inputsMeta); + + Object.entries(inputsMeta).forEach((arr) => { + // the key + const key = arr[0]; + // the value + const { dtype } = arr[1]; + + if (dtype === "string") { + const uniqueVals = [ + ...new Set(this.data.raw.map((obj) => obj[xsOrYs][key])), + ]; + const oneHotMeta = this.createOneHotEncodings(uniqueVals); + inputsMeta[key] = { + ...inputsMeta[key], + ...oneHotMeta, + }; + } + }); + return inputsMeta; + } + + /** + * get the data units, inputshape and output units + * @private + * @param {Array} arrayShape + * @void + */ + getDataUnits(arrayShape = null) { + // if the data has a shape pass it in + if (arrayShape) { + this.meta.inputUnits = arrayShape; + } else { + this.meta.inputUnits = [this.getInputMetaUnits(this.meta.inputs)].flat(); + } + + this.meta.outputUnits = this.getInputMetaUnits(this.meta.outputs); + } + + /** + * @private + * @param {Object} inputsMeta + * @return {number | Array} + */ + // eslint-disable-next-line class-methods-use-this + getInputMetaUnits(inputsMeta) { + let units = 0; + + Object.entries(inputsMeta).forEach((arr) => { + const { dtype } = arr[1]; + if (dtype === "number") { + units += 1; + } else if (dtype === "string") { + const { uniqueValues } = arr[1]; + + const uniqueCount = uniqueValues.length; + units += uniqueCount; + } else if (dtype === "array") { + // TODO: User must input the shape of the + // image size correctly. + units = []; + } + }); + + return units; + } + + /** + * Returns a legend mapping the + * data values to oneHot encoded values + * @private + * @param {Array} _uniqueValuesArray + * @return {Object} + */ + // eslint-disable-next-line class-methods-use-this, no-unused-vars + createOneHotEncodings(_uniqueValuesArray) { + return tf.tidy(() => { + const output = { + uniqueValues: _uniqueValuesArray, + legend: {}, + }; + + const uniqueVals = _uniqueValuesArray; // [...new Set(this.data.raw.map(obj => obj.xs[prop]))] + // get back values from 0 to the length of the uniqueVals array + const onehotValues = uniqueVals.map((item, idx) => idx); + // oneHot encode the values in the 1d tensor + const oneHotEncodedValues = tf.oneHot( + tf.tensor1d(onehotValues, "int32"), + uniqueVals.length + ); + // convert them from tensors back out to an array + const oneHotEncodedValuesArray = oneHotEncodedValues.arraySync(); + + // populate the legend with the key/values + uniqueVals.forEach((uVal, uIdx) => { + output.legend[uVal] = oneHotEncodedValuesArray[uIdx]; + }); + + return output; + }); + } + + /** + * //////////////////////////////////////////////////////// + * Tensor handling + * //////////////////////////////////////////////////////// + */ + + /** + * convertRawToTensors + * converts array of {xs, ys} to tensors + * @param {*} dataRaw + * + * @return {{ inputs: tf.Tensor, outputs: tf.Tensor }} + */ + // eslint-disable-next-line class-methods-use-this, no-unused-vars + convertRawToTensors(dataRaw) { + const meta = Object.assign({}, this.meta); + const dataLength = dataRaw.length; + + return tf.tidy(() => { + const inputArr = []; + const outputArr = []; + + dataRaw.forEach((row) => { + // get xs + // const xs = Object.keys(meta.inputs) + // .map((k) => { + // return row.xs[k]; + // }) + // .flat(); + + // inputArr.push(xs); + + const xs = row.xs; + inputArr.push(xs); + + // get ys + const ys = Object.keys(meta.outputs) + .map((k) => { + return row.ys[k]; + }) + .flat(); + + outputArr.push(ys); + }); + + // const inputs = tf.tensor(inputArr.flat(), [ + // dataLength, + // ...meta.inputUnits, + // ]); + const inputs = tf.tensor(inputArr); + + const outputs = tf.tensor(outputArr.flat(), [ + dataLength, + meta.outputUnits, + ]); + + return { + inputs, + outputs, + }; + }); + } + + /** + * //////////////////////////////////////////////////////// + * data normalization / unnormalization + * //////////////////////////////////////////////////////// + */ + + /** + * normalize the dataRaw input + * @return {Array} + */ + normalizeDataRaw() { + const normXs = this.normalizeInputData(this.meta.inputs, "xs"); + const normYs = this.normalizeInputData(this.meta.outputs, "ys"); + const normalizedData = tsUtils.zipArraySequence(normXs, normYs); + + return normalizedData; + } + + /** + * @param {Object} inputOrOutputMeta + * @param {"xs" | "ys"} xsOrYs + * @return {Array} + */ + normalizeInputData(inputOrOutputMeta, xsOrYs) { + const dataRaw = this.data.raw; + + // the data length + const dataLength = dataRaw.length; + + // the copy of the inputs.meta[inputOrOutput] + const inputMeta = Object.assign({}, inputOrOutputMeta); + + // normalized output object + const normalized = {}; + Object.keys(inputMeta).forEach((k) => { + // get the min and max values + const options = { + min: inputMeta[k].min, + max: inputMeta[k].max, + }; + + // depending on the input type, normalize accordingly + if (inputMeta[k].dtype === "string") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + options.legend = inputMeta[k].legend; + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "number") { + let dataAsArray; + if (xsOrYs === "ys") { + dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]); + } else if (xsOrYs === "xs") { + dataAsArray = this.data.raw.flatMap((item) => + item[xsOrYs].map((obj) => obj[k]) + ); + } + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "array") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + normalized[k] = dataAsArray.map((item) => + this.normalizeArray(item, options) + ); + } + }); + + let output; + if (xsOrYs == "ys") { + output = [...new Array(dataLength).fill(null)].map((item, idx) => { + const row = { + [xsOrYs]: {}, + }; + + Object.keys(inputMeta).forEach((k) => { + row[xsOrYs][k] = normalized[k][idx]; + }); + + return row; + }); + } else if (xsOrYs == "xs") { + // reshape array - already ready for tensorconversion + const features = Object.keys(inputMeta); + const feature_length = features.length; + + const seriesStep = dataRaw[0]["xs"].length; + + const batch = normalized[features[0]].length / seriesStep; + + this.meta.seriesShape = [seriesStep, feature_length]; + console.log("series shape", this.meta.seriesShape); + let zipped = []; + + // zip arrays before reshaping + for (let idx = 0; idx < seriesStep * feature_length * batch; idx++) { + features.forEach((k) => { + zipped.push(normalized[k][idx]); + }); + } + + // reshaping + output = tsUtils.reshapeTo3DArray(zipped, [ + batch, + seriesStep, + feature_length, + ]); + } + + console.log("thismeta", this.meta); + return output; + } + + /** + * normalizeArray + * @param {*} _input + * @param {*} _options + */ + // eslint-disable-next-line no-unused-vars, class-methods-use-this + normalizeArray(inputArray, options) { + const { min, max } = options; + + // if the data are onehot encoded, replace the string + // value with the onehot array + // if none exists, return the given value + if (options.legend) { + const normalized = inputArray.map((v) => { + return options.legend[v] ? options.legend[v] : v; + }); + return normalized; + } + + // if the dtype is a number + if (inputArray.every((v) => typeof v === "number")) { + const normalized = inputArray.map((v) => + nnUtils.normalizeValue(v, min, max) + ); + return normalized; + } + + // otherwise return the input array + // return inputArray; + throw new Error("error in inputArray of normalizeArray() function"); + } + + normalizePredictData(dataRaw, inputOrOutputMeta) { + const inputMeta = Object.assign({}, inputOrOutputMeta); + const xsOrYs = "xs"; + const predict_normalized = {}; + Object.keys(inputMeta).forEach((k) => { + // get the min and max values + const options = { + min: inputMeta[k].min, + max: inputMeta[k].max, + }; + if (inputMeta[k].dtype === "string") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + options.legend = inputMeta[k].legend; + predict_normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "number") { + const dataAsArray = Array(dataRaw).flatMap((item) => + item.map((obj) => obj[k]) + ); + console.log(dataAsArray); + predict_normalized[k] = this.normalizeArray(dataAsArray, options); + } + }); + + console.log("done", predict_normalized); + + const features = Object.keys(inputMeta); + const feature_length = features.length; + + const seriesStep = dataRaw.length; + + const batch = 1; + let zipped = []; + + // zip arrays before reshaping + for (let idx = 0; idx < seriesStep * feature_length * batch; idx++) { + features.forEach((k) => { + zipped.push(predict_normalized[k][idx]); + }); + } + // reshaping + const output = tsUtils.reshapeTo3DArray(zipped, [ + batch, + seriesStep, + feature_length, + ]); + return output; + } + + /** + * unNormalizeArray + * @param {*} _input + * @param {*} _options + */ + // eslint-disable-next-line no-unused-vars, class-methods-use-this + // unnormalizeArray(inputArray, options) { + // const { min, max } = options; + + // // if the data is onehot encoded then remap the + // // values from those oneHot arrays + // if (options.legend) { + // const unnormalized = inputArray.map((v) => { + // let res; + // Object.entries(options.legend).forEach((item) => { + // const key = item[0]; + // const val = item[1]; + // const matches = v + // .map((num, idx) => num === val[idx]) + // .every((truthy) => truthy === true); + // if (matches) res = key; + // }); + // return res; + // }); + + // return unnormalized; + // } + + // // if the dtype is a number + // if (inputArray.every((v) => typeof v === "number")) { + // const unnormalized = inputArray.map((v) => + // nnUtils.unnormalizeValue(v, min, max) + // ); + // return unnormalized; + // } + + // // otherwise return the input array + // // return inputArray; + // throw new Error("error in inputArray of normalizeArray() function"); + // } + + /* + * //////////////////////////////////////////////// + * One hot encoding handling + * //////////////////////////////////////////////// + */ + + /** + * applyOneHotEncodingsToDataRaw + * does not set this.data.raws + * but rather returns them + */ + applyOneHotEncodingsToDataRaw() { + const meta = Object.assign({}, this.meta); + + const output = this.data.raw.map((row) => { + const xs = { + ...row.xs, + }; + const ys = { + ...row.ys, + }; + + // get xs + Object.keys(meta.inputs).forEach((k) => { + if (meta.inputs[k].legend) { + xs[k] = meta.inputs[k].legend[row.xs[k]]; + } + }); + + Object.keys(meta.outputs).forEach((k) => { + if (meta.outputs[k].legend) { + ys[k] = meta.outputs[k].legend[row.ys[k]]; + } + }); + + return { + xs, + ys, + }; + }); + return output; + } + + /** + * //////////////////////////////////////////////// + * saving / loading data + * //////////////////////////////////////////////// + */ + + /** + * Loads data from a URL using the appropriate function + * @param {*} dataUrl + * @param {*} inputs + * @param {*} outputs + * @void + */ + async loadDataFromUrl(dataUrl, inputs, outputs) { + try { + if (dataUrl.endsWith(".csv")) { + await this.loadCSV(dataUrl, inputs, outputs); + } else if (dataUrl.endsWith(".json")) { + await this.loadJSON(dataUrl, inputs, outputs); + } else if (dataUrl.includes("blob")) { + await this.loadBlob(dataUrl, inputs, outputs); + } else { + throw new Error("Not a valid data format. Must be csv or json"); + } + } catch (error) { + console.error(error); + throw new Error(error); + } + } + + /** + * loadJSON + * @param {*} dataUrlOrJson + * @param {*} inputLabels + * @param {*} outputLabels + * @void + */ + async loadJSON(dataUrlOrJson, inputLabels, outputLabels) { + try { + let json; + // handle loading parsedJson + if (dataUrlOrJson instanceof Object) { + json = Object.assign({}, dataUrlOrJson); + } else { + const { data } = await axios.get(dataUrlOrJson); + json = data; + } + + // format the data.raw array + // this.formatRawData(json, inputLabels, outputLabels); + return this.findEntries(json); + } catch (err) { + console.error("error loading json"); + throw new Error(err); + } + } + + /** + * loadCSV + * @param {*} dataUrl + * @param {*} inputLabels + * @param {*} outputLabels + * @void + */ + async loadCSV(dataUrl, inputLabels, outputLabels) { + try { + const myCsv = tf.data.csv(dataUrl); + const loadedData = await myCsv.toArray(); + const json = { + entries: loadedData, + }; + // format the data.raw array + // this.formatRawData(json, inputLabels, outputLabels); + return this.findEntries(json); + } catch (err) { + console.error("error loading csv", err); + throw new Error(err); + } + } + + /** + * loadBlob + * @param {*} dataUrlOrJson + * @param {*} inputLabels + * @param {*} outputLabels + * @void + */ + async loadBlob(dataUrlOrJson, inputLabels, outputLabels) { + try { + const { data } = await axios.get(dataUrlOrJson); + const text = data; // await data.text(); + + if (nnUtils.isJsonOrString(text)) { + const json = JSON.parse(text); + await this.loadJSON(json, inputLabels, outputLabels); + } else { + const json = this.csvToJSON(text); + await this.loadJSON(json, inputLabels, outputLabels); + } + } catch (err) { + console.log("mmm might be passing in a string or something!", err); + throw new Error(err); + } + } + + /** + * loadData from fileinput or path + * @param {string | FileList | Object} filesOrPath + * @return {Promise} + */ + async loadData(filesOrPath) { + try { + let loadedData; + + if (typeof filesOrPath !== "string") { + const file = filesOrPath[0]; + const fr = new FileReader(); + fr.readAsText(file); + if (file.name.includes(".json")) { + const temp = await file.text(); + loadedData = JSON.parse(temp); + } else { + console.log( + 'data must be a json object containing an array called "data" or "entries' + ); + } + } else { + loadedData = await axios.get(filesOrPath, { responseType: "text" }); + const text = JSON.stringify(loadedData.data); + if (nnUtils.isJsonOrString(text)) { + loadedData = JSON.parse(text); + console.log(loadedData); + } else { + console.log( + "Whoops! something went wrong. Either this kind of data is not supported yet or there is an issue with .loadData" + ); + } + } + + this.data.raw = this.findEntries(loadedData); + + // check if a data or entries property exists + if (!this.data.raw.length > 0) { + console.log( + 'data must be a json object containing an array called "data" ' + ); + } + } catch (error) { + throw new Error(error); + } + } + + /** + * saveData + * @param {string} [name] + * @return {Promise} + */ + async saveData(name) { + const today = new Date(); + const date = `${String(today.getFullYear())}-${String( + today.getMonth() + 1 + )}-${String(today.getDate())}`; + const time = `${String(today.getHours())}-${String( + today.getMinutes() + )}-${String(today.getSeconds())}`; + const datetime = `${date}_${time}`; + + let dataName = datetime; + if (name) dataName = name; + + const output = { + data: this.data.raw, + }; + + await saveBlob(JSON.stringify(output), `${dataName}.json`, "text/plain"); + } + + /** + * Saves metadata of the data + * @param {string} modelName + * @return {Promise} + */ + async saveMeta(modelName = "model") { + console.log("meta saved"); + await saveBlob( + JSON.stringify(this.meta), + `${modelName}_meta.json`, + "text/plain" + ); + } + + /** + * load a model and metadata + * @param {string | FileList | Object} filesOrPath + * @return {Promise} + */ + async loadMeta(filesOrPath) { + if (filesOrPath instanceof FileList) { + const file = Array.from(filesOrPath).find((file) => + file.name.includes("_meta.json") + ); + if (!file) { + console.warn("no model_meta.json file found in FileList"); + return; + } + const text = await file.text(); + this.meta = JSON.parse(text); + } else if (filesOrPath instanceof Object) { + // filesOrPath = {model: URL, metadata: URL, weights: URL} + const metadataResult = await axios.get(filesOrPath.metadata); + this.meta = metadataResult.data; + } else { + const loader = modelLoader(filesOrPath); + // TODO: it is not always "model_meta.json", it is "{model_name}_meta.json" + const metaPath = loader.getPath("model_meta.json"); + // TODO: figure out how to use loader.loadMetadataJson() which has wrapped error messages + const metadataResult = await axios.get(metaPath); + this.meta = metadataResult.data; + } + + this.isMetadataReady = true; + this.isWarmedUp = true; + } + + /* + * //////////////////////////////////////////////// + * data loading helpers + * //////////////////////////////////////////////// + */ + + /** + * // TODO: convert ys into strings, if the task is classification + // if (this.config.architecture.task === "classification" && typeof output.ys[prop] !== "string") { + // output.ys[prop] += ""; + // } + * formatRawData + * takes a json and set the this.data.raw + * @param {*} json + * @param {Array} inputLabels + * @param {Array} outputLabels + * @void + */ + // formatRawData(json, inputLabels, outputLabels) { + + // // Recurse through the json object to find + // // an array containing `entries` or `data` + // const dataArray = this.findEntries(json); + + // if (!dataArray.length > 0) { + // console.log(`your data must be contained in an array in \n + // a property called 'entries' or 'data' of your json object`); + // } + + // //////////// + + // // set this.data.raw + // this.data.raw = result; + // } + + /** + * csvToJSON + * Creates a csv from a string + * @param {*} csv + */ + // via: http://techslides.com/convert-csv-to-json-in-javascript + // eslint-disable-next-line class-methods-use-this + csvToJSON(csv) { + // split the string by linebreak + const lines = csv.split("\n"); + const result = []; + // get the header row as an array + const headers = lines[0].split(","); + + // iterate through every row + for (let i = 1; i < lines.length; i += 1) { + // create a json object for each row + const row = {}; + // split the current line into an array + const currentline = lines[i].split(","); + + // for each header, create a key/value pair + headers.forEach((k, idx) => { + row[k] = currentline[idx]; + }); + // add this to the result array + result.push(row); + } + + return { + entries: result, + }; + } + + /** + * findEntries + * recursively attempt to find the entries + * or data array for the given json object + * @param {*} _data + */ + findEntries(_data) { + const parentCopy = Object.assign({}, _data); + + if (parentCopy.entries && parentCopy.entries instanceof Array) { + return parentCopy.entries; + } else if (parentCopy.data && parentCopy.data instanceof Array) { + return parentCopy.data; + } + + const keys = Object.keys(parentCopy); + // eslint-disable-next-line consistent-return + keys.forEach((k) => { + if (typeof parentCopy[k] === "object") { + return this.findEntries(parentCopy[k]); + } + }); + + return parentCopy; + } + + /** + * getData + * return data object's raw array + * to make getting raw data easier + */ + getData() { + const rawArray = this.data.raw; + return rawArray; + } +} + +export default TimeSeriesData; diff --git a/src/TimeSeries/timeSeriesUtils.js b/src/TimeSeries/timeSeriesUtils.js new file mode 100644 index 00000000..975679a6 --- /dev/null +++ b/src/TimeSeries/timeSeriesUtils.js @@ -0,0 +1,401 @@ +import { data, input } from "@tensorflow/tfjs"; +import nnUtils from "../NeuralNetwork/NeuralNetworkUtils"; + +class TimeSeriesUtils { + constructor(options) { + this.options = options || {}; + } + + /* adding data: can only accept the following formats: + - for xInputs: + 1. Sequence of objects (array of objects) + [{x: , y: },{x: , y: },{x: , y: },{x: , y: }] + 2. Sequence of arrays (array of array, order matters) + [[],[],[],[]] + */ + /** + * verifyAndFormatInputs + * @param {*} xInputs + * @param {*} options + * @param {*} classOptions + */ + verifyAndFormatInputs(xInputs, options = null, classOptions) { + const dataFormat = this.checkInputStructure(xInputs, options); + return this.formatInputsToObjects( + xInputs, + options, + classOptions, + dataFormat + ); + } + + checkInputStructure(xInputs, options = null) { + if (!Array.isArray(xInputs)) { + throw new error("Syntax Error: Data Should be in an Array"); + } + let isObjects = true; + let isArrays = true; + let isValues = true; + + for (let i = 0; i < xInputs.length; i++) { + if (nnUtils.getDataType(xInputs[i]) === "object") { + console.log("here"); + isArrays = false; + isValues = false; + if (i > 0) { + if ( + Object.keys(xInputs[i - 1]).length !== + Object.keys(xInputs[i]).length || + nnUtils.getDataType(xInputs[i - 1]) === "object" + ) { + throw new error("Data format is inconsistent"); + } + } + } else if (Array.isArray(xInputs[i])) { + console.log("here2"); + isObjects = false; + isValues = false; + if (i > 0) { + if ( + xInputs[i - 1].length !== xInputs[i].length || + !Array.isArray(xInputs[i - 1]) + ) { + throw new error("Data format is inconsistent"); + } + } + } else { + if (options.inputLabels) { + isObjects = false; + isArrays = false; + } else { + throw new error("inputLabels is needed for 1D array inputs"); + } + } + + if (isObjects) { + return "ObjectSequence"; + } else if (isArrays) { + return "ArraySequence"; + } else if (isValues) { + return "ValueSequence"; + } else { + throw new error("Syntax Error: Input Structure is unknown"); + } + } + } + + formatInputsToObjects(xInputs, options = null, classOptions, dataFormat) { + switch (dataFormat) { + case "ObjectSequence": + return xInputs; + case "ArraySequence": + return this.convertArraySequence(xInputs, options, classOptions); + case "ValueSequence": + return this.convertValueSequence(xInputs, options); + default: + throw new error("Input Data Structure is unknown"); + } + } + + convertArraySequence(xInputs, options = null, classOptions) { + let label = ""; + + if (options !== null) { + if (options.inputLabels) { + label = options.inputLabels; + console.log("here1"); + } + } else if (classOptions !== null) { + if (classOptions.inputs) { + label = classOptions.inputs; + } + } + + if ( + (typeof label === "string" && label === "") || + (Array.isArray(label) && label.length === 0) + ) { + label = this.getLabelFromNestedArray(xInputs); + } + + return xInputs.map((input) => { + const obj = {}; + input.forEach((value, ind) => { + obj[label[ind]] = value; + }); + return obj; + }); + } + + convertValueSequence(xInputs, options = null) { + const { inputLabels } = options; + if (xInputs.length % inputLabels.length !== 0) { + throw new error( + "Invalid Input: Number of Labels don't match amount of values" + ); + } + return xInputs.reduce((acc, _, index, array) => { + if (index % inputLabels.length === 0) { + // Create a new object for the current set of values + const obj = {}; + for (let i = 0; i < inputLabels.length; i++) { + obj[inputLabels[i]] = array[index + i]; + } + acc.push(obj); + } + return acc; + }, []); + } + + verifyAndFormatOutputs(yInputs, options = null, classOptions) { + const { outputs } = classOptions; + + let outputLabels; + + if (options !== null) { + if (options.outputLabels) { + outputLabels = options.outputLabels; + } + } + + if (outputs.length > 0) { + if (outputs.every((item) => typeof item === "string")) { + outputLabels = outputs; + } + } else if (typeof yInputs === "object") { + outputLabels = Object.keys(yInputs); + } else { + outputLabels = nnUtils.createLabelsFromArrayValues(yInputs, "output"); + } + + // Make sure that the inputLabels and outputLabels are arrays + if (!(outputLabels instanceof Array)) { + throw new Error("outputLabels must be an array"); + } + + return nnUtils.formatDataAsObject(yInputs, outputLabels); + } + + prepareLabels(xInputs, yInputs, options = null, classOptions) { + const { inputs, outputs } = this.options; + + let inputLabels; + let outputLabels; + + // options-based values to assign + if (options !== null) { + ({ inputLabels, outputLabels } = options); + } else if (inputs.length > 0 && outputs.length > 0) { + if (inputs.every((item) => typeof item === "string")) { + inputLabels = inputs; + } + if (outputs.every((item) => typeof item === "string")) { + outputLabels = outputs; + } + + // input-based values to assign + } else { + inputLabels = this.getLabelFromNestedArray(xInputs); + if (typeof yInputs === "object") { + outputLabels = Object.keys(yInputs); + } else { + inputLabels = this.getLabelFromNestedArray(yInputs); + } + } + + // Make sure that the inputLabels and outputLabels are arrays + if (!(inputLabels instanceof Array)) { + throw new Error("inputLabels must be an array"); + } + if (!(outputLabels instanceof Array)) { + throw new Error("outputLabels must be an array"); + } + + return inputLabels, outputLabels; + } + + getLabelFromNestedArray(xInputs, prefix = "label") { + // Recursive function to find the deepest level of the array + function traverseArray(array) { + if ( + array.length > 0 && + (typeof array[0] === "string" || typeof array[0] === "number") + ) { + return array.map((_, index) => `${prefix}_${index}`); + } else { + for (const item of array) { + if (Array.isArray(item)) { + const result = traverseArray(item); + if (result) return result; + } + } + } + return null; + } + + if (Array.isArray(xInputs)) { + return traverseArray(xInputs); + } else { + throw new Error("Input data must be an array."); + } + } + + // normalize utilities + reshapeTo3DArray(data, shape) { + const [batch, timeStep, feature] = shape; + let result = []; + let index = 0; + + for (let i = 0; i < batch; i++) { + let batchArray = []; + for (let j = 0; j < timeStep; j++) { + let timeStepArray = []; + for (let k = 0; k < feature; k++) { + timeStepArray.push(data[index]); + index++; + } + batchArray.push(timeStepArray); + } + result.push(batchArray); + } + + return result; + } + + zipArraySequence(arr1, arr2) { + if (arr1.length !== arr2.length) { + console.error("arrays do not have the same length"); + return []; + } + + return arr1.map((xs, idx) => { + const ys = arr2[idx].ys; // Extract the inner `ys` object + return { + xs: xs, + ys: ys, + }; + }); + } + + // point simplification utilities - Ramer-Douglas-Peucker (RDP) algorithm + padCoordinates(allPoints, targetPointCount, maxEpsilon = 50) { + const rdpPoints = []; + + const epsilon = this.findEpsilonForPointCount( + allPoints, + targetPointCount, + maxEpsilon + ); + + const total = allPoints.length; + const start = allPoints[0]; + const end = allPoints[total - 1]; + rdpPoints.push(start); + this.rdp(0, total - 1, allPoints, rdpPoints, epsilon); + rdpPoints.push(end); + + if (rdpPoints.length > targetPointCount) { + return rdpPoints.slice(0, targetPointCount); + } else if (rdpPoints.length < targetPointCount) { + const filler = new Array(targetPointCount - rdpPoints.length).fill( + rdpPoints[rdpPoints.length - 1] + ); + + rdpPoints.push(...filler); + return rdpPoints; + } + + return rdpPoints; + } + + findEpsilonForPointCount(points, targetCount, maxEpsilon) { + let low = 0; + let high = maxEpsilon; + let mid; + let simplifiedPointsCount = 0; + + while (high - low > 0.001) { + // Tolerance for approximation + mid = (low + high) / 2; + simplifiedPointsCount = this.getSimplifiedPointCount(points, mid); + if (simplifiedPointsCount > targetCount) { + low = mid; + } else { + high = mid; + } + } + + return mid; + } + + getSimplifiedPointCount(points, epsilon) { + const rdpPoints = []; + const total = points.length; + const start = points[0]; + const end = points[total - 1]; + rdpPoints.push(start); + this.rdp(0, total - 1, points, rdpPoints, epsilon); + rdpPoints.push(end); + return rdpPoints.length; + } + + rdp(startIndex, endIndex, allPoints, rdpPoints, epsilon) { + const nextIndex = this.findFurthest( + allPoints, + startIndex, + endIndex, + epsilon + ); + if (nextIndex > 0) { + if (startIndex != nextIndex) { + this.rdp(startIndex, nextIndex, allPoints, rdpPoints, epsilon); + } + rdpPoints.push(allPoints[nextIndex]); + if (endIndex != nextIndex) { + this.rdp(nextIndex, endIndex, allPoints, rdpPoints, epsilon); + } + } + } + + findFurthest(points, a, b, epsilon) { + let recordDistance = -1; + const start = points[a]; + const end = points[b]; + let furthestIndex = -1; + for (let i = a + 1; i < b; i++) { + const currentPoint = points[i]; + const d = this.lineDist(currentPoint, start, end); + if (d > recordDistance) { + recordDistance = d; + furthestIndex = i; + } + } + if (recordDistance > epsilon) { + return furthestIndex; + } else { + return -1; + } + } + + lineDist(c, a, b) { + const norm = this.scalarProjection(c, a, b); + return dist(c.x, c.y, norm.x, norm.y); + } + + scalarProjection(p, a, b) { + const ap = { x: p.x - a.x, y: p.y - a.y }; + const ab = { x: b.x - a.x, y: b.y - a.y }; + const abMag = Math.sqrt(ab.x * ab.x + ab.y * ab.y); + ab.x /= abMag; + ab.y /= abMag; + const dot = ap.x * ab.x + ap.y * ab.y; + return { x: a.x + ab.x * dot, y: a.y + ab.y * dot }; + } +} + +const timeSeriesUtils = () => { + const instance = new TimeSeriesUtils(); + return instance; +}; + +export default timeSeriesUtils(); diff --git a/src/index.js b/src/index.js index 09416c1c..cac3b86a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,5 @@ import neuralNetwork from "./NeuralNetwork"; +import timeSeries from "./TimeSeries"; import handPose from "./HandPose"; import sentiment from "./Sentiment"; import faceMesh from "./FaceMesh"; @@ -22,6 +23,7 @@ const withPreload = { neuralNetwork, sentiment, soundClassifier, + timeSeries, }; const ml5 = Object.assign({ p5Utils }, withPreload, { From b8f031e4eb3c37aa9f2de60c56c98085a2af58e8 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Fri, 4 Apr 2025 21:42:44 -0400 Subject: [PATCH 02/47] solved convergence issue with compile() --- src/TimeSeries/index.js | 143 ++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 87 deletions(-) diff --git a/src/TimeSeries/index.js b/src/TimeSeries/index.js index 1e931192..cbde1d7f 100644 --- a/src/TimeSeries/index.js +++ b/src/TimeSeries/index.js @@ -52,11 +52,29 @@ class DIYTimesSeries extends DiyNeuralNetwork { // 2. format the yInput - same logic as NN class const ys = tsUtils.verifyAndFormatOutputs(yInputs, options, this.options); - console.log(xs, ys); + // 3. add data to raw this.neuralNetworkData.addData(xs, ys); } + formatInputsForPredictionAll(_input) { + const { meta } = this.neuralNetworkData; + const inputHeaders = Object.keys(meta.inputs); + + const formatted_inputs = tsUtils.verifyAndFormatInputs( + _input, + null, + this.options + ); + const normalized_inputs = this.neuralNetworkData.normalizePredictData( + formatted_inputs, + meta.inputs + ); + const output = tf.tensor(normalized_inputs); + + return output; + } + createMetaData() { // this method does not get shape for images but instead for timesteps const { inputs } = this.options; @@ -242,95 +260,46 @@ class DIYTimesSeries extends DiyNeuralNetwork { } } - async loadDataFromUrl(dataUrl, inputs, outputs) { - let json; - let dataFromUrl; - try { - if (dataUrl.endsWith(".csv")) { - dataFromUrl = await this.neuralNetworkData.loadCSV( - dataUrl, - inputs, - outputs - ); - } else if (dataUrl.endsWith(".json")) { - dataFromUrl = await this.neuralNetworkData.loadJSON( - dataUrl, - inputs, - outputs - ); - } else if (dataUrl.includes("blob")) { - dataFromUrl = await this.loadBlob(dataUrl, inputs, outputs); - } else { - throw new Error("Not a valid data format. Must be csv or json"); - } - } catch (error) { - console.error(error); - throw new Error(error); + compile() { + const LEARNING_RATE = this.options.learningRate; + + let options = {}; + + if ( + this.options.task === "classification" || + this.options.task === "imageClassification" + ) { + options = { + loss: "categoricalCrossentropy", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; + } else if (this.options.task === "regression") { + options = { + loss: "meanSquaredError", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; } - dataFromUrl.map((item) => { - this.addData(item.xs, item.ys); - }); - - this.createMetaData(); - - this.prepareForTraining(); - } - - formatInputsForPredictionAll(_input) { - const { meta } = this.neuralNetworkData; - const inputHeaders = Object.keys(meta.inputs); - - const formatted_inputs = tsUtils.verifyAndFormatInputs( - _input, - null, - this.options - ); - const normalized_inputs = this.neuralNetworkData.normalizePredictData( - formatted_inputs, - meta.inputs - ); - const output = tf.tensor(normalized_inputs); - - return output; - } - - async classifyInternal(_input) { - const { meta } = this.neuralNetworkData; - const headers = Object.keys(meta.inputs); - - let inputData; - - inputData = this.formatInputsForPredictionAll(_input); - - const unformattedResults = await this.neuralNetwork.classify(inputData); - inputData.dispose(); - - if (meta !== null) { - const label = Object.keys(meta.outputs)[0]; - const vals = Object.entries(meta.outputs[label].legend); - - const formattedResults = unformattedResults.map((unformattedResult) => { - return vals - .map((item, idx) => { - return { - [item[0]]: unformattedResult[idx], - label: item[0], - confidence: unformattedResult[idx], - }; - }) - .sort((a, b) => b.confidence - a.confidence); - }); - - // return single array if the length is less than 2, - // otherwise return array of arrays - if (formattedResults.length < 2) { - return formattedResults[0]; - } - return formattedResults; + options.optimizer = options.optimizer + ? this.neuralNetwork.setOptimizerFunction( + LEARNING_RATE, + options.optimizer + ) + : this.neuralNetwork.setOptimizerFunction(LEARNING_RATE, tf.train.sgd); + + this.neuralNetwork.compile(options); + + // if debug mode is true, then show the model summary + if (this.options.debug) { + this.neuralNetworkVis.modelSummary( + { + name: "Model Summary", + }, + this.neuralNetwork.model + ); } - - return unformattedResults; } padCoordinates(coordinates, targetPointCount) { From bf67b69a97233774ecd97586a980edcec392fb2d Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Fri, 4 Apr 2025 23:06:14 -0400 Subject: [PATCH 03/47] instantiation code form TSDATA --- src/TimeSeries/index.js | 2 - src/TimeSeries/timeSeries.js | 251 -------------- src/TimeSeries/timeSeriesData.js | 572 +------------------------------ 3 files changed, 5 insertions(+), 820 deletions(-) delete mode 100644 src/TimeSeries/timeSeries.js diff --git a/src/TimeSeries/index.js b/src/TimeSeries/index.js index cbde1d7f..82d12a11 100644 --- a/src/TimeSeries/index.js +++ b/src/TimeSeries/index.js @@ -7,7 +7,6 @@ import setBackend from "../utils/setBackend"; import tsUtils from "./timeSeriesUtils"; -// import TimeSeries from "./timeSeries"; import TimeSeriesData from "./timeSeriesData"; // call an extension of DIY Neural Network as a new class, override select methods @@ -17,7 +16,6 @@ class DIYTimesSeries extends DiyNeuralNetwork { super( { ...options, - // neuralNetwork: null, neuralNetworkData: null, }, callback diff --git a/src/TimeSeries/timeSeries.js b/src/TimeSeries/timeSeries.js deleted file mode 100644 index 685e2831..00000000 --- a/src/TimeSeries/timeSeries.js +++ /dev/null @@ -1,251 +0,0 @@ -import * as tf from "@tensorflow/tfjs"; -import { saveBlob } from "../utils/io"; -import timeSeries from "."; - -/* - -Things changed from neural network class: - -1. No neuro evolution - -*/ - -class TimeSeries { - constructor() { - // flags - this.isTrained = false; - this.isCompiled = false; - this.isLayered = false; - /** - * @type {tf.Sequential | null} - the TensorFlow model - */ - this.model = null; - - // methods - this.init = this.init.bind(this); - this.createModel = this.createModel.bind(this); - this.addLayer = this.addLayer.bind(this); - this.compile = this.compile.bind(this); - this.setOptimizerFunction = this.setOptimizerFunction.bind(this); - this.train = this.train.bind(this); - this.predict = this.predict.bind(this); - this.classify = this.classify.bind(this); - this.save = this.save.bind(this); - this.load = this.load.bind(this); - - // initialize - this.init(); - } - - /** - * initialize with create model - */ - init() { - this.createModel(); - } - - /** - * creates a sequential model - * uses switch/case for potential future where different formats are supported - * @param {*} _type - */ - createModel(_type = "sequential") { - switch (_type.toLowerCase()) { - case "sequential": - this.model = tf.sequential(); - return this.model; - default: - this.model = tf.sequential(); - return this.model; - } - } - - /** - * add layer to the model - * if the model has 2 or more layers switch the isLayered flag - * @param {tf.Layer} layer - * @void - */ - addLayer(layer) { - this.model.add(layer); - - // check if it has at least an input and output layer - if (this.model.layers.length >= 2) { - this.isLayered = true; - } - } - - /** - * Compile the model - * if the model is compiled, set the isCompiled flag to true - * @param {*} _modelOptions - */ - compile(_modelOptions) { - this.model.compile(_modelOptions); - this.isCompiled = true; - } - - /** - * Set the optimizer function given the learning rate - * as a parameter - * @param {*} learningRate - * @param {*} optimizer - */ - setOptimizerFunction(learningRate, optimizer) { - return optimizer.call(this, learningRate); - } - - /** - * Train the model - * @param {Object} _options - */ - async train(_options) { - const TRAINING_OPTIONS = _options; - - const xs = TRAINING_OPTIONS.inputs; - const ys = TRAINING_OPTIONS.outputs; - console.log("train", xs, ys); - const { batchSize, epochs, shuffle, validationSplit, whileTraining } = - TRAINING_OPTIONS; - - await this.model.fit(xs, ys, { - batchSize, - epochs, - shuffle, - validationSplit, - callbacks: whileTraining, - }); - - xs.dispose(); - ys.dispose(); - - this.isTrained = true; - } - - /** - * returns the prediction as an array synchronously - * @param {*} _inputs - */ - predictSync(_inputs) { - const output = tf.tidy(() => { - return this.model.predict(_inputs); - }); - const result = output.arraySync(); - - output.dispose(); - _inputs.dispose(); - - return result; - } - - /** - * returns the prediction as an array - * @param {*} _inputs - */ - async predict(_inputs) { - const output = tf.tidy(() => { - return this.model.predict(_inputs); - }); - const result = await output.array(); - - output.dispose(); - _inputs.dispose(); - - return result; - } - - /** - * classify is the same as .predict() - * @param {*} _inputs - */ - async classify(_inputs) { - return this.predict(_inputs); - } - - /** - * classify is the same as .predict() - * @param {*} _inputs - */ - classifySync(_inputs) { - return this.predictSync(_inputs); - } - - // predictMultiple - // classifyMultiple - // are the same as .predict() - - /** - * save the model.json and the weights.bin files - * @param {string} modelName - * @return {Promise} - */ - async save(modelName = "model") { - await this.model.save( - tf.io.withSaveHandler(async (data) => { - this.weightsManifest = { - modelTopology: data.modelTopology, - weightsManifest: [ - { - paths: [`./${modelName}.weights.bin`], - weights: data.weightSpecs, - }, - ], - }; - console.log("data.weightData", data.weightData); - await saveBlob( - data.weightData, - `${modelName}.weights.bin`, - "application/octet-stream" - ); - console.log("this.weightsManifest", this.weightsManifest); - await saveBlob( - JSON.stringify(this.weightsManifest), - `${modelName}.json`, - "text/plain" - ); - }) - ); - } - - /** - * loads the model and weights - * @param {string | FileList | Object} filesOrPath - */ - async load(filesOrPath) { - if (filesOrPath instanceof FileList) { - const files = Array.from(filesOrPath); - // find the correct files - const model = files.find( - (file) => file.name.includes(".json") && !file.name.includes("_meta") - ); - const weights = files.find((file) => file.name.includes(".bin")); - // load the model - this.model = await tf.loadLayersModel( - tf.io.browserFiles([model, weights]) - ); - } else if (filesOrPath instanceof Object) { - this.model = await tf.loadLayersModel( - tf.io.http(filesOrPath.model, { - // Override the weights path from the JSON weightsManifest - weightUrlConverter: (weightFileName) => { - return filesOrPath.weights || weightFileName; - }, - }) - ); - } else { - this.model = await tf.loadLayersModel(filesOrPath); - } - - this.isCompiled = true; - this.isLayered = true; - this.isTrained = true; - } - - /** - * dispose and release the memory for the model - */ - dispose() { - this.model.dispose(); - } -} -export default TimeSeries; diff --git a/src/TimeSeries/timeSeriesData.js b/src/TimeSeries/timeSeriesData.js index f9b106c3..43746b88 100644 --- a/src/TimeSeries/timeSeriesData.js +++ b/src/TimeSeries/timeSeriesData.js @@ -6,72 +6,11 @@ import nnUtils from "../NeuralNetwork/NeuralNetworkUtils"; import tsUtils from "./timeSeriesUtils"; -class TimeSeriesData { - constructor() { - this.meta = { - inputUnits: null, // Number - outputUnits: null, // Number - // objects describing input/output data by property name - inputs: {}, // { name1: {dtype}, name2: {dtype} } - outputs: {}, // { name1: {dtype} } - isNormalized: false, // Boolean - keep this in meta for model saving/loading - }; - - this.isMetadataReady = false; - this.isWarmedUp = false; - - this.data = { - raw: [], // array of {xs:[{},{}], ys:{}} - }; - } +import NeuralNetworkData from "../NeuralNetwork/NeuralNetworkData"; - /** - * //////////////////////////////////////////////////////// - * Add Data - * //////////////////////////////////////////////////////// - */ - - /** - * Add Data - * @param {object} xInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array - * @param {*} yInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array - * @void - updates this.data - */ - addData(xInputObj, yInputObj) { - this.data.raw.push({ - xs: xInputObj, - ys: yInputObj, - }); - } - - /** - * //////////////////////////////////////////////////////// - * Summarize Data - * //////////////////////////////////////////////////////// - */ - - /** - * create the metadata from the data - * this covers: - * 1. getting the datatype from the data - * 2. getting the min and max from the data - * 3. getting the oneHot encoded values - * 4. getting the inputShape and outputUnits from the data - * @param {Array} [inputShape] - * @void - */ - createMetadata(inputShape = null) { - // get the data type for each property - this.getDTypesFromSeriesData(); - // get the stats - min, max - this.getDataStats(); - // onehot encode - this.getDataOneHot(); - // calculate the input units from the data - this.getDataUnits(inputShape); - // get the shape of batch - - this.isMetadataReady = true; +class TimeSeriesData extends NeuralNetworkData { + constructor() { + super(); } /** @@ -81,7 +20,7 @@ class TimeSeriesData { * @private * @void - updates this.meta */ - getDTypesFromSeriesData() { + getDTypesFromData() { const meta = { ...this.meta, inputs: {}, @@ -108,16 +47,6 @@ class TimeSeriesData { this.meta = meta; } - /** - * get stats about the data - * @private - * @void - */ - getDataStats() { - this.meta.inputs = this.getInputMetaStats(this.meta.inputs, "xs"); - this.meta.outputs = this.getInputMetaStats(this.meta.outputs, "ys"); - } - /** * get back the min and max of each label * @private @@ -153,133 +82,6 @@ class TimeSeriesData { return inputMeta; } - /** - * getDataOneHot - * creates onehot encodings for the input and outputs - * and adds them to the meta info - * @private - * @void - */ - getDataOneHot() { - this.meta.inputs = this.getInputMetaOneHot(this.meta.inputs, "xs"); - this.meta.outputs = this.getInputMetaOneHot(this.meta.outputs, "ys"); - } - - /** - * getOneHotMeta - * @param {Object} _inputsMeta - * @param {"xs" | "ys"} xsOrYs - * @return {Object} - */ - getInputMetaOneHot(_inputsMeta, xsOrYs) { - const inputsMeta = Object.assign({}, _inputsMeta); - - Object.entries(inputsMeta).forEach((arr) => { - // the key - const key = arr[0]; - // the value - const { dtype } = arr[1]; - - if (dtype === "string") { - const uniqueVals = [ - ...new Set(this.data.raw.map((obj) => obj[xsOrYs][key])), - ]; - const oneHotMeta = this.createOneHotEncodings(uniqueVals); - inputsMeta[key] = { - ...inputsMeta[key], - ...oneHotMeta, - }; - } - }); - return inputsMeta; - } - - /** - * get the data units, inputshape and output units - * @private - * @param {Array} arrayShape - * @void - */ - getDataUnits(arrayShape = null) { - // if the data has a shape pass it in - if (arrayShape) { - this.meta.inputUnits = arrayShape; - } else { - this.meta.inputUnits = [this.getInputMetaUnits(this.meta.inputs)].flat(); - } - - this.meta.outputUnits = this.getInputMetaUnits(this.meta.outputs); - } - - /** - * @private - * @param {Object} inputsMeta - * @return {number | Array} - */ - // eslint-disable-next-line class-methods-use-this - getInputMetaUnits(inputsMeta) { - let units = 0; - - Object.entries(inputsMeta).forEach((arr) => { - const { dtype } = arr[1]; - if (dtype === "number") { - units += 1; - } else if (dtype === "string") { - const { uniqueValues } = arr[1]; - - const uniqueCount = uniqueValues.length; - units += uniqueCount; - } else if (dtype === "array") { - // TODO: User must input the shape of the - // image size correctly. - units = []; - } - }); - - return units; - } - - /** - * Returns a legend mapping the - * data values to oneHot encoded values - * @private - * @param {Array} _uniqueValuesArray - * @return {Object} - */ - // eslint-disable-next-line class-methods-use-this, no-unused-vars - createOneHotEncodings(_uniqueValuesArray) { - return tf.tidy(() => { - const output = { - uniqueValues: _uniqueValuesArray, - legend: {}, - }; - - const uniqueVals = _uniqueValuesArray; // [...new Set(this.data.raw.map(obj => obj.xs[prop]))] - // get back values from 0 to the length of the uniqueVals array - const onehotValues = uniqueVals.map((item, idx) => idx); - // oneHot encode the values in the 1d tensor - const oneHotEncodedValues = tf.oneHot( - tf.tensor1d(onehotValues, "int32"), - uniqueVals.length - ); - // convert them from tensors back out to an array - const oneHotEncodedValuesArray = oneHotEncodedValues.arraySync(); - - // populate the legend with the key/values - uniqueVals.forEach((uVal, uIdx) => { - output.legend[uVal] = oneHotEncodedValuesArray[uIdx]; - }); - - return output; - }); - } - - /** - * //////////////////////////////////////////////////////// - * Tensor handling - * //////////////////////////////////////////////////////// - */ - /** * convertRawToTensors * converts array of {xs, ys} to tensors @@ -337,12 +139,6 @@ class TimeSeriesData { }); } - /** - * //////////////////////////////////////////////////////// - * data normalization / unnormalization - * //////////////////////////////////////////////////////// - */ - /** * normalize the dataRaw input * @return {Array} @@ -446,38 +242,6 @@ class TimeSeriesData { return output; } - /** - * normalizeArray - * @param {*} _input - * @param {*} _options - */ - // eslint-disable-next-line no-unused-vars, class-methods-use-this - normalizeArray(inputArray, options) { - const { min, max } = options; - - // if the data are onehot encoded, replace the string - // value with the onehot array - // if none exists, return the given value - if (options.legend) { - const normalized = inputArray.map((v) => { - return options.legend[v] ? options.legend[v] : v; - }); - return normalized; - } - - // if the dtype is a number - if (inputArray.every((v) => typeof v === "number")) { - const normalized = inputArray.map((v) => - nnUtils.normalizeValue(v, min, max) - ); - return normalized; - } - - // otherwise return the input array - // return inputArray; - throw new Error("error in inputArray of normalizeArray() function"); - } - normalizePredictData(dataRaw, inputOrOutputMeta) { const inputMeta = Object.assign({}, inputOrOutputMeta); const xsOrYs = "xs"; @@ -526,53 +290,6 @@ class TimeSeriesData { return output; } - /** - * unNormalizeArray - * @param {*} _input - * @param {*} _options - */ - // eslint-disable-next-line no-unused-vars, class-methods-use-this - // unnormalizeArray(inputArray, options) { - // const { min, max } = options; - - // // if the data is onehot encoded then remap the - // // values from those oneHot arrays - // if (options.legend) { - // const unnormalized = inputArray.map((v) => { - // let res; - // Object.entries(options.legend).forEach((item) => { - // const key = item[0]; - // const val = item[1]; - // const matches = v - // .map((num, idx) => num === val[idx]) - // .every((truthy) => truthy === true); - // if (matches) res = key; - // }); - // return res; - // }); - - // return unnormalized; - // } - - // // if the dtype is a number - // if (inputArray.every((v) => typeof v === "number")) { - // const unnormalized = inputArray.map((v) => - // nnUtils.unnormalizeValue(v, min, max) - // ); - // return unnormalized; - // } - - // // otherwise return the input array - // // return inputArray; - // throw new Error("error in inputArray of normalizeArray() function"); - // } - - /* - * //////////////////////////////////////////////// - * One hot encoding handling - * //////////////////////////////////////////////// - */ - /** * applyOneHotEncodingsToDataRaw * does not set this.data.raws @@ -610,36 +327,6 @@ class TimeSeriesData { return output; } - /** - * //////////////////////////////////////////////// - * saving / loading data - * //////////////////////////////////////////////// - */ - - /** - * Loads data from a URL using the appropriate function - * @param {*} dataUrl - * @param {*} inputs - * @param {*} outputs - * @void - */ - async loadDataFromUrl(dataUrl, inputs, outputs) { - try { - if (dataUrl.endsWith(".csv")) { - await this.loadCSV(dataUrl, inputs, outputs); - } else if (dataUrl.endsWith(".json")) { - await this.loadJSON(dataUrl, inputs, outputs); - } else if (dataUrl.includes("blob")) { - await this.loadBlob(dataUrl, inputs, outputs); - } else { - throw new Error("Not a valid data format. Must be csv or json"); - } - } catch (error) { - console.error(error); - throw new Error(error); - } - } - /** * loadJSON * @param {*} dataUrlOrJson @@ -689,255 +376,6 @@ class TimeSeriesData { throw new Error(err); } } - - /** - * loadBlob - * @param {*} dataUrlOrJson - * @param {*} inputLabels - * @param {*} outputLabels - * @void - */ - async loadBlob(dataUrlOrJson, inputLabels, outputLabels) { - try { - const { data } = await axios.get(dataUrlOrJson); - const text = data; // await data.text(); - - if (nnUtils.isJsonOrString(text)) { - const json = JSON.parse(text); - await this.loadJSON(json, inputLabels, outputLabels); - } else { - const json = this.csvToJSON(text); - await this.loadJSON(json, inputLabels, outputLabels); - } - } catch (err) { - console.log("mmm might be passing in a string or something!", err); - throw new Error(err); - } - } - - /** - * loadData from fileinput or path - * @param {string | FileList | Object} filesOrPath - * @return {Promise} - */ - async loadData(filesOrPath) { - try { - let loadedData; - - if (typeof filesOrPath !== "string") { - const file = filesOrPath[0]; - const fr = new FileReader(); - fr.readAsText(file); - if (file.name.includes(".json")) { - const temp = await file.text(); - loadedData = JSON.parse(temp); - } else { - console.log( - 'data must be a json object containing an array called "data" or "entries' - ); - } - } else { - loadedData = await axios.get(filesOrPath, { responseType: "text" }); - const text = JSON.stringify(loadedData.data); - if (nnUtils.isJsonOrString(text)) { - loadedData = JSON.parse(text); - console.log(loadedData); - } else { - console.log( - "Whoops! something went wrong. Either this kind of data is not supported yet or there is an issue with .loadData" - ); - } - } - - this.data.raw = this.findEntries(loadedData); - - // check if a data or entries property exists - if (!this.data.raw.length > 0) { - console.log( - 'data must be a json object containing an array called "data" ' - ); - } - } catch (error) { - throw new Error(error); - } - } - - /** - * saveData - * @param {string} [name] - * @return {Promise} - */ - async saveData(name) { - const today = new Date(); - const date = `${String(today.getFullYear())}-${String( - today.getMonth() + 1 - )}-${String(today.getDate())}`; - const time = `${String(today.getHours())}-${String( - today.getMinutes() - )}-${String(today.getSeconds())}`; - const datetime = `${date}_${time}`; - - let dataName = datetime; - if (name) dataName = name; - - const output = { - data: this.data.raw, - }; - - await saveBlob(JSON.stringify(output), `${dataName}.json`, "text/plain"); - } - - /** - * Saves metadata of the data - * @param {string} modelName - * @return {Promise} - */ - async saveMeta(modelName = "model") { - console.log("meta saved"); - await saveBlob( - JSON.stringify(this.meta), - `${modelName}_meta.json`, - "text/plain" - ); - } - - /** - * load a model and metadata - * @param {string | FileList | Object} filesOrPath - * @return {Promise} - */ - async loadMeta(filesOrPath) { - if (filesOrPath instanceof FileList) { - const file = Array.from(filesOrPath).find((file) => - file.name.includes("_meta.json") - ); - if (!file) { - console.warn("no model_meta.json file found in FileList"); - return; - } - const text = await file.text(); - this.meta = JSON.parse(text); - } else if (filesOrPath instanceof Object) { - // filesOrPath = {model: URL, metadata: URL, weights: URL} - const metadataResult = await axios.get(filesOrPath.metadata); - this.meta = metadataResult.data; - } else { - const loader = modelLoader(filesOrPath); - // TODO: it is not always "model_meta.json", it is "{model_name}_meta.json" - const metaPath = loader.getPath("model_meta.json"); - // TODO: figure out how to use loader.loadMetadataJson() which has wrapped error messages - const metadataResult = await axios.get(metaPath); - this.meta = metadataResult.data; - } - - this.isMetadataReady = true; - this.isWarmedUp = true; - } - - /* - * //////////////////////////////////////////////// - * data loading helpers - * //////////////////////////////////////////////// - */ - - /** - * // TODO: convert ys into strings, if the task is classification - // if (this.config.architecture.task === "classification" && typeof output.ys[prop] !== "string") { - // output.ys[prop] += ""; - // } - * formatRawData - * takes a json and set the this.data.raw - * @param {*} json - * @param {Array} inputLabels - * @param {Array} outputLabels - * @void - */ - // formatRawData(json, inputLabels, outputLabels) { - - // // Recurse through the json object to find - // // an array containing `entries` or `data` - // const dataArray = this.findEntries(json); - - // if (!dataArray.length > 0) { - // console.log(`your data must be contained in an array in \n - // a property called 'entries' or 'data' of your json object`); - // } - - // //////////// - - // // set this.data.raw - // this.data.raw = result; - // } - - /** - * csvToJSON - * Creates a csv from a string - * @param {*} csv - */ - // via: http://techslides.com/convert-csv-to-json-in-javascript - // eslint-disable-next-line class-methods-use-this - csvToJSON(csv) { - // split the string by linebreak - const lines = csv.split("\n"); - const result = []; - // get the header row as an array - const headers = lines[0].split(","); - - // iterate through every row - for (let i = 1; i < lines.length; i += 1) { - // create a json object for each row - const row = {}; - // split the current line into an array - const currentline = lines[i].split(","); - - // for each header, create a key/value pair - headers.forEach((k, idx) => { - row[k] = currentline[idx]; - }); - // add this to the result array - result.push(row); - } - - return { - entries: result, - }; - } - - /** - * findEntries - * recursively attempt to find the entries - * or data array for the given json object - * @param {*} _data - */ - findEntries(_data) { - const parentCopy = Object.assign({}, _data); - - if (parentCopy.entries && parentCopy.entries instanceof Array) { - return parentCopy.entries; - } else if (parentCopy.data && parentCopy.data instanceof Array) { - return parentCopy.data; - } - - const keys = Object.keys(parentCopy); - // eslint-disable-next-line consistent-return - keys.forEach((k) => { - if (typeof parentCopy[k] === "object") { - return this.findEntries(parentCopy[k]); - } - }); - - return parentCopy; - } - - /** - * getData - * return data object's raw array - * to make getting raw data easier - */ - getData() { - const rawArray = this.data.raw; - return rawArray; - } } export default TimeSeriesData; From 1228a59e6d7737597cffcf005e785121508da51e Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Wed, 7 May 2025 15:37:13 -0400 Subject: [PATCH 04/47] added previous examples and refactored layers --- .../index.html | 10 +- examples/timeSeries-hand-gestures/sketch.js | 192 +++++++++++++++++ .../timeSeries-instantiate-example/sketch.js | 94 -------- .../index.html | 45 ++++ .../model/model.json | 1 + .../model/model.weights.bin | Bin 0 -> 17128 bytes .../model/model_meta.json | 1 + .../sketch.js | 131 +++++++++++ .../timeSeries-train-mouse-gesture/sketch.js | 204 ------------------ .../index.html | 21 +- .../timeSeries-weather-prediction/sketch.js | 183 ++++++++++++++++ .../weather_data.json | 196 +++++++++++++++++ src/TimeSeries/index.js | 165 ++------------ src/TimeSeries/tsLayers.js | 152 +++++++++++++ 14 files changed, 927 insertions(+), 468 deletions(-) rename examples/{timeSeries-instantiate-example => timeSeries-hand-gestures}/index.html (76%) create mode 100644 examples/timeSeries-hand-gestures/sketch.js delete mode 100644 examples/timeSeries-instantiate-example/sketch.js create mode 100644 examples/timeSeries-load-model-hand-gestures/index.html create mode 100644 examples/timeSeries-load-model-hand-gestures/model/model.json create mode 100644 examples/timeSeries-load-model-hand-gestures/model/model.weights.bin create mode 100644 examples/timeSeries-load-model-hand-gestures/model/model_meta.json create mode 100644 examples/timeSeries-load-model-hand-gestures/sketch.js delete mode 100644 examples/timeSeries-train-mouse-gesture/sketch.js rename examples/{timeSeries-train-mouse-gesture => timeSeries-weather-prediction}/index.html (50%) create mode 100644 examples/timeSeries-weather-prediction/sketch.js create mode 100644 examples/timeSeries-weather-prediction/weather_data.json create mode 100644 src/TimeSeries/tsLayers.js diff --git a/examples/timeSeries-instantiate-example/index.html b/examples/timeSeries-hand-gestures/index.html similarity index 76% rename from examples/timeSeries-instantiate-example/index.html rename to examples/timeSeries-hand-gestures/index.html index c406a501..90d238c2 100644 --- a/examples/timeSeries-instantiate-example/index.html +++ b/examples/timeSeries-hand-gestures/index.html @@ -2,20 +2,20 @@ 👋 Hello! This is an ml5.js example made and shared with ❤️. Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - - This example demonstrates training a mouse gesture classifier with ml5.neuralNetwork. + + This example demonstrates training a Sign Language classifier through ml5.TimeSeries. --> - - + - ml5.js neuralNetwork Gesture Classifier Example + ml5.js Time Series Hand Gesture Train and Save + diff --git a/examples/timeSeries-hand-gestures/sketch.js b/examples/timeSeries-hand-gestures/sketch.js new file mode 100644 index 00000000..c95392ff --- /dev/null +++ b/examples/timeSeries-hand-gestures/sketch.js @@ -0,0 +1,192 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates training a Hand Gesture classifier through ml5.TimeSeries. + */ + +let seqLength = 50; + +let handPose; +let video; + +let hands = []; +let sequence = []; + +let recordingFinished = false; +let predictedWord = ""; + +// UI variables +let trainingWords = {}; + +function preload() { + // Load the handPose model + handPose = ml5.handPose(); + + // setup the timeseries neural network + let options = { + outputs: ["label"], + task: "classification", + dataMode: "spatial", + debug: "true", + learningRate: 0.001, + }; + model = ml5.timeSeries(options); +} + +function setup() { + createCanvas(640, 480); + + // setup video capture + video = createCapture(VIDEO); + video.size(640, 480); + video.hide(); + + // place UI elements + UI(); + + // use handpose model on video + handPose.detectStart(video, gotHands); +} + +function draw() { + // draw video on frame + image(video, 0, 0, width, height); + + drawPredictedWord(); + + // if hands are found then start recording + if (hands.length > 0 && recordingFinished == false) { + if (sequence.length <= seqLength) { + // get coordinates from hands (21 points) + handpoints = drawPoints(); + sequence.push(handpoints); + + // once sequence reaches the seqLength, add sequence as just one X value + } else if (sequence.length > 0) { + // get the training word from the input box + let train_word = nameField.value(); + + // if there is a word currently in the box then add data with that label + if (train_word.length > 0) { + // add data to the model + let target = { label: train_word }; + model.addData(sequence, target); + trainingWordsUpdate(); + + // if there is no word in the box then classify instead + } else { + // classify the data + model.classify(sequence, gotResults); + } + + // reset the sequence + sequence = []; + recordingFinished = true; + } + + // can only record again when hand is out of frame + } else { + if (hands.length == 0) { + recordingFinished = false; + } + } +} + +function drawPoints() { + let handpoints = []; + // iterate through both hands + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + for (let j = 0; j < hand.keypoints.length; j++) { + // access the keypoints in the hand + let keypoint = hand.keypoints[j]; + handpoints.push(keypoint.x, keypoint.y); + + fill(0, 255, 0); + noStroke(); + circle(keypoint.x, keypoint.y, 5); + } + } + // assign to a different variable before clearing + let output = handpoints; + handpoints = []; + + return output; +} + +// Callback function for when handPose outputs data +function gotHands(results) { + // save the output to the hands variable + hands = results; +} + +function trainModelAndSave() { + model.normalizeData(); + let options = { + epochs: 100, + }; + model.train(options, whileTraining, finishedTraining); + nameField.value(""); +} + +function whileTraining(epoch) { + console.log(epoch); +} + +function finishedTraining() { + console.log("finished training."); + model.save("model"); +} + +function gotResults(results) { + predictedWord = results[0].label; + console.log(predictedWord); + text(predictedWord, 200, 200); +} + +function UI() { + nameField = createInput(""); + nameField.attribute("placeholder", "Type the word to train"); + nameField.position(110, 500); + nameField.size(250); + + instructionP = createP( + 'I want to train:

1.) Type any word you want to pair with a gesture, e.g. "HELLO"
2.) Do the gesture associated to the word, make sure to do it until the points disappear.
3.) Move your hand out of the frame and repeat the gesture, do this multiple times
4.) Do the same for other words e.g. "BYE"
5.) Once all data is collected, press Train and Save

Tip: have at least 5 datasets for each word' + ); + instructionP.style("width", "640px"); + dataCountsP = createP("-> After the gesture a tally will appear here <-"); + + train_but = createButton("Train and Save"); + train_but.mouseClicked(trainModelAndSave); + train_but.style("font-family", "Georgia"); + train_but.style("font-size", "20px"); + train_but.position(500, 490); +} + +function drawPredictedWord() { + textSize(100); + fill(255); + text(predictedWord, 100, height / 2); +} + +function trainingWordsUpdate() { + let tempWord = nameField.value(); + console.log(Object.keys(trainingWords)); + if (!(tempWord in trainingWords)) { + trainingWords[tempWord] = 1; + } else { + trainingWords[tempWord]++; + } + + let counts = ""; + let keys = Object.keys(trainingWords); + console.log("keys", keys); + + for (let k of keys) { + counts += k + " : " + trainingWords[k] + "
"; + } + + dataCountsP.html(counts); +} diff --git a/examples/timeSeries-instantiate-example/sketch.js b/examples/timeSeries-instantiate-example/sketch.js deleted file mode 100644 index fcd0e8d9..00000000 --- a/examples/timeSeries-instantiate-example/sketch.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * 👋 Hello! This is an ml5.js example made and shared with ❤️. - * Learn more about the ml5.js project: https://ml5js.org/ - * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - * - * This example demonstrates training a mouse gesture classifier with ml5.neuralNetwork. - */ - -// Step 1: load data or create some data -let data = [ - { x: 0.99, y: 0.02, label: "right" }, - { x: 0.76, y: -0.1, label: "right" }, - { x: -1.0, y: 0.12, label: "left" }, - { x: -0.9, y: -0.1, label: "left" }, - { x: 0.02, y: 0.98, label: "down" }, - { x: -0.2, y: 0.75, label: "down" }, - { x: 0.01, y: -0.9, label: "up" }, - { x: -0.1, y: -0.8, label: "up" }, -]; - -let classifer; -let label = "training"; - -let start, end; - -function setup() { - createCanvas(640, 240); - // For this example to work across all browsers - // "webgl" or "cpu" needs to be set as the backend - ml5.setBackend("webgl"); - - // Step 2: set your neural network options - let options = { - task: "classification", - debug: true, - }; - - // Step 3: initialize your neural network - classifier = ml5.timeSeries(options); - - // Step 4: add data to the neural network - for (let i = 0; i < data.length; i++) { - let item = data[i]; - let inputs = [item.x, item.y]; - let outputs = [item.label]; - classifier.addData(inputs, outputs); - } - - // Step 5: normalize your data; - classifier.normalizeData(); - - // Step 6: train your neural network - classifier.train({ epochs: 100 }, finishedTraining); -} -// Step 7: use the trained model -function finishedTraining() { - label = "ready"; -} - -// Step 8: make a classification - -function draw() { - background(200); - textAlign(CENTER, CENTER); - textSize(64); - text(label, width / 2, height / 2); - if (start && end) { - strokeWeight(8); - line(start.x, start.y, end.x, end.y); - } -} - -function mousePressed() { - start = createVector(mouseX, mouseY); - end = createVector(mouseX, mouseY); -} - -function mouseDragged() { - end = createVector(mouseX, mouseY); -} - -function mouseReleased() { - let dir = p5.Vector.sub(end, start); - dir.normalize(); - let inputs = [dir.x, dir.y]; - console.log(inputs); - classifier.classify(inputs, gotResults); -} - -// Step 9: define a function to handle the results of your classification -function gotResults(results) { - label = results[0].label; - console.log(results); -} diff --git a/examples/timeSeries-load-model-hand-gestures/index.html b/examples/timeSeries-load-model-hand-gestures/index.html new file mode 100644 index 00000000..92363d69 --- /dev/null +++ b/examples/timeSeries-load-model-hand-gestures/index.html @@ -0,0 +1,45 @@ + + + + + + + + ml5.js Time Series Hand Gesture load model + + + + + + +
+

+ This example loads a model that is trained with ASL hand gestures for + Hello and Goodbye.
+
+ + Instructions:
+ 1.) Use one hand to do a gesture in front of the camera
+ 2.) Wait for the points to disappear or the prediction appears on + screen
+ 3.) To predict again, remove your hands in the frame and do the gesture + again

+ + How to do gestures for Hello and Goodbye in ASL:
+ Hello: + https://babysignlanguage.com/dictionary/hello/
+ Goodbye: + https://babysignlanguage.com/dictionary/goodbye/
+

+ + diff --git a/examples/timeSeries-load-model-hand-gestures/model/model.json b/examples/timeSeries-load-model-hand-gestures/model/model.json new file mode 100644 index 00000000..ad7c44f6 --- /dev/null +++ b/examples/timeSeries-load-model-hand-gestures/model/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.8.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./hello.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,42,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[176,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/examples/timeSeries-load-model-hand-gestures/model/model.weights.bin b/examples/timeSeries-load-model-hand-gestures/model/model.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..e57f181601f525897b348ac9ca8a03203e2c4420 GIT binary patch literal 17128 zcmW)nc{r8d*TxNH9ztX&MI=IKFr2+MDWnXkR2np@Xdp$?w<1Z>V8~FS(1a9`;_S6a zQc{MZD20+n(ws=|`Mvwk^T)Za=bZiAYpr{I?xH1@@F>s_eE;Ud7`T9&Y7f)#z1{JZ*iy5Yac_;Qpo zOb=G2GJV!`pHnW`nGuOw9LMte{a%79xga3p-?3ln=`ekLCs}aT7y3?7IOIPT{91Hq zTksnks?diPQTg!kBp30IQZjdtg3sDhLo z)PGfjl*kO6@xqNxel3Y}D%7CWI09F(Jg~pJ1(tN>vrPxhaOoC9Fs{_&5zeXb)*)N4+OSh@UJ} z-7XI|btK5B50mL&ZVFVq%7NZ`6+X1!0JyBuqMyEfspCRQesUzqf*K zxobICQ<~qi?gCqtW1zBBhh|?egW$y#P$IFFj#w$n6W2aq&I+6Hk#Q5c zE9ugOdxvrK(}Qr;MVM7mjStpn;=h`Sv@(7V_*FabvUBd#h1o!<>oK-qt_zg>+sXzW z=u?xp?ReVjF=|Oy32N2Cp?}z7n3^2MK93GW*}a2sQc4VQ=tWqyCYns_yN*Y@W-^27 zDKvKUT6(hVJsT6Z5c)Oz$t0CDxOmNoKe=d&DspyIDl#3`>v2d5xd-jB-I#moD_f*y zz`MSlW!^K#;wj|`JbJ-o=>03thXrn@*PxY+y}trqeQLqL|E@u>`wuvE-x!wvO~41- zfvWv?gnkwDvc%W7V02m!(Q2@P|Ew=SbMq&5XVG}r)%TsD{2y4GJc36~`HA2Et)BBf#2?H)m)Rt)+* zI!@Y7jN%g|W7+i?OZYaswfHdh3^>G%;U8><6Ve`z`+w}F&nvhv;DQMswpfq)SH$Cm zPajz7%Oc2-n?ap-%w+Taj;Bss8~^i^;)na6!Vr}rw(sH=TK-`V|8@NYIbbl6j(clM z{@G^2kDnUcGdGq@RXBn#pU8sa?p&5XVG{i}_6{V1armYX=m+pP2 zMfJm{bD#BEyi3~{Hu4U9w6~9_@>Te5d=M@!3&B6W`h3f}1X7SZ30!pL`P{7%Bx~eN z;OpDiL@!ldUfaNC|8|3tQPucM;TX*D$tRYP68uwr2Rs_B&DTgo!qd)z+^C+HOoAh76~@-w-YqEMUr)V`0@NeY(G=4X!?%jU~6v3+p;wzz~^t|1IEZZVTo-8jLJMlWZW*nm%hi~ zIaUWMe-%Q)t5mE@ih^xX@1V-PPPkzA4Q!rbLC+SA zJ$a86$Hw8g1X;cyyAu*?KY%gUK@Ybi(%|Zf3yxS3SJxosc=RIr`=1j`KV(KXSwF;} znNi@MGn&2jR_AUTr1<-Ndi-*qBJ~RDhWnvK=p#RgE`B|Qp4MMS2hI6yzVD&i6?+;vwref%o!N^xD^C+`D!VE59R5;p8&7SHDIvY9zW*J;1-^rz#~4D#Hg+1g=_TLlD0b-x92R;7*~Uz1=`qr z_*H4ZIt?26>m_>2d2sV!Yd){$1TNRG!RLKTg&pa~pvKgT_tpB~2CEXBw_k!bH@grG zzs2;KwFDb%52Ag~w$c@j37VMCHS#<5oGDHE@Ij68uo}O(VL4Lh|#sP!l=D^bmHBEV5pQ0v)FHVqMl3^Neba} zR|G5ze9eS=-2}7W&qJRZ(J+!+gO`q~yymS0J=If#Sxvocy4w=il`#P(&hCK|3;&j<=;opmKZ}T zUz~)jnfIY>b27Z%w-JVKj%GtvQ=D#VMKwRR;=u3*_;7eCTK8Urko0Qd=EZTS6Lbvk zyt~D`U%o}XtFEAKSw$ZIn8PE2efgVp$!L)y&)4iwg=N*J;Xik2+A~jZw6i==--nqe3$2qKgT&Vh}R5o`~y!iP8X;KBD%`0)&(k6O)z z)3#m3&sR6llUo{DZsJXLrLmZNYu4r;`|O$Is`*?)SDZIIw&KQq+922Jh(7l<>8Mxv zD8G9@X8#$=`_f*+q*>XhJ<}Vj<<1DTQuIk+fH%IW$-@tmMfJe%Ck}}nLoTN`lK!7E z%=2?UHg1;&qs(gjG50vZ)4F zTfc$H&o7aG!A@-G?XN^;*KnSs6NdUNCeYSnL=VV)A=yKUpmC%kqtC9nz%c$E0d)BmE6OY+$;8Q-#;hWU|fZ3Mibe@bH zu!qfd?0-1T7Y{m&W3X5TiCwu4@l}HfV7-0Rdcm~r>zH= zg-H(V=r-p6D!V{s7soB>hA8#=3_d@vOSdc-PIDw{SZ_=|SZ%%{7%^fKNWYY&vksaN z@#jhKFX@`_>J=rpkQ9x#p3Q7t++D%~Qw0xx&f#G@ zRj_)xE|of3fPMZWI6u}0*Y?k%MO9xg8{UKEw<$0x{V44sQI37iT!%Z3LZrMvF!)(4gnP?1pt|9Q8hAqPF`*TsM3N8$$9?qv!dlLx@MyZwaQG2os~|@2)=uE-m7{UXZ5!GXwO^&4{0)>|8$ ztc6&AY8kEnmx6a1WVyRHg|0FS2&?{r&rY4gq0W=&liD)!x4I8vuGvtz8~H>+iEzU` zX9XH7r||C?+O*2N8QqsH!;ovoVEHW}l;_cSz&n3=MJ{5*nuZi z51_cSBQyDZ6OUKyf)4j(ur%@yJFgf`F3TIy^&dURX}SVlLJwQBNSUwba^b$u-{QFs zlGx_I8n&)m3Cv3o+5T2Axqb$fU(e+?jq*{OiBtL64zMRpoTf?=`uIf_98NkZoZxg1 z3kMsS*Ul>3*?$G(77T|VYh7+JQj)s7zk(rUs&HoVTCyvR@F&Awpv_hn+HEi%vqAz$ z)(mle*X$gu%N_y}&7av1{sLY;)WBQA>X;*cTRJ%VBF0ZLfL>uVTh!nW=F083xI%~j zcSDbQ?DN3e5gcN7+0w^30gyC*9LMkG6~4xxc*&U1=rFQ^%(fzt!7`gRlpl_QeZg7cb=R zcgoTWC#tc0q$(}pGx!cC8UAA9DE>5hAa zWU#jZl=t}KHND>`tEmcXv8aCfj-{JN^uqmP6RBH8GiGM1Q-eF%DEDbRJl?+w8$4Ii z`T-LzKl>Nt4~pjP^d#`<&lSF(v4V(+&dmP3Wc$T>LbQ5IPDBeymx~R1I%OhJJ}t?m zhQ1}UZrrD;Hh~!W;Sluoc~Qe(v+?AMw;&cNAVi zx+T%r$xm6z$IrN{WG;6*A;s$@XF^EA6!@d84qv|2!Vs5fe6~^sDe@M=P!)HsF!g{y zPPmIt#)-TjK$ce|W#Xmzt6_V^6Vfrbik61hz;f>->V7B=7sri&H7}RK#BI5-rRq5> zzUD@!*on0EhZ9Y`wH^ZIzJ|MY>a??SI`|Jt(vXiFb-ko`VBJaTeDn}kUgyJ~-`L8` zK3WQj9j)j~?I_4ni=k-VLoEMUaKnY}RAG~lY#!~xGmKOrM{F}S_BEv!bj8r8+lS70 zdKcrq1dHz2LT++04a&|)@$#2S{KCZqXgt@=O1~|jkMc$d6n~85r^4iU&heG_H82~~ z|Lvj1PG@0me>A=G)e$}0|DpODMecSjhjxvANss(~1Y;i+<3G8tq|yHqT+XZ$3Xhcu z&jqaEw>C%fpI_Wi+h8mmqF9M5Kj)#t;v(UMz4Oqc<~SWb=QUIGzJisuDI_6cA8zp5 zz}tTvf@^{I$obeQkXOBfF8y_dHcRah_F7HG2BXzb8kvf=Y1RS>w_!v{Pt+GU&*yQO zMxa&p4xBHaV%xEe^l0uUD5qmY zUO{NlQDQnlK&Rj4urAS#q^^0z*zFWjdwe|xIV_|F9}xRvtT9w@6PMQIv9ogy!PRhW zK74{XFc`@y62{ZIOE%+1iF=a7**CovU0s}i;O*QVN=Q?(V>$=XW}{IuE-{J8QuocK1ECazr1Usn#JyM}M2Ys8n+$=j?+po0YU z%U8$gMm9JiW(DsvEFzzBg}BMjm&?Qt<^4Ad@fdF?J;Vmlw|^+TSMv=YK03lZeRPH1 zGfwmRvT*8@F^Lvs^uyV3A3kqk9A8_x64$qffq8Q8yr>F$+1VOYwbF!&$-0QzYDojuCzgnS-+sU86CV^#J`v2FC z|9$qqQ&px$}3m8=@wJDZ#I2{w@E2J=)6md+S9YNwQsbNWAEVB!a7gjueL!7;{ zpy#2I@Vk65GfDXg#_k_r&f#>lHjtzrGy`#I`4=+Yx*s;0_>rpLB5x{pEKBZ>$4nJt z@^)-kYETFt}$fgl3c-~g8isulF6dqWU$KXA8?Y+Ez}&5f^%jp03J7t2Odg<)qV23 zEl>v&)(i?~ty&1T_Gr?P>PTd7_OO3#pD^>D5q)L_M#UDT^_4*D5ebKBUl)KJS4hmH}$tw&5T#cUrKBtF2a*DG+6@<@Kd z-hn}$BDGSyg3(@?OlLXe#$)91k{Dw@W!kaW$_cAydjsCxgQf+U5PoYn47?}@y@!$b z=GAPFoM%m6XR4431^!SfI}2Ph58#NQov2o}0LN}Of_p|9T>5G^l2@v9jWdUaja|&f zuoSHt6|i-_6cC>!xV^p>Vm`RS%&9JLYS=lDld$H6jY2T7?#3NaM@Vtm2+VDog$sK2 z!`tn>f+Np7Ag)Xsm83QC_D3Nkm?W{3!%y)@R0Q$I0zqAsESk8O(3?M^Kqp3;RzEF= z0l#Z3GQg1h?tg|pc88$QZwiljr-0LHPQe2^d4A>Tb*2)bz{|y_<3GzJv|N4^G`1VV zJqI18AsGew?=O&Vi`sBwt2G@MBSt$OInbp`AHq1H4QUkh_v1c@iM+P)eCft!U_O#C z%6AG4?3hU9b31X7Mk>z8ccD2CwQ1VYI_$Yt32Ssxprqpt$c~ZbE7Wt~zvX6BeRL%e zq(s2LH!)Gqb_-(Hzd*fkYgilWjekJr2?jSX^PCR9Odhl8c?jY0U+}kX zI2rFQ#mmAa@nhCBx+qqSdgq_O(pwVrq|GNXkRii*T$>=ITasG7I!&s2Zm=as^Mvya7EvD*#3iEm$xwo&`G`C)4(+lVd$HeA5UU z9A~Wv%TKvMMv5-%4>X|K3*InQ6;ry3+yl!EWso&b6P2#~hJaxOWK)k9@qN6KZMSOx z$2B|Q%*L%;-oUr-+qM^94=zJ&YXl<3zy)gX9vOU%T9Wo*@9uz%I^orUY<(r-hGFLX`79GhryfJ7-E)oSLBsd3i4+>M%C^J z;-RaAi(A~#YrYfR)7n6O#wU}e8*wOmM2Wi|a9|#}?+{#f;M2%gB<86k3N&L-^;k6a zFIa}{odm3#tHAT+Buts2L*oyxgNE|wQ1WCkG<}MN+s3-MOW_I@-87>6f|?nInDcvc zDslTq9ipAHzvTGTSa!HOpB4Iw=HY)!1ksvSQOywEy&g_7j zX(wRo)~v*8|=nKqIsg0(D~G9P;+{S<*^g7OiqFuf2bmbmm=7pffXLv zcn8kD$YzJv#z9HT0Bn#s4`!7+arW2ASf+82bhgC_qKkD{@6HFf{hlV+-hBoRmoo9P z+Yr{!XU4alsb#}%IpYWUConJ63pzS(!TQ_2m?Zxbj;;9v5ARiDnP{Fh%bia3&Gw>H zk_|VNm8ae{pOE~N!Y^~@&|e)vczo#%>{l@&2b|>a&G{aDG37EbeJ3f@zIc)SJog0? zw~R$;|5-To&3`by+nYAeA^2{kJoMzxpa~b}gVwp-croif>IJCKp|jn=qIDdf@a!Rk zi8P>Tk}H?KF%*xk5+l-H14PyKBiPvvh4?*cR8Ogmw9Yk!?tM1|hw{$i*{6SDP(hbE zor=J+CGT*sHC1qNi5wkOHV+=joWa$Pn=vN-GqlVQ&9=Ma$kd0$*qJbvd~`Noflv?X z=Mv$q?H=LBp$f1da|e2V?ZYL0)?g9j1Cz}nF?mM?YIYfU;L)y4b|f?G$Vq< zJb48wp~HFED@(fn=UYe}qeo9#MU&MEJ*3CeABBFcc7A{Cv9QIG@1Gb9N1V*z-%mq2 zJKG+=+v)MXFlX-k{63bJcwv3A1n(ba3!Oj3AVTdETlD=TTm1GI&L~??OdZn6e@9m{ z{i#yW{5zFq)q-jA4)L6{$*45T0X*1Ck@@v_C2|Vxuh_=sSuMhmEk| zd@Oqrug(pJDL}WcBNZz@0m?)4>36l+_|8w6U6ye{^RE$jdQUTx&9}u{+v}NJyE>h5 zqYW$O$HHfyonSF=5wBZR3LeLPg4Sy@@$d2qoc>=Jj2o}ZYdWvO;nM)-yBcB5cz3FK zJ_$|Y=LpA3zGePz&FIppWBDVaKz6fOh9BRvn|Qcc!Fxdh#4s&x(-A}N{Pd(vlBe+d z9s}-hU7dz&wvkya$q>G&6wW^R0jJYcc}fc4SB(*UN{}FfjJ(#BsiEl zAJ-}#f{}y0(6#P4c69>`xqgfMIKK^4FX`X`@krdEF$;Da_JhmABxzpc5-@R<ll{bE?a4+X4zUCz3Sd!cUpR5qvT8m4^9CnLAo^BK1z zh^y^*D*5Xth|hlln;rbwP4#$qeasQIwcm!P<>u^~sVPm3ae;#1nV`5CK*RJ9e7NL9 zkH=NwMuY2cNMj7_*om0q5mXu%Y)d9@&%~Dh9O+$KRW@h8G7VqRKsNnlLJz}lf-N7F zc<;dwHrj47ymN_1I4@DUiE8pI)6(JcdmFy}Z8eh^B2Hrr$Dp}LL+z?_;nS1=wrSS^ z8a<)_w@oWz>G72Q@xBFx^KzKvS~)w5CnI?7<3Kd?c`AHjdj?-VI|XBPnEG8=~?1pDg%LEe)TFN>Ra>vl(;Ou(wqQ;rG!>5@IkBgYDDt zbnJ7oM7n^O|0%^p+cywBH2uPr&z&7L_@cL(&|c5V`w$)OSuEcDcX9fJ!my zdw(SveS8EjwqFN#D$kMCK6zv+Q{t!33?usc&fw7d6kV^MfKzpvaC#BK*I^#Kxy=YD5E;%elan5oGEr?{226G|XFd0r}kK@K(i+IYwGxi{M68`?~1*-of zP@*D%_?bG>ac4U4>1_&*m!0XlWLMDJ?SjcI&8+R_Awfg)IIbuw&fm3^lD`vvvmASW z_Dswc_Q&;sHhqX0qf+2-M+BL?{~ywIvta9)D98}|PV|?af^|oy(_g`5xVj(~KhKqc z+9moh?`{w({Cz}TTCAo~c8dY081SNcb8ZkRN$V}-x%6<#Jv*alp+Os@Xx3qs^<9jw zFUQRWk*vq<2%EC~3QRcYkFF|1_&|XwfA1qr#WK#p#yOfC9xTGCy5aC5_Mb4rCIlXu z09|Eu8F%L2!!jSnnhM9$|CCkex=9XveEVK3e>xo>+1CpLob2Jk1T_%6ljrlVUqaUn zZ$RAX8hRcXOJCm(6LhTGjhCwP@IrYZ%(s_;Xe%|^eP$eY8!ACVyDD*9-gU@T+Jd2r zuGoEgqCoTgCWAGd$RA7E;^wYcJe3-Tk_%Po%YQ!vhfMq6R+I)9AJ2i6@^9htRH0DH z!;S}8Xu;g;8Sr03G(JD~3FmHH0_=1Y-nUDG(cdIc@x^7R_!GvK&DkmJu#*LRunOCr zrvfY2IVFOut*-Vm`~(eV3SkDn|a19k_GgMsWE-1SF_7Wx-4C7BV}ys-h|8a-)g*$Cb# zBgJQSSVH9W%Mk8677qCyf?pSx!mNAR>|@<$JThoW(%p_g@&31Xx8WO%eiDmnqmszx z4fn|4#Y}8IFa;J&7)>U=FNQt-4=_pLJo8V{^iPo5Zi?$O4`YJTC#YH2 zfFIB8W77}zVDI*M^wmTiZj|+%EMK;Ox{e=5E4D~dq3%fV`uPFX*AxlzHnkBo!Bf0f zFp0jgoi3VL9O(E?JAg%vFzTQX^sgSkBToil#gd`4%W4o)W!(6}0}Q9Qeu6>^MmoM^ zuzkfsd^ut}HTCg@XZ>gKn65I5?oEK2pL@vF*A7C7fit+x)SV2S_XmcUB!P871zLUh z1LKmk`QXZ1EOn{|&7W`|%0zrrQtCwB{x1#O&Mty^$p@hPEQS5L#ZW%W1khz69VMkp z#6))Aq4B44_0I8V_UI(|-FbnY+A5H$FooPJP$owtN@0<|6wlC*=ZYui zK*c$ZKKd_kL(5}`J_s<}VFFY?pM-tuenIhuC^~*#B_6AJkK&11^y0j6up*{|L=^qT zEfH_Qv-2DFJh{(y7mLBJ(Q7f=J`ZxbHnTHhrlC!y7GJ;3fgW6ap6#4G6m+DA!JF=8 zw1)yTYf#~5!}>{TkRG~M?#DG#V%YUdTHw6nJ!uuqY2w#8@oR)n{KQ4G`7SUw@5LQ1N63S_dok{BB8%8PlO${zjfzdW z{F|LLE;wL@%UAuveGi8c6Q@{~wb_9d%~;PvO?8AHp0*Lzfi~gT5b9P+O2hicUnc;Q_y(qsN;5UZ_joIN0&QD;0S3 zvK@VzUJL_xpVTUa;Da1<5*xLau9p_j2gC_-t9|+B_x8~HbQ^tJbrd~TMj{??py|~% zaI{?;MvHpk^ogHvmuMH|EWLsSNbdx-*rP0A_H`njt3!_-y8=i4PUJ^*_o4j`J!*UC z05&;Gb5@is%qVrImkO3~|M+1)5wb zu>xFX>;t)wDln-o7HGQ{vb$qcxYy?m_;od~p!9_(D-_ z42c=Xv!nW;zCTjf=90;d`K|+@+Ix6&yN9gqyF=8TW}QF^0OiN+IrkHN}J5)rWm<-Bkb_eq#^^I ze0y025xdp+j;?deF)js1rAX7?6)T|nTRS=YV+^(3Yf6tAc=F$)o8ilrC_$3gJG|aL z9lkDnjB}*#f%b!5!Sc~LSmR#{iz?Nqe7!QiQ7b`d?q_%xbDjOyJcL%758=-zjpTyr zOQnHIJ6P-6pWvUWLd!(Gis`K*aA@lYy7<9Exc5$*F3hrq0h>xpJ-C-#v7U^+@*mOb z@)3+mx&tpC|Az~1MMJ4eK0eJa0ROsE(7NL`+%9wCFE_8n#0n2uYki4$`;Wxyv;`H_ z*1{W^vHZxJ1cAwASNd8#o+bA##I(KFaa7|aa;(q^T`q-X2oHKy&uY^ZX)Q`X(av%*`;!+O*lENk-HM>Uwn@>=Z{qPzi!L{E z*nrQ9>I9b}(#ZHuf97M93Wn{{7=s2o>{a{@NAVZkLZqHZ={?9xPZI<4T&esLaPV8GWW zn!}qCB`RsC$IFIyGvV2EREnqtYq+wr>>1_Xv*Z6NPL4n`Gj zp#P3ep;;Sj`RCE5ymOTm&41X>bS5;BJ>QbB!EZKQ^vW4(XAXmdZzZX_=@6QAIuebB zwc}2cC}@5*ifUY;DD&edZWEtM_dWjrHQQVzQ^)|j#+~pd$Z#3ku{i{(p7z>R?HX?r11J4DvquW7G zGA7Fs!fiD86=y?QyI8cpNi@MRsTEN7?04zWC(-zR@F;c!NMj~bL$h~&_}CX=jL}5) zIxrOdM{Xdmm$nLY-b?^L{{Y<8J`%+?P>``|0~K2_x;*Rxrn}t~EZC<;qbn!VPP1Eh zu52z}IZK6V70iOpZgXD1M9(-k4$^nY@Z6MM7#dAzUcqPfSKJe7vxnl!`BF@BPb^9P zDnlhbr_-@6r6Mk(sC1>9Cj@?yLz~;7@UJupPTm{C%+df{*QP_|8bS@q-op0{vp?ZJg~Q-BWCJz1?uzLO-55MsK%dX^;G5TH zgZC0`-lQ;@|1x_Hbea(UiF|_&#TZC9nF!;)t^%DXWpXF&04Be92Wt{!Xf{US(#4q& zm~#y*McUoHy?{)wNx8F1SW`isEG%9n>a#PR;Mu9;dFttD*!}MjR@^yE#wwapuPO_!I%g|Z8|MiQ$Vk&c zzf?HrcoThP$MCbFU1j**dt`6eaqzh*&z)E~c0w(ziH?K4vz*C`s3a)p%pa`?0&`f;Gd#&$bNB-9UOfdx}J`veo%pFt>z@)@k5Ze&|;g6 z7U8vE6WXP=0+XsLm~pN;S!25vex0uuJ{}TEO1`#=IJQctd;N@gPtfDqmp;Sd(1SQE zXB7EmB*AZ;h{F9>6!1!u3sv?K;1u(j=za#^OPr2inrtu`8I!`=LoNB6dr~xRTqLyY zie@Q!uKcfvpBNrK4RjaJe+E{a_A7_V* zf_@7((2yPr2Oqq{6AJ>^5TP2Kw=Ko)_ZSnr-xUCx!$(nPngj2Al=*or32NZ%NKgLx zhP~OwJo=O#)%`P;tFzza)NLn9U#F9giz2b1-jf*IiG-1wAK;MiyKs-jDi9Ox!g3x4 zf~SN6sBI5|-tf_QVaiV^?hIppo{R>Q-0d*zbrx*Q9!V2hH`D7at0)-PlP8s<`0WX@ zaP04E_`7@|L|ste8+TQ~1rR`)}pgTW$?hG`GFGpjAuW%@A zE;fptB693Gp3*ys3)`glcC|Ms^~#k#PHKYF31+Nlj{~;a9vA%nDM{iYKHF97Q^ON$ z(=cq&Axtz#BaXEh5LtH(*XXS*-8adT{D|zpo2^Lv=R3f%tU54j$`b6~E>HbBWvOL` z1I_+DmFARA$9QWma5F0uf|VMqs@7*`Pl@IW-?bPXZ3RJ&!Pxm#7vF@Z!JTbK$oRS# zjOeK*9S_AZXs#$j6r)7vZW+b*dY=ce1Y6LXaTLxgcQK9kpHM4oki5Dsz$3n`ur*PW z)&{m<+Okuqm~2R&`P_l5mof0gYaOoFI1J{=doj#82g+C2aJvl|!fJsr-M>Bq|Lu&& z_iHR+)5UDuE7EVJCBx~st6!j4e<~~tnF}Uq0?=Pt3iDT(;eV6Od70WbpjUOM?$dUm zc$hl`-~RwoMTWd&u?)~52&v_lP)m4)IO)6)Wtp_;=#+7AHPDgH4z3f(bRNK6TZ@Tb zT(@AAy&-jMRN@LH67=+JcY5mQM(X(54$p}G7uGKX=39@O!cx%e?P~h?s3{$JMV2MY zG+^w4Z)AeaV9EAnpUL=Z`D~Y~D>a%<`J5A7_$6r}6eu5pXcu>;t9=a@R^+qU3!V7i z?mukF+GS+beiQsYB9rCqy#OA^U8&VxNj^909)9yM6cm3;hoD!Y9xp{4?J~pZo#s*W zlbRT+4k-n5^(nA)qBif(egx5>btEy~n>P%MCJ?jAnmafFW35sa%IgB<8%+k z$Q@(Nh8lcTL?l(4GJtW5%JKSQSFAqgNu}x@;)Jzg#5iXe11D?j9N|k9r~9FM{9Qp@ z;RT%KQzhb97hsi^815V@OaG-!6Q*o!LE+jD0(B8zp}6x6Lw*#J7GDCvHYw;!J&hlt zw}YL6Hq8oZhYM;)aWJY@#OX%Dj}^7x5qul=w09zn`-v+84B@`+DkvHwfsVye=N#m1dFAIMh)8gzRofUA z?LW+YT^_@&H09FO>Bq5l@B?0XWJ%pGOvIt@7ja)l8+h?_8GSkL9(vx+!bc$*5caYG z_RmU(H*F$Tpk9|R+x`HG#Z%y$mnHQzpTgu1Z^53EYA_{C3A8r678Z6(@Vn)|!QIpw zdWT(xO*5it?(gw*bEgxFB>VW>$6?g|ts4Du(;M}SCh^yEhJ#jS6DtS_AufZttfyx+ z`y$E^Nj*1Uk2}Id*`QglPyG&@yRi+%-|~mmA5M}%aXotZo)~^~y3W=|%mssuzj1HG zY0$m!0O~~iw)uBiw(+|p>3n&eG|lqI?DR)?XT(;lsi=VsFaF>XWe2#iH5nW>T*Jrm z$~e9GDw%wzgGhCJCbjXNWaF7PEIqpsejJHs-SgF9#%^ae@2Mk;JbDAeM;r%z0VDH# zhtbP7^WnMVMbKF+3!5ziaQ%ubIG|@OSQ(mZ*Yp0V@DPM!`1PystME7ZcxR+=LhpAd zQ=d$N2gcJet0efPn;v92+X%Nd4v_rQKgdUOO`Kkkg)Ixu!g7%>(s%qQ_)EWsoX1nR z8%(0T&#sZnNq13h-#y!am~bj-IT36<6zExtH@MTOo)vIK@DV$KQ@mopQB0o{PN>D$ zapl;%KZzNrEhEWEFQ9+qXezG!1y9SRlHlA&ctYAz^XFH2`tw05*9njz?@^pQM)z^{wx1P4%v+p@xc$+6d8t> z-|LbR(VP)Au^yyp|sWv+rJkii|#(Y~=?(;}y8Qr6L`6un454 zP7r#S$FW5{4iGC7HQ?gUMNIyTiPUhqI#t{y4km9_(yV(7ibVe1Ugb25-<$zWXGM4T zt^`)VcFuF{Xh2vvm~LFjqx6Q;=Z}2p^Okw|B5?_nvL?Z6$>G#4ZUCO=-)C+IR|>VR znF@mLuEe1=A0hBU3GQ)QOS69JLXEf&&Xt|YU7B>+54AZ$&3GlcTDSydYwn`yk^!jI z&SxXd1E9-S3r$*e;FHWqJLRjfu&=%ddchQ2%p`?w&#$u(2`g?M>PB<*C-SX^FHl`e zj2ewiguF+8p?pa)3;lZ=hMMWoNmhoa?=8x5bd<9AnIoaF!keX(m&8 z1NW?tfjo;YxE$w2PCh%%3z;_eUB3!~a@FWFuW$JE=6Y@`Qx3K_pJC9MK<-!d9fzv@ zN0Z_npiAadI@Q9JDs(M}t`kYn@G2Z;@7a%iPezf;Ik{l@wF21#Y2=?MeOy*eTGax` z$J3gyCbAhO{Jtsj=i`NB>``d6z9G=L5CvTu&CtJEjPuO>^s$OP4Sm#z2FlMcOG=pr zq@08;ovZCaEquW^U?E+$<19aM+aKp|VIX}!2M;TY(*yNFa#A^oh-por#_}RYeXbQv z+8ux`V}FzNcW3gj+OweJdJU!aRg_kh%F`(_^B~mwJk$E3j>?L1B>dJay5!JF2wY?h z@?odJT$FFQw{irRgSU)mqi|uJuF%` zg3X@3j)@;#OT3NLXlY3(TQL}pf?y+5`!@~tP8)$YG?((_FUN}RcnA&zy@O4QR=|a9 zW$60T3dfCaV$#Ncr16^{n<}gZh8Ruug#PXmiG`Nn-du3JD%1k z)Z*fmmh`3FS$MJHFewm$9dlFRj7m* z=Ct0%Ka=KR<>_G}Hvcj%bWVc+qpPqvv=h3D=fXY78!+*~S9aE_5En$4QpHO@q5fbc z^A%r!nl3vbG^mzv;{w6?I#Ir*J&+3QXW-c*di=Ir8{70V7(#aidvzPpNQVtV%0JHjeiB!zhh7a%bu-naw{wG%q_O}I4 z>uf=VTXrzZ)uBu>%Yfd~yM;$c8xH&Og}4%V797+ES#2#iX~az4Z!m)TdoRUR1vS_p zu1X*AW7vBs34RFTap9z^_7kXT12T^){& zKO_0tlGCi^;TCH3@Un=PGbSG&&xB(g-on!l$7A)u4Dc*6p?w=W;mv9}{B+5RG&qlf z!w014C1o-26xGe9$Sne&#e2|Xf)ibG>mVAd=yR1nuV7)G7NlB!VA_+s`8LxIV)Is; zdcFR|zW54I*)$nk-$XNMgLZUWECq&Jvf=JCQ3kIng{;mtp`BG?lvOxT3F%jmn`S_V zS=O-oQ;pf-C2?S-kjtE=g$nk6un=TL9)q@obJ!H1z_WZ?$qNq?L5Oc8yuGBreZD7x z&p`_;5%G4R{lkgY_d(G45r{5-F2T#XldSDZ81}7oXTvg$h;VHVd9Bq3#d)Lol?xJ} zsV_%IU)v3S=e*(e0~_oPZ9%d9XN8*#wC#H5J%M|%_tCEI2{w*NhPvYS@F7r(OJ@CF z01^W2{ro#knUg*0=Muhb*XF%^!j3)K7hFAqH2FOkJ0!ljl@z`*+Uh;#b5_0I3q`#~ z5h%Vz9}m352GG4Aw2Zt6`vE?GfrdK;9`n758^FB;U#PuMtTsM<9@IVjO<2AhukO9z zgv34jih@2i&1}7V%D6q*kr%y^VXwSJ8rr=_OY}VrzE->wA@aS!y*Ip_8wS1~H`%)l zbCo^5;axo>pkzHyC7`|fy+yo!^(#J8!5lt=yo|kbK%YG*hwDAu|3tmMAp<^14s*S+ zGuFLeS@ylk4`n_{1(80hfh@jJI_bRt*sncQm^(hGQw6;p_!zz=uwy>Py^lS4Nj|<5 z_#8g^^U%DV{usWnh%Y_Uk9a+Xin6@yrjNZ2ln6e9&VfEDHXy!ige1KZT&X=3(5bzn zD-yoU!D&5R;pM$wNN>Km@~u4^yn8+;Fd95R#1K8+a0xtAFxxvPNB=vHnRh(t5{5fT zwvRk=@|-(P4yn9$WllT*0000S6h{^-70iy6GgpxnL*?oK}F zJ1o8TO~<~a8sk0+pNPJ?s$RZOYC%0FWdOh0bEQ70;2b{~e2hK(Ob|cFC$T;BAJM+s zoiROr!vDQBcLBbw8TUS#>?ps>0X4pIp-H}6tT4W`cBMXQ!Yn-tUG2VB?6W;p;i9~Y Sdi}j8ejY#HA~?I>A~-v7%Qid! literal 0 HcmV?d00001 diff --git a/examples/timeSeries-load-model-hand-gestures/model/model_meta.json b/examples/timeSeries-load-model-hand-gestures/model/model_meta.json new file mode 100644 index 00000000..1c0165c7 --- /dev/null +++ b/examples/timeSeries-load-model-hand-gestures/model/model_meta.json @@ -0,0 +1 @@ +{"inputUnits":[42],"outputUnits":2,"inputs":{"label_0":{"dtype":"number","min":4.151249399907168,"max":586.4725394909854},"label_1":{"dtype":"number","min":186.47223882383636,"max":496.34918695509003},"label_2":{"dtype":"number","min":12.818880217505907,"max":564.7860747522525},"label_3":{"dtype":"number","min":160.9460986889124,"max":478.89482602620234},"label_4":{"dtype":"number","min":20.681431005110262,"max":557.1173870582799},"label_5":{"dtype":"number","min":135.1274696802808,"max":454.0862355189599},"label_6":{"dtype":"number","min":29.375938053231934,"max":562.4826339023859},"label_7":{"dtype":"number","min":113.22511415628927,"max":455.15365538508894},"label_8":{"dtype":"number","min":37.27265551578051,"max":573.3838980891996},"label_9":{"dtype":"number","min":98.00531862273047,"max":473.4382341601794},"label_10":{"dtype":"number","min":2.706973037101564,"max":599.2858408346702},"label_11":{"dtype":"number","min":117.7350326456234,"max":453.76022921684716},"label_12":{"dtype":"number","min":11.635752695869659,"max":612.8243751678727},"label_13":{"dtype":"number","min":91.05094143918305,"max":481.6467136241304},"label_14":{"dtype":"number","min":22.9353041163117,"max":621.0127886598051},"label_15":{"dtype":"number","min":61.619264849841635,"max":499.63536096409143},"label_16":{"dtype":"number","min":33.53953084457643,"max":626.4181148091915},"label_17":{"dtype":"number","min":28.455718477478662,"max":512.7953875856006},"label_18":{"dtype":"number","min":-2.8065139589559984,"max":617.7828981986556},"label_19":{"dtype":"number","min":117.6886729722432,"max":459.5357193516273},"label_20":{"dtype":"number","min":3.7782929928570064,"max":633.7038985044576},"label_21":{"dtype":"number","min":86.77279076496669,"max":486.0751342925063},"label_22":{"dtype":"number","min":16.177018651157255,"max":642.8366376068107},"label_23":{"dtype":"number","min":51.687144639081325,"max":502.64037741142846},"label_24":{"dtype":"number","min":28.1461509145229,"max":650.2419536370577},"label_25":{"dtype":"number","min":15.922382743702723,"max":516.9301399988833},"label_26":{"dtype":"number","min":-6.382516546058305,"max":630.7077663350849},"label_27":{"dtype":"number","min":120.16376158664924,"max":461.0881814514869},"label_28":{"dtype":"number","min":-1.4074379536407533,"max":647.5041251714117},"label_29":{"dtype":"number","min":90.58035685591811,"max":485.04491883378125},"label_30":{"dtype":"number","min":10.174906800459325,"max":658.4893875478738},"label_31":{"dtype":"number","min":71.76407331703523,"max":500.55112323964187},"label_32":{"dtype":"number","min":21.11718120932074,"max":668.566957655395},"label_33":{"dtype":"number","min":39.557348432978586,"max":514.4287318106208},"label_34":{"dtype":"number","min":-7.9534800405596595,"max":641.3232619371444},"label_35":{"dtype":"number","min":126.31599791044414,"max":465.6320514399833},"label_36":{"dtype":"number","min":-3.8369034650104927,"max":658.2044139172733},"label_37":{"dtype":"number","min":103.73604938021917,"max":481.03793223993495},"label_38":{"dtype":"number","min":3.7075645592075435,"max":668.8017566330357},"label_39":{"dtype":"number","min":88.76136006394765,"max":494.63688258092407},"label_40":{"dtype":"number","min":6.9609311353376135,"max":676.9525074586147},"label_41":{"dtype":"number","min":75.97401514052241,"max":506.7948506427954}},"outputs":{"label":{"dtype":"string","min":0,"max":1,"uniqueValues":["hello","bye"],"legend":{"hello":[1,0],"bye":[0,1]}}},"isNormalized":true,"seriesShape":[51,42]} \ No newline at end of file diff --git a/examples/timeSeries-load-model-hand-gestures/sketch.js b/examples/timeSeries-load-model-hand-gestures/sketch.js new file mode 100644 index 00000000..11ab6076 --- /dev/null +++ b/examples/timeSeries-load-model-hand-gestures/sketch.js @@ -0,0 +1,131 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates loading a Hand Gesture classifier through ml5.TimeSeries. + * This example is trained with the ASL gestures for Hello and Goodbye + * + * Reference to sign hello and goodbye in ASL: + * Hello: https://babysignlanguage.com/dictionary/hello/ + * Goodbye: https://babysignlanguage.com/dictionary/goodbye/ + */ + +// change this to make the recording longer +let seqLength = 50; + +let handPose; +let video; +let hands = []; +let sequence = []; +let recordingFinished = false; +let predictedWord = ""; + +function preload() { + // Load the handPose model + handPose = ml5.handPose(); + + // setup the timeseries neural network + let options = { + task: "classification", + dataMode: "spatial", + }; + + model = ml5.timeSeries(options); +} + +function setup() { + let canvas = createCanvas(640, 480); + canvas.parent("canvasDiv"); + + // create video capture + video = createCapture(VIDEO); + video.size(640, 480); + video.hide(); + + handPose.detectStart(video, gotHands); + + // setup the model files to load + let modelDetails = { + model: "model/model.json", + metadata: "model/model_meta.json", + weights: "model/model.weights.bin", + }; + + // load the model and call modelLoaded once finished + model.load(modelDetails, modelLoaded); +} +// call back for load model +function modelLoaded() { + console.log("model loaded!"); +} + +function draw() { + // draw video on the canvas + image(video, 0, 0, width, height); + + // put the text on screen after a prediction + placePredictedText(); + + // if hands are found then start recording + if (hands.length > 0 && recordingFinished == false) { + if (sequence.length <= seqLength) { + // get coordinates from hands (21 points) + handpoints = drawPoints(); + sequence.push(handpoints); + + // once sequence reaches the seqLength, add sequence as just one X value + } else if (sequence.length > 0) { + // classify based on the collected data + model.classify(sequence, gotResults); + + // reset the sequence + sequence = []; + recordingFinished = true; + } + + // can only record again when hand is out of frame + } else { + if (hands.length == 0) { + recordingFinished = false; + } + } +} + +// draw the points on the hands +function drawPoints() { + let handpoints = []; + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + fill(0, 255, 0); + noStroke(); + circle(keypoint.x, keypoint.y, 5); + handpoints.push(keypoint.x, keypoint.y); + } + } + let output = handpoints; + handpoints = []; + return output; +} + +// Callback function for when handPose outputs data +function gotHands(results) { + // save the output to the hands variable + hands = results; +} + +// call back for accessing the results +function gotResults(results) { + predictedWord = results[0].label; + console.log(predictedWord); + text(predictedWord, 100, 100); +} + +// for drawing text on screen +function placePredictedText() { + textSize(100); + fill(255); + text(predictedWord, 100, height / 2); +} diff --git a/examples/timeSeries-train-mouse-gesture/sketch.js b/examples/timeSeries-train-mouse-gesture/sketch.js deleted file mode 100644 index c0139c65..00000000 --- a/examples/timeSeries-train-mouse-gesture/sketch.js +++ /dev/null @@ -1,204 +0,0 @@ -/* - * 👋 Hello! This is an ml5.js example made and shared with ❤️. - * Learn more about the ml5.js project: https://ml5js.org/ - * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - * - * This example demonstrates How to train your own mouse gesture classifier through ml5.TimeSeries. - */ - -let model; -let counts = { - circleDataCount: 0, - squareDataCount: 0, -}; -let currShape = "circle"; -let state = "collection"; - -let pressedOnce = true; -let frameCount = 0; -let datapoints; -let sequence = []; -let recCircle, recSquare, trainBut; - -// Training Data lenghts -let ink_multiplier = 3; -let num_seq = 20; - -function preload() { - let options = { - inputs: ["x", "y"], - outputs: ["label"], - task: "classification", - spatialData: "true", - debug: "true", - learningRate: 0.005, - }; - - model = ml5.timeSeries(options); -} - -function setup() { - // p5 js elements - let canvas = createCanvas(600, 400); - canvas.parent("canvasDiv"); - background(220); - UI(); - - // set framerate to constant rate for constant data collection - frameRate(60); -} - -function draw() { - // record data when the mouse is pressed inside the canvas - if (mouseIsPressed && pressedOnce && mouseY < 400 && mouseX < 600) { - // draw lines through coordinates - line(pmouseX, pmouseY, mouseX, mouseY); - - frameCount++; - - let inputs = { x: mouseX, y: mouseY }; - - sequence.push(inputs); - - if (sequence.length == num_seq * ink_multiplier) { - pressedOnce = false; - frameCount = 0; - - // if state is collection, add whole sequence as X, and shape as Y - if (state == "collection") { - let target = { label: currShape }; - model.addData(sequence, target); - - // add to the count for each - counts[currShape + "DataCount"] += 1; - console.log(counts); - updateDataCountUI(); - - // reset the screen - background(220); - textSize(20); - fill(0); - text("Recording: " + currShape, 50, 50); - // if prediction, classify using the whole sequence - } else if (state == "prediction") { - model.classify(sequence, gotResults); - - background(220); - } - - // reset the sequence - sequence = []; - } - } - inkBar(); -} - -function trainModel() { - // normalize Data first before Training - model.normalizeData(); - - // set the number of epochs for training - let options = { - epochs: 40, - }; - model.train(options, whileTraining, finishedTraining); - - background(220); - state = "training"; - text("Training...", 50, 50); - recCircle.style("background-color", ""); - recSquare.style("background-color", ""); - trainBut.style("background-color", "#f0f0f0"); -} - -function whileTraining(epoch, loss) { - console.log(epoch); -} - -function finishedTraining() { - background(220); - text("Training Finished, Draw again to predict", 50, 50); - state = "prediction"; -} - -function gotResults(results) { - let label = results[0].label; - - fill(0); - text("Prediction: " + label, 50, 50); -} - -// code to signify drawing can be done again -function mouseReleased() { - pressedOnce = true; -} - -////////////// UI Elements //////////// - -// code to visualize how much ink left -function inkBar() { - datapoints = map(frameCount, 0, ink_multiplier * num_seq, 0, num_seq); - - bar_height = 250; - height_miltiplier = bar_height / num_seq; - push(); - fill(0); - textSize(15); - text("Ink:", 550, 90); - rect(550, 100, 25, num_seq * height_miltiplier); - fill(255); - rect(550, 100, 25, datapoints * height_miltiplier); - pop(); -} - -// code for UI elements such as buttons -function UI() { - textSize(20); - - recCircle = select("#recCircle"); - recSquare = select("#recSquare"); - trainBut = select("#trainBut"); - - recCircle.mouseClicked(recordCircle); - recCircle.style("background-color", "#f0f0f0"); - recSquare.mouseClicked(recordSquare); - trainBut.mouseClicked(trainModel); - - function recordCircle() { - state = "collection"; - currShape = "circle"; - - background(220); - text("Recording: circle", 50, 50); - recCircle.style("background-color", "#f0f0f0"); - recSquare.style("background-color", ""); - trainBut.style("background-color", ""); - } - - function recordSquare() { - state = "collection"; - currShape = "square"; - - background(220); - text("Recording: square", 50, 50); - recCircle.style("background-color", ""); - recSquare.style("background-color", "#f0f0f0"); - trainBut.style("background-color", ""); - } - dataCountsP = createP( - "circle data: " + - counts.circleDataCount + - "
square data: " + - counts.squareDataCount - ); -} - -// Update the HTML UI with the current data counts -function updateDataCountUI() { - dataCountsP.html( - "circle data: " + - counts.circleDataCount + - "
square data: " + - counts.squareDataCount - ); -} diff --git a/examples/timeSeries-train-mouse-gesture/index.html b/examples/timeSeries-weather-prediction/index.html similarity index 50% rename from examples/timeSeries-train-mouse-gesture/index.html rename to examples/timeSeries-weather-prediction/index.html index acdfde38..72a3ffdd 100644 --- a/examples/timeSeries-train-mouse-gesture/index.html +++ b/examples/timeSeries-weather-prediction/index.html @@ -3,7 +3,7 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates How to train your own quickdraw classifier through ml5.TimeSeries. + This example demonstrates Training a Stock Price Predictor through ml5.TimeSeries. --> @@ -11,27 +11,18 @@ - ml5.js Time Series Train Mouse Gesture classifier Example + ml5.js Time Series Weather Prediction Example -
- - - -

- Instructions:
- 1.) Press the "Record Circle" or "Record Square" and start drawing until - the ink runs out
- 2.) Draw multiple times for each shape
2.) Press "Train" and wait for - training to finish
- 3.) Draw again to predict drawn shape

- Tip: Collect at least 5 drawings for each: -

+
+ + +
diff --git a/examples/timeSeries-hand-gestures/sketch.js b/examples/timeSeries-hand-gestures/sketch.js index 6474dc32..cf6952a4 100644 --- a/examples/timeSeries-hand-gestures/sketch.js +++ b/examples/timeSeries-hand-gestures/sketch.js @@ -6,19 +6,18 @@ * This example demonstrates training a Hand Gesture classifier through ml5.TimeSeries. */ -let seqLength = 50; - -let handPose; let video; - +let handPose; let hands = []; + let sequence = []; +let targetLength = 30; -let recordingFinished = false; -let predictedWord = ""; +let gestures = ["Gesture #1", "Gesture #2"]; +let counts = { "Gesture #1": 0, "Gesture #2": 0 }; -// UI variables -let trainingWords = {}; +let state = "collection"; +let currGesture = gestures[0]; //set currGesture to gesture 1 by default function preload() { // Load the handPose model @@ -36,14 +35,15 @@ function preload() { } function setup() { - createCanvas(640, 480); + let canvas = createCanvas(640, 480); + canvas.parent("canvasDiv"); // setup video capture video = createCapture(VIDEO); video.size(640, 480); video.hide(); - // place UI elements + // Setup the UI buttons for training UI(); // use handpose model on video @@ -54,46 +54,73 @@ function draw() { // draw video on frame image(video, 0, 0, width, height); - drawPredictedWord(); - - // if hands are found then start recording - if (hands.length > 0 && recordingFinished == false) { - if (sequence.length <= seqLength) { - // get coordinates from hands (21 points) - handpoints = drawPoints(); - sequence.push(handpoints); - - // once sequence reaches the seqLength, add sequence as just one X value - } else if (sequence.length > 0) { - // get the training word from the input box - let trainWord = nameField.value(); - - // if there is a word currently in the box then add data with that label - if (trainWord.length > 0) { - // add data to the model - let target = { label: trainWord }; - model.addData(sequence, target); - trainingWordsUpdate(); - - // if there is no word in the box then classify instead - } else { - // classify the data - model.classify(sequence, gotResults); - } - - // reset the sequence - sequence = []; - recordingFinished = true; + // helpful tooltip to include + textSize(20); + stroke(255); + fill(0); + text(state + " : " + currGesture, 50, 50); + + // if hand is detected in the frame, start recording gesture + if (hands.length > 0) { + handpoints = drawPoints(); + sequence.push(handpoints); + + // add collected data to model once the hand is gone and state is collection + } else if (hands.length <= 0 && sequence.length > 0) { + if (state == "collection") { + // pad the length of the coordinates to targetLength + let inputData = model.padCoordinates(sequence, targetLength); + let outputData = { label: currGesture }; + + // add data to the model + model.addData(inputData, outputData); + + // Update the counts for the UI + counts[currGesture]++; + updateDataCountUI(); + + // pad the data and use for prediction if state is prediction + } else if (state == "prediction") { + let predictData = model.padCoordinates(sequence, targetLength); + model.classify(predictData, gotResults); } - // can only record again when hand is out of frame - } else { - if (hands.length == 0) { - recordingFinished = false; - } + // reset the sequence + sequence = []; } } +// Train the data when 'Train abd Save Model' button is pressed +function train() { + // The data should be normalized before training + model.normalizeData(); + + currGesture = ""; + + // Train the model + let trainingOptions = { + epochs: 50, + }; + model.train(trainingOptions, finishedTraining); +} + +// When the model is trained, save the model +function finishedTraining() { + state = "prediction"; + model.save(); +} + +// callback for predict +function gotResults(results) { + currGesture = results[0].label; +} + +// Callback function for when handPose outputs data +function gotHands(results) { + hands = results; +} + +// draw visuals for hand points and flatten values into an array function drawPoints() { let handpoints = []; // iterate through both hands @@ -109,6 +136,7 @@ function drawPoints() { circle(keypoint.x, keypoint.y, 5); } } + // assign to a different variable before clearing let output = handpoints; handpoints = []; @@ -116,77 +144,38 @@ function drawPoints() { return output; } -// Callback function for when handPose outputs data -function gotHands(results) { - // save the output to the hands variable - hands = results; -} - -function trainModelAndSave() { - model.normalizeData(); - let options = { - epochs: 100, - }; - model.train(options, whileTraining, finishedTraining); - nameField.value(""); -} - -function whileTraining(epoch) { - console.log(epoch); -} - -function finishedTraining() { - console.log("finished training."); - model.save("model"); -} - -function gotResults(results) { - predictedWord = results[0].label; - console.log(predictedWord); - text(predictedWord, 200, 200); -} - +////////////// UI Elements //////////// function UI() { - nameField = createInput(""); - nameField.attribute("placeholder", "Type the word to train"); - nameField.position(110, 500); - nameField.size(250); - - instructionP = createP( - 'I want to train:

1.) Type any word you want to pair with a gesture, e.g. "HELLO"
2.) Do the gesture associated to the word, make sure to do it until the points disappear.
3.) Move your hand out of the frame and repeat the gesture, do this multiple times
4.) Do the same for other words e.g. "BYE"
5.) Once all data is collected, press Train and Save

Tip: have at least 5 datasets for each word' + dataCountsP = createP( + "Gesture 1 data: " + + counts[gestures[0]] + + "
Gesture 2 data: " + + counts[gestures[0]] ); - instructionP.style("width", "640px"); - dataCountsP = createP("-> After the gesture a tally will appear here <-"); - - train_but = createButton("Train and Save"); - train_but.mouseClicked(trainModelAndSave); - train_but.style("font-family", "Georgia"); - train_but.style("font-size", "20px"); - train_but.position(500, 490); + rockButton = createButton("Add Gesture #1 Data"); + rockButton.mousePressed(addGesture1); + paperButton = createButton("Add Gesture #2 Data"); + paperButton.mousePressed(addGesture2); + trainButton = createButton("Train and Save Model"); + trainButton.mousePressed(train); } -function drawPredictedWord() { - textSize(100); - fill(255); - text(predictedWord, 100, height / 2); +// Set the current handPose data to the model as "Gesture #1" +function addGesture1() { + currGesture = gestures[0]; } -function trainingWordsUpdate() { - let tempWord = nameField.value(); - console.log(Object.keys(trainingWords)); - if (!(tempWord in trainingWords)) { - trainingWords[tempWord] = 1; - } else { - trainingWords[tempWord]++; - } - - let counts = ""; - let keys = Object.keys(trainingWords); - console.log("keys", keys); - - for (let k of keys) { - counts += k + " : " + trainingWords[k] + "
"; - } +// Set the current handPose data to the model as "Gesture #2" +function addGesture2() { + currGesture = gestures[1]; +} - dataCountsP.html(counts); +// Update the HTML UI with the current data counts +function updateDataCountUI() { + dataCountsP.html( + "Gesture 1 data: " + + counts[gestures[0]] + + "
Gesture 2 data: " + + counts[gestures[1]] + ); } diff --git a/examples/timeSeries-load-model-hand-gestures/model/model.json b/examples/timeSeries-load-model-hand-gestures/model/model.json index 3ab9aa30..98820b95 100644 --- a/examples/timeSeries-load-model-hand-gestures/model/model.json +++ b/examples/timeSeries-load-model-hand-gestures/model/model.json @@ -31,7 +31,7 @@ "bias_constraint": null, "name": "conv1d_Conv1D1", "trainable": true, - "batch_input_shape": [null, 51, 42], + "batch_input_shape": [null, 50, 42], "dtype": "float32" } }, @@ -72,7 +72,7 @@ "bias_constraint": null, "name": "conv1d_Conv1D2", "trainable": true, - "batch_input_shape": [null, 51, 42], + "batch_input_shape": [null, 50, 42], "dtype": "float32" } }, diff --git a/examples/timeSeries-load-model-hand-gestures/model/model_meta.json b/examples/timeSeries-load-model-hand-gestures/model/model_meta.json index 1c0165c7..6f201552 100644 --- a/examples/timeSeries-load-model-hand-gestures/model/model_meta.json +++ b/examples/timeSeries-load-model-hand-gestures/model/model_meta.json @@ -1 +1,227 @@ -{"inputUnits":[42],"outputUnits":2,"inputs":{"label_0":{"dtype":"number","min":4.151249399907168,"max":586.4725394909854},"label_1":{"dtype":"number","min":186.47223882383636,"max":496.34918695509003},"label_2":{"dtype":"number","min":12.818880217505907,"max":564.7860747522525},"label_3":{"dtype":"number","min":160.9460986889124,"max":478.89482602620234},"label_4":{"dtype":"number","min":20.681431005110262,"max":557.1173870582799},"label_5":{"dtype":"number","min":135.1274696802808,"max":454.0862355189599},"label_6":{"dtype":"number","min":29.375938053231934,"max":562.4826339023859},"label_7":{"dtype":"number","min":113.22511415628927,"max":455.15365538508894},"label_8":{"dtype":"number","min":37.27265551578051,"max":573.3838980891996},"label_9":{"dtype":"number","min":98.00531862273047,"max":473.4382341601794},"label_10":{"dtype":"number","min":2.706973037101564,"max":599.2858408346702},"label_11":{"dtype":"number","min":117.7350326456234,"max":453.76022921684716},"label_12":{"dtype":"number","min":11.635752695869659,"max":612.8243751678727},"label_13":{"dtype":"number","min":91.05094143918305,"max":481.6467136241304},"label_14":{"dtype":"number","min":22.9353041163117,"max":621.0127886598051},"label_15":{"dtype":"number","min":61.619264849841635,"max":499.63536096409143},"label_16":{"dtype":"number","min":33.53953084457643,"max":626.4181148091915},"label_17":{"dtype":"number","min":28.455718477478662,"max":512.7953875856006},"label_18":{"dtype":"number","min":-2.8065139589559984,"max":617.7828981986556},"label_19":{"dtype":"number","min":117.6886729722432,"max":459.5357193516273},"label_20":{"dtype":"number","min":3.7782929928570064,"max":633.7038985044576},"label_21":{"dtype":"number","min":86.77279076496669,"max":486.0751342925063},"label_22":{"dtype":"number","min":16.177018651157255,"max":642.8366376068107},"label_23":{"dtype":"number","min":51.687144639081325,"max":502.64037741142846},"label_24":{"dtype":"number","min":28.1461509145229,"max":650.2419536370577},"label_25":{"dtype":"number","min":15.922382743702723,"max":516.9301399988833},"label_26":{"dtype":"number","min":-6.382516546058305,"max":630.7077663350849},"label_27":{"dtype":"number","min":120.16376158664924,"max":461.0881814514869},"label_28":{"dtype":"number","min":-1.4074379536407533,"max":647.5041251714117},"label_29":{"dtype":"number","min":90.58035685591811,"max":485.04491883378125},"label_30":{"dtype":"number","min":10.174906800459325,"max":658.4893875478738},"label_31":{"dtype":"number","min":71.76407331703523,"max":500.55112323964187},"label_32":{"dtype":"number","min":21.11718120932074,"max":668.566957655395},"label_33":{"dtype":"number","min":39.557348432978586,"max":514.4287318106208},"label_34":{"dtype":"number","min":-7.9534800405596595,"max":641.3232619371444},"label_35":{"dtype":"number","min":126.31599791044414,"max":465.6320514399833},"label_36":{"dtype":"number","min":-3.8369034650104927,"max":658.2044139172733},"label_37":{"dtype":"number","min":103.73604938021917,"max":481.03793223993495},"label_38":{"dtype":"number","min":3.7075645592075435,"max":668.8017566330357},"label_39":{"dtype":"number","min":88.76136006394765,"max":494.63688258092407},"label_40":{"dtype":"number","min":6.9609311353376135,"max":676.9525074586147},"label_41":{"dtype":"number","min":75.97401514052241,"max":506.7948506427954}},"outputs":{"label":{"dtype":"string","min":0,"max":1,"uniqueValues":["hello","bye"],"legend":{"hello":[1,0],"bye":[0,1]}}},"isNormalized":true,"seriesShape":[51,42]} \ No newline at end of file +{ + "inputUnits": [42], + "outputUnits": 2, + "inputs": { + "label_0": { + "dtype": "number", + "min": 4.151249399907168, + "max": 586.4725394909854 + }, + "label_1": { + "dtype": "number", + "min": 186.47223882383636, + "max": 496.34918695509003 + }, + "label_2": { + "dtype": "number", + "min": 12.818880217505907, + "max": 564.7860747522525 + }, + "label_3": { + "dtype": "number", + "min": 160.9460986889124, + "max": 478.89482602620234 + }, + "label_4": { + "dtype": "number", + "min": 20.681431005110262, + "max": 557.1173870582799 + }, + "label_5": { + "dtype": "number", + "min": 135.1274696802808, + "max": 454.0862355189599 + }, + "label_6": { + "dtype": "number", + "min": 29.375938053231934, + "max": 562.4826339023859 + }, + "label_7": { + "dtype": "number", + "min": 113.22511415628927, + "max": 455.15365538508894 + }, + "label_8": { + "dtype": "number", + "min": 37.27265551578051, + "max": 573.3838980891996 + }, + "label_9": { + "dtype": "number", + "min": 98.00531862273047, + "max": 473.4382341601794 + }, + "label_10": { + "dtype": "number", + "min": 2.706973037101564, + "max": 599.2858408346702 + }, + "label_11": { + "dtype": "number", + "min": 117.7350326456234, + "max": 453.76022921684716 + }, + "label_12": { + "dtype": "number", + "min": 11.635752695869659, + "max": 612.8243751678727 + }, + "label_13": { + "dtype": "number", + "min": 91.05094143918305, + "max": 481.6467136241304 + }, + "label_14": { + "dtype": "number", + "min": 22.9353041163117, + "max": 621.0127886598051 + }, + "label_15": { + "dtype": "number", + "min": 61.619264849841635, + "max": 499.63536096409143 + }, + "label_16": { + "dtype": "number", + "min": 33.53953084457643, + "max": 626.4181148091915 + }, + "label_17": { + "dtype": "number", + "min": 28.455718477478662, + "max": 512.7953875856006 + }, + "label_18": { + "dtype": "number", + "min": -2.8065139589559984, + "max": 617.7828981986556 + }, + "label_19": { + "dtype": "number", + "min": 117.6886729722432, + "max": 459.5357193516273 + }, + "label_20": { + "dtype": "number", + "min": 3.7782929928570064, + "max": 633.7038985044576 + }, + "label_21": { + "dtype": "number", + "min": 86.77279076496669, + "max": 486.0751342925063 + }, + "label_22": { + "dtype": "number", + "min": 16.177018651157255, + "max": 642.8366376068107 + }, + "label_23": { + "dtype": "number", + "min": 51.687144639081325, + "max": 502.64037741142846 + }, + "label_24": { + "dtype": "number", + "min": 28.1461509145229, + "max": 650.2419536370577 + }, + "label_25": { + "dtype": "number", + "min": 15.922382743702723, + "max": 516.9301399988833 + }, + "label_26": { + "dtype": "number", + "min": -6.382516546058305, + "max": 630.7077663350849 + }, + "label_27": { + "dtype": "number", + "min": 120.16376158664924, + "max": 461.0881814514869 + }, + "label_28": { + "dtype": "number", + "min": -1.4074379536407533, + "max": 647.5041251714117 + }, + "label_29": { + "dtype": "number", + "min": 90.58035685591811, + "max": 485.04491883378125 + }, + "label_30": { + "dtype": "number", + "min": 10.174906800459325, + "max": 658.4893875478738 + }, + "label_31": { + "dtype": "number", + "min": 71.76407331703523, + "max": 500.55112323964187 + }, + "label_32": { + "dtype": "number", + "min": 21.11718120932074, + "max": 668.566957655395 + }, + "label_33": { + "dtype": "number", + "min": 39.557348432978586, + "max": 514.4287318106208 + }, + "label_34": { + "dtype": "number", + "min": -7.9534800405596595, + "max": 641.3232619371444 + }, + "label_35": { + "dtype": "number", + "min": 126.31599791044414, + "max": 465.6320514399833 + }, + "label_36": { + "dtype": "number", + "min": -3.8369034650104927, + "max": 658.2044139172733 + }, + "label_37": { + "dtype": "number", + "min": 103.73604938021917, + "max": 481.03793223993495 + }, + "label_38": { + "dtype": "number", + "min": 3.7075645592075435, + "max": 668.8017566330357 + }, + "label_39": { + "dtype": "number", + "min": 88.76136006394765, + "max": 494.63688258092407 + }, + "label_40": { + "dtype": "number", + "min": 6.9609311353376135, + "max": 676.9525074586147 + }, + "label_41": { + "dtype": "number", + "min": 75.97401514052241, + "max": 506.7948506427954 + } + }, + "outputs": { + "label": { + "dtype": "string", + "min": 0, + "max": 1, + "uniqueValues": ["Hello", "Bye"], + "legend": { "Hello": [1, 0], "Bye": [0, 1] } + } + }, + "isNormalized": true, + "seriesShape": [51, 42] +} diff --git a/examples/timeSeries-load-model-hand-gestures/sketch.js b/examples/timeSeries-load-model-hand-gestures/sketch.js index 772e6fe4..e413748a 100644 --- a/examples/timeSeries-load-model-hand-gestures/sketch.js +++ b/examples/timeSeries-load-model-hand-gestures/sketch.js @@ -11,14 +11,13 @@ * Goodbye: https://babysignlanguage.com/dictionary/goodbye/ */ -// change this to make the recording longer -let seqLength = 50; - let handPose; let video; let hands = []; + let sequence = []; -let recordingFinished = false; +let targetLength = 50; + let predictedWord = ""; function preload() { @@ -55,6 +54,7 @@ function setup() { // load the model and call modelLoaded once finished model.load(modelDetails, modelLoaded); } + // call back for load model function modelLoaded() { console.log("model loaded!"); @@ -68,27 +68,18 @@ function draw() { placePredictedText(); // if hands are found then start recording - if (hands.length > 0 && recordingFinished == false) { - if (sequence.length <= seqLength) { - // get coordinates from hands (21 points) - handpoints = drawPoints(); - sequence.push(handpoints); - - // once sequence reaches the seqLength, add sequence as just one X value - } else if (sequence.length > 0) { - // classify based on the collected data - model.classify(sequence, gotResults); - - // reset the sequence - sequence = []; - recordingFinished = true; - } - - // can only record again when hand is out of frame - } else { - if (hands.length == 0) { - recordingFinished = false; - } + if (hands.length > 0) { + // get coordinates from hands (21 points) + handpoints = drawPoints(); + sequence.push(handpoints); + + // pad the data and use for prediction + } else if (hands.length <= 0 && sequence.length > 0) { + let predictData = model.padCoordinates(sequence, targetLength); + model.classify(predictData, gotResults); + + // reset the sequence + sequence = []; } } @@ -107,6 +98,7 @@ function drawPoints() { } let output = handpoints; handpoints = []; + return output; } @@ -119,13 +111,12 @@ function gotHands(results) { // call back for accessing the results function gotResults(results) { predictedWord = results[0].label; - console.log(predictedWord); text(predictedWord, 100, 100); } // for drawing text on screen function placePredictedText() { - textSize(100); - fill(255); - text(predictedWord, 100, height / 2); + textSize(20); + fill(0); + text("Predicted Gesture : " + predictedWord, 50, 50); } diff --git a/examples/timeSeries-train-mouse-gesture RDP/sketch.js b/examples/timeSeries-train-mouse-gesture RDP/sketch.js index d030319e..24e86def 100644 --- a/examples/timeSeries-train-mouse-gesture RDP/sketch.js +++ b/examples/timeSeries-train-mouse-gesture RDP/sketch.js @@ -71,7 +71,7 @@ function trainModel() { let options = { epochs: 40, }; - model.train(options, whileTraining, finishedTraining); + model.train(options, finishedTraining); background(220); state = "training"; @@ -82,10 +82,6 @@ function trainModel() { trainBut.attribute("disabled", true); } -function whileTraining(epoch, loss) { - console.log(epoch); -} - function finishedTraining() { background(220); text("Training Finished, Draw again to predict", 50, 50); @@ -93,7 +89,6 @@ function finishedTraining() { } function gotResults(results) { - // console.log("results", results); let label = results[0].label; currShape = label; } diff --git a/examples/timeSeries-weather-prediction/sketch.js b/examples/timeSeries-weather-prediction/sketch.js index c524d61e..24e99c7b 100644 --- a/examples/timeSeries-weather-prediction/sketch.js +++ b/examples/timeSeries-weather-prediction/sketch.js @@ -20,7 +20,8 @@ let features = [ "precipitation", ]; -let targets = features; // must be the same to add predicted to data +let targets = features; // must be the same to add predicted values back to data +let windowLength = 10; // Optional: define the size of the window for batch // load JSON data with same formatting from the internet, this means // loadData() cannot yet be used as it is formatted differently @@ -31,6 +32,7 @@ function preload() { let options = { task: "regression", debug: "true", + learningRate: 0.0075, // A smaller learning rate used for more stable training inputs: features, outputs: targets, }; @@ -43,7 +45,7 @@ function setup() { background(220); //run a sliding window algorithm for time based data - let batchData = model.slidingWindow(data, features, targets); + let batchData = model.slidingWindow(data, features, targets, windowLength); let inputs = batchData.sequences; let outputs = batchData.targets; @@ -55,7 +57,7 @@ function setup() { model.normalizeData(); let options = { - epochs: 100, + epochs: 70, }; model.train(options, finishedTraining); @@ -65,16 +67,23 @@ function setup() { function draw() { background(220); textSize(20); - if (state == "training") text("Training", 200, 200); - else if (state == "prediction") { - text("Predicted Precipitation", 200, 200); - text(precipitation, 200, 250); + if (state == "training") { + text("Training", 320, 200); + } else if (state == "prediction") { + text("Predicted Precipitation", 320, 200); + text(precipitation, 320, 250); + + // helpful visual based on predicted value + push(); + textSize(precipitation * 5 + 10); + text("🌧️", 320, 150); + pop(); } } // predict data function predictData() { - seq = model.sampleWindow(data); //helper function to get sample from data + seq = model.sampleWindow(data); //helper function paired with the slidingWindow to get sample from data model.predict(seq, gotResults); } @@ -87,7 +96,7 @@ function gotResults(results) { // code for adding new data to the dataset to be used for future prediction function addNewData(results) { (new_values = { - date: " for the next hour", + date: " for the next hour", temperature: results[0].value, // get string convert to float and round to 2 decimal points humidity: results[1].value, wind_speed: results[2].value, @@ -98,7 +107,6 @@ function addNewData(results) { } function finishedTraining() { - console.log("Training Done!"); state = "prediction"; } @@ -106,4 +114,6 @@ function finishedTraining() { function UI() { pred_but = select("#pred_but"); pred_but.mouseClicked(predictData); + + textAlign(CENTER); } diff --git a/src/TimeSeries/index.js b/src/TimeSeries/index.js index 86d99a77..7bdca827 100644 --- a/src/TimeSeries/index.js +++ b/src/TimeSeries/index.js @@ -176,8 +176,8 @@ class DIYTimesSeries extends DiyNeuralNetwork { this.featureKeys = featureKeys; if (batchLength == null) { - this.batchLength = int(data.length * 0.2); // set targetlength as a fraction of the total - } else if (targetLength >= data.length) { + this.batchLength = int(data.length * 0.2); // set batchlength as a fraction of the total + } else if (batchLength >= data.length) { throw new Error("batchLength must be smaller than total length of data"); } else { this.batchLength = batchLength; From 60d0eb48bd444e493a787a655947aa2f342388b2 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Wed, 4 Jun 2025 15:53:00 -0400 Subject: [PATCH 09/47] Merged timeseries-as-task to timeseries-base and bug fixes --- src/NeuralNetwork/taskSelection.js | 72 ++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/NeuralNetwork/taskSelection.js diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js new file mode 100644 index 00000000..a9135c28 --- /dev/null +++ b/src/NeuralNetwork/taskSelection.js @@ -0,0 +1,72 @@ +import DiyNeuralNetwork from "./index.js"; +import DIYTimesSeries from "../TimeSeries/index.js"; + +// helper function to check if tasks follows specified convention +const isTimeSeriesTask = (task) => { + const timeSeriesTasks = ["timed_classification", "timed_regression"]; + return timeSeriesTasks.includes(task); +}; + +// factory function for DIY Neural Network +const createNeuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + let cb; + + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + cb = outputsOrCallback; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + cb = callback; + } + + const instance = new DiyNeuralNetwork(options, cb); + return instance; +}; + +// factory function for DIY Timeseries +const createTimeSeries = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + let cb; + + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + cb = outputsOrCallback; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + cb = callback; + } + + const instance = new DIYTimesSeries(options, cb); + return instance; +}; + +// Selection logic for either NeuralNetwork or TimeSeries +const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + + // Parse options first to check the task + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + } + + // Choose which factory function to call based on task + if (isTimeSeriesTask(options.task)) { + return createTimeSeries(inputsOrOptions, outputsOrCallback, callback); + } else { + return createNeuralNetwork(inputsOrOptions, outputsOrCallback, callback); + } +}; + +export default neuralNetwork; From 1356185986332fbfbd7acafc996a446c929c8e4a Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Wed, 4 Jun 2025 15:53:38 -0400 Subject: [PATCH 10/47] merged timeseries as task to timeseries-base and bug fixes --- examples/timeSeries-hand-gestures/sketch.js | 5 +- .../sketch.js | 5 +- .../sketch.js | 8 +-- .../timeSeries-weather-prediction/sketch.js | 4 +- src/NeuralNetwork/index.js | 23 +------- src/NeuralNetwork/taskSelection.js | 7 ++- src/TimeSeries/index.js | 59 ++++++++----------- src/TimeSeries/tsLayers.js | 4 +- src/index.js | 2 +- 9 files changed, 44 insertions(+), 73 deletions(-) diff --git a/examples/timeSeries-hand-gestures/sketch.js b/examples/timeSeries-hand-gestures/sketch.js index cf6952a4..40054d2a 100644 --- a/examples/timeSeries-hand-gestures/sketch.js +++ b/examples/timeSeries-hand-gestures/sketch.js @@ -26,12 +26,11 @@ function preload() { // setup the timeseries neural network let options = { outputs: ["label"], - task: "classification", - spatialData: "true", + task: "sequenceClassificationConv", debug: "true", learningRate: 0.001, // the default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update) }; - model = ml5.timeSeries(options); + model = ml5.neuralNetwork(options); } function setup() { diff --git a/examples/timeSeries-load-model-hand-gestures/sketch.js b/examples/timeSeries-load-model-hand-gestures/sketch.js index e413748a..7d140f08 100644 --- a/examples/timeSeries-load-model-hand-gestures/sketch.js +++ b/examples/timeSeries-load-model-hand-gestures/sketch.js @@ -26,11 +26,10 @@ function preload() { // setup the timeseries neural network let options = { - task: "classification", - spatialData: "true", + task: "sequenceClassificationConv", }; - model = ml5.timeSeries(options); + model = ml5.neuralNetwork(options); } function setup() { diff --git a/examples/timeSeries-train-mouse-gesture RDP/sketch.js b/examples/timeSeries-train-mouse-gesture RDP/sketch.js index 24e86def..cd9a1152 100644 --- a/examples/timeSeries-train-mouse-gesture RDP/sketch.js +++ b/examples/timeSeries-train-mouse-gesture RDP/sketch.js @@ -17,13 +17,13 @@ function preload() { let options = { inputs: ["x", "y"], outputs: ["label"], - task: "classification", - spatialData: "true", - debug: "true", + task: "sequenceClassificationConv", + spatialData: true, + debug: true, learningRate: 0.005, }; - model = ml5.timeSeries(options); + model = ml5.neuralNetwork(options); } function setup() { diff --git a/examples/timeSeries-weather-prediction/sketch.js b/examples/timeSeries-weather-prediction/sketch.js index 24e99c7b..02b385fe 100644 --- a/examples/timeSeries-weather-prediction/sketch.js +++ b/examples/timeSeries-weather-prediction/sketch.js @@ -30,13 +30,13 @@ function preload() { // set the options to initialize timeSeries Neural Network let options = { - task: "regression", + task: "sequenceRegression", debug: "true", learningRate: 0.0075, // A smaller learning rate used for more stable training inputs: features, outputs: targets, }; - model = ml5.timeSeries(options); + model = ml5.neuralNetwork(options); } function setup() { diff --git a/src/NeuralNetwork/index.js b/src/NeuralNetwork/index.js index e3972c15..40ed5ec3 100644 --- a/src/NeuralNetwork/index.js +++ b/src/NeuralNetwork/index.js @@ -1249,24 +1249,5 @@ class DiyNeuralNetwork { } } -const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { - let options; - let cb; - - if (inputsOrOptions instanceof Object) { - options = inputsOrOptions; - cb = outputsOrCallback; - } else { - options = { - inputs: inputsOrOptions, - outputs: outputsOrCallback, - }; - cb = callback; - } - - const instance = new DiyNeuralNetwork(options, cb); - return instance; -}; - -export { DiyNeuralNetwork }; // Named export for extending -export default neuralNetwork; // Default export remains +export { DiyNeuralNetwork }; // Named export for extending in DIY TimeSeries +export default DiyNeuralNetwork; //export for taskSelection.js diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js index a9135c28..9a0a0bfb 100644 --- a/src/NeuralNetwork/taskSelection.js +++ b/src/NeuralNetwork/taskSelection.js @@ -3,7 +3,12 @@ import DIYTimesSeries from "../TimeSeries/index.js"; // helper function to check if tasks follows specified convention const isTimeSeriesTask = (task) => { - const timeSeriesTasks = ["timed_classification", "timed_regression"]; + const timeSeriesTasks = [ + "sequenceClassification", + "sequenceRegression", + "sequenceClassificationConv", + "sequenceRegressionConv", + ]; return timeSeriesTasks.includes(task); }; diff --git a/src/TimeSeries/index.js b/src/TimeSeries/index.js index 7bdca827..cf5394b3 100644 --- a/src/TimeSeries/index.js +++ b/src/TimeSeries/index.js @@ -22,7 +22,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { callback ); // call all options set in the this class which is the default, extra option for dataMode - this.options = { ...this.options, spatialData: false, ...(options || {}) }; + this.options = { ...this.options, ...(options || {}) }; this.neuralNetworkData = this.options.neuralNetworkData || new TimeSeriesData(); @@ -94,30 +94,26 @@ class DIYTimesSeries extends DiyNeuralNetwork { ); const task = this.options.task; - const spatialData = this.options.spatialData; let taskConditions = task; - if (spatialData === true || spatialData === "true") { - taskConditions = `${task}_spatial`; - } - switch (taskConditions.toLowerCase()) { - case "regression": + switch (taskConditions) { + case "sequenceRegression": layers = tsLayers.regression; return this.createNetworkLayers(layers); - case "classification": + case "sequenceClassification": layers = tsLayers.classification; return this.createNetworkLayers(layers); - case "classification_spatial": - layers = tsLayers.classification_spatial; + case "sequenceClassificationConv": + layers = tsLayers.classificationConv; return this.createNetworkLayers(layers); - case "regression_spatial": - layers = tsLayers.regression_spatial; + case "sequenceRegressionConv": + layers = tsLayers.regressionConv; return this.createNetworkLayers(layers); default: - console.warn("no inputUnits or outputUnits defined"); + console.warn("Task is undefined or no inputUnits/outputUnits defined"); layers = tsLayers.default; return this.createNetworkLayers(layers); } @@ -130,15 +126,25 @@ class DIYTimesSeries extends DiyNeuralNetwork { let options = {}; if ( - this.options.task === "classification" || - this.options.task === "imageClassification" + this.options.task === "sequenceClassification" || + this.options.task === "sequenceClassificationConv" ) { options = { loss: "categoricalCrossentropy", optimizer: tf.train.adam, metrics: ["accuracy"], }; - } else if (this.options.task === "regression") { + } else if ( + this.options.task === "sequenceRegression" || + this.options.task === "sequenceRegressionConv" + ) { + options = { + loss: "meanSquaredError", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; + } else { + // if no task given - must be in NN class instead of this options = { loss: "meanSquaredError", optimizer: tf.train.adam, @@ -201,23 +207,4 @@ class DIYTimesSeries extends DiyNeuralNetwork { } } -const timeSeries = (inputsOrOptions, outputsOrCallback, callback) => { - let options; - let cb; - - if (inputsOrOptions instanceof Object) { - options = inputsOrOptions; - cb = outputsOrCallback; - } else { - options = { - inputs: inputsOrOptions, - outputs: outputsOrCallback, - }; - cb = callback; - } - - const instance = new DIYTimesSeries(options, cb); - return instance; -}; - -export default timeSeries; +export default DIYTimesSeries; //export for taskSelection diff --git a/src/TimeSeries/tsLayers.js b/src/TimeSeries/tsLayers.js index a77ae9b1..4a6acfe8 100644 --- a/src/TimeSeries/tsLayers.js +++ b/src/TimeSeries/tsLayers.js @@ -4,7 +4,7 @@ export const createTsLayers = ( outputUnits = null ) => { return { - classification_spatial: [ + classificationConv: [ { type: "conv1d", filters: 8, @@ -65,7 +65,7 @@ export const createTsLayers = ( activation: "softmax", }, ], - regression_spatial: [ + regressionConv: [ { type: "conv1d", filters: 8, diff --git a/src/index.js b/src/index.js index cac3b86a..31027288 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,4 @@ -import neuralNetwork from "./NeuralNetwork"; +import neuralNetwork from "./NeuralNetwork/taskSelection"; import timeSeries from "./TimeSeries"; import handPose from "./HandPose"; import sentiment from "./Sentiment"; From c5f78cf7ad5cd81d8f6be02c7ebe54b75c3c87b6 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 5 Jun 2025 14:30:46 -0400 Subject: [PATCH 11/47] Transferred files into NeuralNetwork Folder and code cleanup --- .../index.html | 19 ++++++++++-------- .../sketch.js | 2 +- .../index.html | 2 +- .../model/model.json | 0 .../model/model.weights.bin | Bin .../model/model_meta.json | 0 .../sketch.js | 2 +- .../index.html | 4 ++-- .../sketch.js | 5 ++--- .../index.html | 4 ++-- .../sketch.js | 2 +- .../weather_data.json | 0 .../Sequential}/index.js | 6 +++--- .../Sequential}/timeSeriesData.js | 8 ++++---- .../Sequential}/timeSeriesUtils.js | 2 +- .../Sequential}/tsLayers.js | 0 src/NeuralNetwork/taskSelection.js | 2 +- src/index.js | 2 -- 18 files changed, 30 insertions(+), 30 deletions(-) rename examples/{timeSeries-hand-gestures => neuralNetwork-sequence-hand-gestures}/index.html (56%) rename examples/{timeSeries-hand-gestures => neuralNetwork-sequence-hand-gestures}/sketch.js (98%) rename examples/{timeSeries-load-model-hand-gestures => neuralNetwork-sequence-load-model-hand-gestures}/index.html (96%) rename examples/{timeSeries-load-model-hand-gestures => neuralNetwork-sequence-load-model-hand-gestures}/model/model.json (100%) rename examples/{timeSeries-load-model-hand-gestures => neuralNetwork-sequence-load-model-hand-gestures}/model/model.weights.bin (100%) rename examples/{timeSeries-load-model-hand-gestures => neuralNetwork-sequence-load-model-hand-gestures}/model/model_meta.json (100%) rename examples/{timeSeries-load-model-hand-gestures => neuralNetwork-sequence-load-model-hand-gestures}/sketch.js (98%) rename examples/{timeSeries-train-mouse-gesture RDP => neuralNetwork-sequence-train-mouse-gesture RDP}/index.html (87%) rename examples/{timeSeries-train-mouse-gesture RDP => neuralNetwork-sequence-train-mouse-gesture RDP}/sketch.js (95%) rename examples/{timeSeries-weather-prediction => neuralNetwork-sequence-weather-prediction}/index.html (89%) rename examples/{timeSeries-weather-prediction => neuralNetwork-sequence-weather-prediction}/sketch.js (97%) rename examples/{timeSeries-weather-prediction => neuralNetwork-sequence-weather-prediction}/weather_data.json (100%) rename src/{TimeSeries => NeuralNetwork/Sequential}/index.js (97%) rename src/{TimeSeries => NeuralNetwork/Sequential}/timeSeriesData.js (97%) rename src/{TimeSeries => NeuralNetwork/Sequential}/timeSeriesUtils.js (99%) rename src/{TimeSeries => NeuralNetwork/Sequential}/tsLayers.js (100%) diff --git a/examples/timeSeries-hand-gestures/index.html b/examples/neuralNetwork-sequence-hand-gestures/index.html similarity index 56% rename from examples/timeSeries-hand-gestures/index.html rename to examples/neuralNetwork-sequence-hand-gestures/index.html index 1e3c0642..1fdd80c3 100644 --- a/examples/timeSeries-hand-gestures/index.html +++ b/examples/neuralNetwork-sequence-hand-gestures/index.html @@ -3,7 +3,7 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates training a Sign Language classifier through ml5.TimeSeries. + This example demonstrates training a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task. --> @@ -11,7 +11,7 @@ - ml5.js Time Series Hand Gesture Train and Save + ml5.js Neural Network Hand Gesture Train and Save @@ -21,12 +21,15 @@

- To add data, press the "Add Gesture #1 Data" button and then show your - hand to the screen, put down your hand when you are done. You can record - again once you show your hand back in the frame. Likewise for gesture #2. - When you have a good amount of data for each gesture class, press the - "Train and Save Model" button to train the model and save it for future - use. + Instructions:
+ 1.) To add data, press the "Add Gesture #1 Data" button
+ 2.) Use one hand to do a gesture in front of the camera
+ 3.) Once done with the gesture, take your hand out of the frame
+ 4.) Repeat until there is a good amount of data
+ 5.) Do the same for gesture #2 by pressing the "Add Gesture #2 Data" + button before doing the hand gesture in the frame
+ 6.) Press the "Train and Save Model" button to train the model and save it + for future use.

diff --git a/examples/timeSeries-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-hand-gestures/sketch.js similarity index 98% rename from examples/timeSeries-hand-gestures/sketch.js rename to examples/neuralNetwork-sequence-hand-gestures/sketch.js index 40054d2a..e2298d9d 100644 --- a/examples/timeSeries-hand-gestures/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gestures/sketch.js @@ -3,7 +3,7 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates training a Hand Gesture classifier through ml5.TimeSeries. + * This example demonstrates training a Hand Gesture classifier through ml5.neuralNetwork with sequeceClassification Task. */ let video; diff --git a/examples/timeSeries-load-model-hand-gestures/index.html b/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html similarity index 96% rename from examples/timeSeries-load-model-hand-gestures/index.html rename to examples/neuralNetwork-sequence-load-model-hand-gestures/index.html index 92363d69..0d8ae703 100644 --- a/examples/timeSeries-load-model-hand-gestures/index.html +++ b/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html @@ -3,7 +3,7 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates loading a Sign Language classifier through ml5.TimeSeries. + This example demonstrates loading a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task. --> diff --git a/examples/timeSeries-load-model-hand-gestures/model/model.json b/examples/neuralNetwork-sequence-load-model-hand-gestures/model/model.json similarity index 100% rename from examples/timeSeries-load-model-hand-gestures/model/model.json rename to examples/neuralNetwork-sequence-load-model-hand-gestures/model/model.json diff --git a/examples/timeSeries-load-model-hand-gestures/model/model.weights.bin b/examples/neuralNetwork-sequence-load-model-hand-gestures/model/model.weights.bin similarity index 100% rename from examples/timeSeries-load-model-hand-gestures/model/model.weights.bin rename to examples/neuralNetwork-sequence-load-model-hand-gestures/model/model.weights.bin diff --git a/examples/timeSeries-load-model-hand-gestures/model/model_meta.json b/examples/neuralNetwork-sequence-load-model-hand-gestures/model/model_meta.json similarity index 100% rename from examples/timeSeries-load-model-hand-gestures/model/model_meta.json rename to examples/neuralNetwork-sequence-load-model-hand-gestures/model/model_meta.json diff --git a/examples/timeSeries-load-model-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js similarity index 98% rename from examples/timeSeries-load-model-hand-gestures/sketch.js rename to examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js index 7d140f08..85777a32 100644 --- a/examples/timeSeries-load-model-hand-gestures/sketch.js +++ b/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js @@ -3,7 +3,7 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates loading a Hand Gesture classifier through ml5.TimeSeries. + * This example demonstrates loading a Hand Gesture classifier through ml5.neuralNetwork with sequeceClassification Task. * This example is trained with the ASL gestures for Hello and Goodbye * * Reference to sign hello and goodbye in ASL: diff --git a/examples/timeSeries-train-mouse-gesture RDP/index.html b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html similarity index 87% rename from examples/timeSeries-train-mouse-gesture RDP/index.html rename to examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html index 6407c4eb..0e99a9ee 100644 --- a/examples/timeSeries-train-mouse-gesture RDP/index.html +++ b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html @@ -3,7 +3,7 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates How to train your own quickdraw classifier through ml5.TimeSeries. + This example demonstrates How to train your own quickdraw classifier through ml5.neuralNetwork with sequeceClassification Task. --> @@ -11,7 +11,7 @@ - ml5.js Time Series Train Mouse Gesture classifier Example + ml5.js Neural Network Train Mouse Gesture classifier Example diff --git a/examples/timeSeries-train-mouse-gesture RDP/sketch.js b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js similarity index 95% rename from examples/timeSeries-train-mouse-gesture RDP/sketch.js rename to examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js index cd9a1152..6e8e8fcd 100644 --- a/examples/timeSeries-train-mouse-gesture RDP/sketch.js +++ b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js @@ -3,7 +3,7 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates How to train your own mouse gesture classifier through ml5.TimeSeries. + * This example demonstrates How to train your own mouse gesture classifier through ml5.neuralNetwork with sequeceClassification Task. */ let model; @@ -18,9 +18,8 @@ function preload() { inputs: ["x", "y"], outputs: ["label"], task: "sequenceClassificationConv", - spatialData: true, debug: true, - learningRate: 0.005, + learningRate: 0.005, // learning rate decreased for better convergence }; model = ml5.neuralNetwork(options); diff --git a/examples/timeSeries-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html similarity index 89% rename from examples/timeSeries-weather-prediction/index.html rename to examples/neuralNetwork-sequence-weather-prediction/index.html index b7dcb388..353148c0 100644 --- a/examples/timeSeries-weather-prediction/index.html +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -3,7 +3,7 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates Training a Weather Predictor through ml5.TimeSeries. + This example demonstrates Training a Weather Predictor through ml5.neuralNetwork with sequeceRegression Task. --> @@ -11,7 +11,7 @@ - ml5.js Time Series Weather Prediction Example + ml5.js Neural Network Weather Prediction Example diff --git a/examples/timeSeries-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js similarity index 97% rename from examples/timeSeries-weather-prediction/sketch.js rename to examples/neuralNetwork-sequence-weather-prediction/sketch.js index 02b385fe..da45a65b 100644 --- a/examples/timeSeries-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -3,7 +3,7 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates Loading JSON data and Training a Weather Predictor through ml5.TimeSeries. + * This example demonstrates Loading JSON data and Training a Weather Predictor through ml5.neuralNetwork with sequeceRegression Task. */ let model; diff --git a/examples/timeSeries-weather-prediction/weather_data.json b/examples/neuralNetwork-sequence-weather-prediction/weather_data.json similarity index 100% rename from examples/timeSeries-weather-prediction/weather_data.json rename to examples/neuralNetwork-sequence-weather-prediction/weather_data.json diff --git a/src/TimeSeries/index.js b/src/NeuralNetwork/Sequential/index.js similarity index 97% rename from src/TimeSeries/index.js rename to src/NeuralNetwork/Sequential/index.js index cf5394b3..4ae04427 100644 --- a/src/TimeSeries/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -1,9 +1,9 @@ import * as tf from "@tensorflow/tfjs"; -import { DiyNeuralNetwork } from "../NeuralNetwork"; +import { DiyNeuralNetwork } from ".."; -import callCallback from "../utils/callcallback"; -import setBackend from "../utils/setBackend"; +import callCallback from "../../utils/callcallback"; +import setBackend from "../../utils/setBackend"; import tsUtils from "./timeSeriesUtils"; diff --git a/src/TimeSeries/timeSeriesData.js b/src/NeuralNetwork/Sequential/timeSeriesData.js similarity index 97% rename from src/TimeSeries/timeSeriesData.js rename to src/NeuralNetwork/Sequential/timeSeriesData.js index 6c1fa14b..b64352bb 100644 --- a/src/TimeSeries/timeSeriesData.js +++ b/src/NeuralNetwork/Sequential/timeSeriesData.js @@ -1,12 +1,12 @@ import * as tf from "@tensorflow/tfjs"; import axios from "axios"; -import { saveBlob } from "../utils/io"; -import modelLoader from "../utils/modelLoader"; -import nnUtils from "../NeuralNetwork/NeuralNetworkUtils"; +import { saveBlob } from "../../utils/io"; +import modelLoader from "../../utils/modelLoader"; +import nnUtils from "../NeuralNetworkUtils"; import tsUtils from "./timeSeriesUtils"; -import NeuralNetworkData from "../NeuralNetwork/NeuralNetworkData"; +import NeuralNetworkData from "../NeuralNetworkData"; class TimeSeriesData extends NeuralNetworkData { constructor() { diff --git a/src/TimeSeries/timeSeriesUtils.js b/src/NeuralNetwork/Sequential/timeSeriesUtils.js similarity index 99% rename from src/TimeSeries/timeSeriesUtils.js rename to src/NeuralNetwork/Sequential/timeSeriesUtils.js index 8eaa2001..a9fcb912 100644 --- a/src/TimeSeries/timeSeriesUtils.js +++ b/src/NeuralNetwork/Sequential/timeSeriesUtils.js @@ -1,5 +1,5 @@ import { data, input } from "@tensorflow/tfjs"; -import nnUtils from "../NeuralNetwork/NeuralNetworkUtils"; +import nnUtils from "../NeuralNetworkUtils"; class TimeSeriesUtils { constructor(options) { diff --git a/src/TimeSeries/tsLayers.js b/src/NeuralNetwork/Sequential/tsLayers.js similarity index 100% rename from src/TimeSeries/tsLayers.js rename to src/NeuralNetwork/Sequential/tsLayers.js diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js index 9a0a0bfb..b0277dea 100644 --- a/src/NeuralNetwork/taskSelection.js +++ b/src/NeuralNetwork/taskSelection.js @@ -1,5 +1,5 @@ import DiyNeuralNetwork from "./index.js"; -import DIYTimesSeries from "../TimeSeries/index.js"; +import DIYTimesSeries from "./Sequential/index.js"; // helper function to check if tasks follows specified convention const isTimeSeriesTask = (task) => { diff --git a/src/index.js b/src/index.js index 31027288..7db37053 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,4 @@ import neuralNetwork from "./NeuralNetwork/taskSelection"; -import timeSeries from "./TimeSeries"; import handPose from "./HandPose"; import sentiment from "./Sentiment"; import faceMesh from "./FaceMesh"; @@ -23,7 +22,6 @@ const withPreload = { neuralNetwork, sentiment, soundClassifier, - timeSeries, }; const ml5 = Object.assign({ p5Utils }, withPreload, { From b9a1959770e62c0bccdbed1782970283cd3025b9 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Mon, 30 Jun 2025 11:58:30 -0400 Subject: [PATCH 12/47] Fixed comment formatting --- .../sketch.js | 40 +++++++++---------- .../sketch.js | 30 +++++++------- .../sketch.js | 21 +++++----- .../sketch.js | 30 +++++++------- 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-hand-gestures/sketch.js index e2298d9d..d0e65f6c 100644 --- a/examples/neuralNetwork-sequence-hand-gestures/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gestures/sketch.js @@ -17,18 +17,18 @@ let gestures = ["Gesture #1", "Gesture #2"]; let counts = { "Gesture #1": 0, "Gesture #2": 0 }; let state = "collection"; -let currGesture = gestures[0]; //set currGesture to gesture 1 by default +let currGesture = gestures[0]; // Set currGesture to gesture 1 by default function preload() { // Load the handPose model handPose = ml5.handPose(); - // setup the timeseries neural network + // Setup the timeseries neural network let options = { outputs: ["label"], task: "sequenceClassificationConv", debug: "true", - learningRate: 0.001, // the default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update) + learningRate: 0.001, // The default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update) }; model = ml5.neuralNetwork(options); } @@ -37,7 +37,7 @@ function setup() { let canvas = createCanvas(640, 480); canvas.parent("canvasDiv"); - // setup video capture + // Setup video capture video = createCapture(VIDEO); video.size(640, 480); video.hide(); @@ -45,46 +45,46 @@ function setup() { // Setup the UI buttons for training UI(); - // use handpose model on video + // Use handpose model on video handPose.detectStart(video, gotHands); } function draw() { - // draw video on frame + // Draw video on frame image(video, 0, 0, width, height); - // helpful tooltip to include + // Helpful tooltip to include textSize(20); stroke(255); fill(0); text(state + " : " + currGesture, 50, 50); - // if hand is detected in the frame, start recording gesture + // If hand is detected in the frame, start recording gesture if (hands.length > 0) { handpoints = drawPoints(); sequence.push(handpoints); - // add collected data to model once the hand is gone and state is collection + // Add collected data to model once the hand is gone and state is collection } else if (hands.length <= 0 && sequence.length > 0) { if (state == "collection") { - // pad the length of the coordinates to targetLength + // Pad the length of the coordinates to targetLength let inputData = model.padCoordinates(sequence, targetLength); let outputData = { label: currGesture }; - // add data to the model + // Add data to the model model.addData(inputData, outputData); - // Update the counts for the UI + // Udate the counts for the UI counts[currGesture]++; updateDataCountUI(); - // pad the data and use for prediction if state is prediction + // Pad the data and use for prediction if state is prediction } else if (state == "prediction") { let predictData = model.padCoordinates(sequence, targetLength); model.classify(predictData, gotResults); } - // reset the sequence + // Reset the sequence sequence = []; } } @@ -109,7 +109,7 @@ function finishedTraining() { model.save(); } -// callback for predict +// Callback for predict function gotResults(results) { currGesture = results[0].label; } @@ -119,14 +119,14 @@ function gotHands(results) { hands = results; } -// draw visuals for hand points and flatten values into an array +// Draw visuals for hand points and flatten values into an array function drawPoints() { let handpoints = []; - // iterate through both hands + // Iterate through both hands for (let i = 0; i < hands.length; i++) { let hand = hands[i]; for (let j = 0; j < hand.keypoints.length; j++) { - // access the keypoints in the hand + // Access the keypoints in the hand let keypoint = hand.keypoints[j]; handpoints.push(keypoint.x, keypoint.y); @@ -136,14 +136,14 @@ function drawPoints() { } } - // assign to a different variable before clearing + // Assign to a different variable before clearing let output = handpoints; handpoints = []; return output; } -////////////// UI Elements //////////// +// UI Elements function UI() { dataCountsP = createP( "Gesture 1 data: " + diff --git a/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js index 85777a32..bfd7b6eb 100644 --- a/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js +++ b/examples/neuralNetwork-sequence-load-model-hand-gestures/sketch.js @@ -24,7 +24,7 @@ function preload() { // Load the handPose model handPose = ml5.handPose(); - // setup the timeseries neural network + // Setup the timeseries neural network let options = { task: "sequenceClassificationConv", }; @@ -36,53 +36,53 @@ function setup() { let canvas = createCanvas(640, 480); canvas.parent("canvasDiv"); - // create video capture + // Create video capture video = createCapture(VIDEO); video.size(640, 480); video.hide(); handPose.detectStart(video, gotHands); - // setup the model files to load + // Setup the model files to load let modelDetails = { model: "model/model.json", metadata: "model/model_meta.json", weights: "model/model.weights.bin", }; - // load the model and call modelLoaded once finished + // Load the model and call modelLoaded once finished model.load(modelDetails, modelLoaded); } -// call back for load model +// Callback for load model function modelLoaded() { console.log("model loaded!"); } function draw() { - // draw video on the canvas + // Draw video on the canvas image(video, 0, 0, width, height); - // put the text on screen after a prediction + // Put the text on screen after a prediction placePredictedText(); - // if hands are found then start recording + // If hands are found then start recording if (hands.length > 0) { - // get coordinates from hands (21 points) + // Get coordinates from hands (21 points) handpoints = drawPoints(); sequence.push(handpoints); - // pad the data and use for prediction + // Pad the data and use for prediction } else if (hands.length <= 0 && sequence.length > 0) { let predictData = model.padCoordinates(sequence, targetLength); model.classify(predictData, gotResults); - // reset the sequence + // Reset the sequence sequence = []; } } -// draw the points on the hands +// Draw the points on the hands function drawPoints() { let handpoints = []; for (let i = 0; i < hands.length; i++) { @@ -103,17 +103,17 @@ function drawPoints() { // Callback function for when handPose outputs data function gotHands(results) { - // save the output to the hands variable + // Save the output to the hands variable hands = results; } -// call back for accessing the results +// Call back for accessing the results function gotResults(results) { predictedWord = results[0].label; text(predictedWord, 100, 100); } -// for drawing text on screen +// For drawing text on screen function placePredictedText() { textSize(20); fill(0); diff --git a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js index 6e8e8fcd..30420b75 100644 --- a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js +++ b/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js @@ -19,14 +19,13 @@ function preload() { outputs: ["label"], task: "sequenceClassificationConv", debug: true, - learningRate: 0.005, // learning rate decreased for better convergence + learningRate: 0.005, // Learning rate decreased for better convergence }; model = ml5.neuralNetwork(options); } function setup() { - // p5 js elements let canvas = createCanvas(600, 400); canvas.parent("canvasDiv"); background(220); @@ -34,19 +33,19 @@ function setup() { } function draw() { - // record data when the mouse is pressed inside the canvas + // Record data when the mouse is pressed inside the canvas if (mouseIsPressed) { - // draw lines through coordinates + // Draw lines through coordinates line(pmouseX, pmouseY, mouseX, mouseY); let inputs = { x: mouseX, y: mouseY }; sequence.push(inputs); } } -// code to signify drawing can be done again +// Code to signify drawing can be done again function mouseReleased() { if (mouseY < height && mouseX < width) { - // if state is collection, add whole sequence as X, and shape as Y + // If state is collection, add whole sequence as X, and shape as Y if (state == "collection") { let target = { label: currShape }; let paddedCoordinates = model.padCoordinates(sequence, targetLength); @@ -58,15 +57,15 @@ function mouseReleased() { clearScreen(); } } - // reset the sequence + // Reset the sequence sequence = []; } function trainModel() { - // normalize Data first before Training + // Normalize Data first before Training model.normalizeData(); - // set the number of epochs for training + // Set the number of epochs for training let options = { epochs: 40, }; @@ -92,7 +91,7 @@ function gotResults(results) { currShape = label; } -////////////// UI Elements //////////// +// UI Elements let recCircle, recSquare, trainBut; function UI() { @@ -123,7 +122,7 @@ function UI() { } } -// cleanup screen and removed drawn elements, add helpful text +// Cleanup screen and removed drawn elements, add helpful text function clearScreen() { background(220); textSize(20); diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index da45a65b..de3d5a10 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -20,15 +20,15 @@ let features = [ "precipitation", ]; -let targets = features; // must be the same to add predicted values back to data +let targets = features; // Must be the same to add predicted values back to data let windowLength = 10; // Optional: define the size of the window for batch -// load JSON data with same formatting from the internet, this means -// loadData() cannot yet be used as it is formatted differently +// Load JSON data with same formatting from the internet, this means +// LoadData() cannot yet be used as it is formatted differently function preload() { json_data = loadJSON("weather_data.json"); - // set the options to initialize timeSeries Neural Network + // Set the options to initialize timeSeries Neural Network let options = { task: "sequenceRegression", debug: "true", @@ -44,16 +44,16 @@ function setup() { createCanvas(640, 400); background(220); - //run a sliding window algorithm for time based data + // Run a sliding window algorithm for time based data let batchData = model.slidingWindow(data, features, targets, windowLength); let inputs = batchData.sequences; let outputs = batchData.targets; - // feed data into the model + // Feed data into the model for (let i = 0; i < inputs.length; i++) { model.addData(inputs[i], outputs[i]); } - // normalize the data after adding everything + // Normalize the data after adding everything model.normalizeData(); let options = { @@ -73,7 +73,7 @@ function draw() { text("Predicted Precipitation", 320, 200); text(precipitation, 320, 250); - // helpful visual based on predicted value + // Helpful visual based on predicted value push(); textSize(precipitation * 5 + 10); text("🌧️", 320, 150); @@ -81,23 +81,23 @@ function draw() { } } -// predict data +// Predict data function predictData() { - seq = model.sampleWindow(data); //helper function paired with the slidingWindow to get sample from data + seq = model.sampleWindow(data); //Helper function paired with the slidingWindow to get sample from data model.predict(seq, gotResults); } -// put the new data in the dataset so this will be considered for any new predictions +// Put the new data in the dataset so this will be considered for any new predictions function gotResults(results) { precipitation = results[4].value; - addNewData(results); //optional but will be helpful in using new prediction as part of dataset + addNewData(results); // Optional but will be helpful in using new prediction as part of dataset } -// code for adding new data to the dataset to be used for future prediction +// Code for adding new data to the dataset to be used for future prediction function addNewData(results) { (new_values = { date: " for the next hour", - temperature: results[0].value, // get string convert to float and round to 2 decimal points + temperature: results[0].value, // Get string convert to float and round to 2 decimal points humidity: results[1].value, wind_speed: results[2].value, pressure: results[3].value, @@ -110,7 +110,7 @@ function finishedTraining() { state = "prediction"; } -// get buttons and assign functions (UI) +// Get buttons and assign functions (UI) function UI() { pred_but = select("#pred_but"); pred_but.mouseClicked(predictData); From a6afd6dfaa55dac4bb229de875c5cda64b679736 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Mon, 30 Jun 2025 21:56:14 -0400 Subject: [PATCH 13/47] Changed Class names to Sequential --- src/NeuralNetwork/Sequential/index.js | 28 +++++++++---------- .../Sequential/{tsLayers.js => seqLayers.js} | 2 +- .../{timeSeriesData.js => sequentialData.js} | 13 ++++----- ...{timeSeriesUtils.js => sequentialUtils.js} | 8 +++--- src/NeuralNetwork/taskSelection.js | 8 +++--- 5 files changed, 28 insertions(+), 31 deletions(-) rename src/NeuralNetwork/Sequential/{tsLayers.js => seqLayers.js} (98%) rename src/NeuralNetwork/Sequential/{timeSeriesData.js => sequentialData.js} (96%) rename src/NeuralNetwork/Sequential/{timeSeriesUtils.js => sequentialUtils.js} (99%) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 4ae04427..362634c7 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -1,18 +1,16 @@ import * as tf from "@tensorflow/tfjs"; - import { DiyNeuralNetwork } from ".."; import callCallback from "../../utils/callcallback"; import setBackend from "../../utils/setBackend"; -import tsUtils from "./timeSeriesUtils"; - -import TimeSeriesData from "./timeSeriesData"; -import { createTsLayers } from "./tsLayers"; +import seqUtils from "./sequentialUtils"; +import SequentialData from "./sequentialData"; +import { createSeqLayers } from "./seqLayers"; // call an extension of DIY Neural Network as a new class, override select methods // which are seen below: -class DIYTimesSeries extends DiyNeuralNetwork { +class DIYSequential extends DiyNeuralNetwork { constructor(options, callback) { super( { @@ -25,7 +23,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { this.options = { ...this.options, ...(options || {}) }; this.neuralNetworkData = - this.options.neuralNetworkData || new TimeSeriesData(); + this.options.neuralNetworkData || new SequentialData(); this.init = this.init.bind(this); this.ready = callCallback(this.init(), callback); @@ -43,10 +41,10 @@ class DIYTimesSeries extends DiyNeuralNetwork { addData(xInputs, yInputs, options = null) { // 1. verify format between the three possible types of xinputs - const xs = tsUtils.verifyAndFormatInputs(xInputs, options, this.options); + const xs = seqUtils.verifyAndFormatInputs(xInputs, options, this.options); // 2. format the yInput - same logic as NN class - const ys = tsUtils.verifyAndFormatOutputs(yInputs, options, this.options); + const ys = seqUtils.verifyAndFormatOutputs(yInputs, options, this.options); // 3. add data to raw this.neuralNetworkData.addData(xs, ys); @@ -56,7 +54,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { const { meta } = this.neuralNetworkData; const inputHeaders = Object.keys(meta.inputs); - const formatted_inputs = tsUtils.verifyAndFormatInputs( + const formatted_inputs = seqUtils.verifyAndFormatInputs( _input, null, this.options @@ -87,7 +85,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { addDefaultLayers() { let layers; - const tsLayers = createTsLayers( + const tsLayers = createSeqLayers( this.neuralNetworkData.meta.seriesShape, this.options.hiddenUnits, this.numberOfClasses // For output units if needed @@ -175,7 +173,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { // RDP algorithm padCoordinates(coordinates, targetPointCount) { const maxEpsilon = int(coordinates.length / 2); - return tsUtils.padCoordinates(coordinates, targetPointCount, maxEpsilon); + return seqUtils.padCoordinates(coordinates, targetPointCount, maxEpsilon); } slidingWindow(data, featureKeys, targetKeys, batchLength = null) { @@ -189,7 +187,7 @@ class DIYTimesSeries extends DiyNeuralNetwork { this.batchLength = batchLength; } - return tsUtils.createSlidingWindowData( + return seqUtils.createSlidingWindowData( data, this.batchLength, this.featureKeys, @@ -203,8 +201,8 @@ class DIYTimesSeries extends DiyNeuralNetwork { "Your data must be formated through the slidingWindow method first!" ); } - return tsUtils.getLatestSequence(data, this.batchLength, this.featureKeys); + return seqUtils.getLatestSequence(data, this.batchLength, this.featureKeys); } } -export default DIYTimesSeries; //export for taskSelection +export default DIYSequential; //export for taskSelection diff --git a/src/NeuralNetwork/Sequential/tsLayers.js b/src/NeuralNetwork/Sequential/seqLayers.js similarity index 98% rename from src/NeuralNetwork/Sequential/tsLayers.js rename to src/NeuralNetwork/Sequential/seqLayers.js index 4a6acfe8..aa053f93 100644 --- a/src/NeuralNetwork/Sequential/tsLayers.js +++ b/src/NeuralNetwork/Sequential/seqLayers.js @@ -1,4 +1,4 @@ -export const createTsLayers = ( +export const createSeqLayers = ( seriesShape, hiddenUnits, outputUnits = null diff --git a/src/NeuralNetwork/Sequential/timeSeriesData.js b/src/NeuralNetwork/Sequential/sequentialData.js similarity index 96% rename from src/NeuralNetwork/Sequential/timeSeriesData.js rename to src/NeuralNetwork/Sequential/sequentialData.js index b64352bb..f100a241 100644 --- a/src/NeuralNetwork/Sequential/timeSeriesData.js +++ b/src/NeuralNetwork/Sequential/sequentialData.js @@ -3,12 +3,11 @@ import axios from "axios"; import { saveBlob } from "../../utils/io"; import modelLoader from "../../utils/modelLoader"; import nnUtils from "../NeuralNetworkUtils"; - -import tsUtils from "./timeSeriesUtils"; +import seqUtils from "./sequentialUtils"; import NeuralNetworkData from "../NeuralNetworkData"; -class TimeSeriesData extends NeuralNetworkData { +class SequentialData extends NeuralNetworkData { constructor() { super(); } @@ -133,7 +132,7 @@ class TimeSeriesData extends NeuralNetworkData { normalizeDataRaw() { const normXs = this.normalizeInputData(this.meta.inputs, "xs"); const normYs = this.normalizeInputData(this.meta.outputs, "ys"); - const normalizedData = tsUtils.zipArraySequence(normXs, normYs); + const normalizedData = seqUtils.zipArraySequence(normXs, normYs); return normalizedData; } @@ -217,7 +216,7 @@ class TimeSeriesData extends NeuralNetworkData { } // reshaping - output = tsUtils.reshapeTo3DArray(zipped, [ + output = seqUtils.reshapeTo3DArray(zipped, [ batch, seriesStep, feature_length, @@ -264,7 +263,7 @@ class TimeSeriesData extends NeuralNetworkData { }); } // reshaping - const output = tsUtils.reshapeTo3DArray(zipped, [ + const output = seqUtils.reshapeTo3DArray(zipped, [ batch, seriesStep, feature_length, @@ -359,4 +358,4 @@ class TimeSeriesData extends NeuralNetworkData { } } -export default TimeSeriesData; +export default SequentialData; diff --git a/src/NeuralNetwork/Sequential/timeSeriesUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js similarity index 99% rename from src/NeuralNetwork/Sequential/timeSeriesUtils.js rename to src/NeuralNetwork/Sequential/sequentialUtils.js index a9fcb912..f8a54ae1 100644 --- a/src/NeuralNetwork/Sequential/timeSeriesUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -1,7 +1,7 @@ import { data, input } from "@tensorflow/tfjs"; import nnUtils from "../NeuralNetworkUtils"; -class TimeSeriesUtils { +class SequentialUtils { constructor(options) { this.options = options || {}; } @@ -479,9 +479,9 @@ class TimeSeriesUtils { } } -const timeSeriesUtils = () => { - const instance = new TimeSeriesUtils(); +const sequentialUtils = () => { + const instance = new SequentialUtils(); return instance; }; -export default timeSeriesUtils(); +export default sequentialUtils(); diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js index b0277dea..74ecc78a 100644 --- a/src/NeuralNetwork/taskSelection.js +++ b/src/NeuralNetwork/taskSelection.js @@ -1,5 +1,5 @@ import DiyNeuralNetwork from "./index.js"; -import DIYTimesSeries from "./Sequential/index.js"; +import DIYSequential from "./Sequential/index.js"; // helper function to check if tasks follows specified convention const isTimeSeriesTask = (task) => { @@ -33,7 +33,7 @@ const createNeuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { }; // factory function for DIY Timeseries -const createTimeSeries = (inputsOrOptions, outputsOrCallback, callback) => { +const createSequential = (inputsOrOptions, outputsOrCallback, callback) => { let options; let cb; @@ -48,7 +48,7 @@ const createTimeSeries = (inputsOrOptions, outputsOrCallback, callback) => { cb = callback; } - const instance = new DIYTimesSeries(options, cb); + const instance = new DIYSequential(options, cb); return instance; }; @@ -68,7 +68,7 @@ const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { // Choose which factory function to call based on task if (isTimeSeriesTask(options.task)) { - return createTimeSeries(inputsOrOptions, outputsOrCallback, callback); + return createSequential(inputsOrOptions, outputsOrCallback, callback); } else { return createNeuralNetwork(inputsOrOptions, outputsOrCallback, callback); } From 174b1d81ba7f66b7842de494ab6251290edc3c6b Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 10 Jul 2025 10:25:26 -0400 Subject: [PATCH 14/47] changes based on PR comments --- .../index.html | 19 -- .../sketch.js | 69 +++- .../index.html | 11 - .../model/model.json | 0 .../model/model.weights.bin | Bin .../model/model_meta.json | 0 .../sketch.js | 43 ++- .../index.html | 11 +- .../sketch.js | 13 +- .../index.html | 24 ++ .../sketch.js | 148 ++++++++ .../index.html | 9 +- .../sketch.js | 42 ++- src/NeuralNetwork/Sequential/index.js | 5 + .../Sequential/sequentialUtils.js | 317 ++++++++++++++++-- src/NeuralNetwork/index.js | 2 +- src/NeuralNetwork/taskSelection.js | 12 +- 17 files changed, 605 insertions(+), 120 deletions(-) rename examples/{neuralNetwork-sequence-hand-gestures => neuralNetwork-sequence-hand-gesture}/index.html (55%) rename examples/{neuralNetwork-sequence-hand-gestures => neuralNetwork-sequence-hand-gesture}/sketch.js (73%) rename examples/{neuralNetwork-sequence-load-model-hand-gestures => neuralNetwork-sequence-hand-gestures-load-model}/index.html (74%) rename examples/{neuralNetwork-sequence-load-model-hand-gestures => neuralNetwork-sequence-hand-gestures-load-model}/model/model.json (100%) rename examples/{neuralNetwork-sequence-load-model-hand-gestures => neuralNetwork-sequence-hand-gestures-load-model}/model/model.weights.bin (100%) rename examples/{neuralNetwork-sequence-load-model-hand-gestures => neuralNetwork-sequence-hand-gestures-load-model}/model/model_meta.json (100%) rename examples/{neuralNetwork-sequence-load-model-hand-gestures => neuralNetwork-sequence-hand-gestures-load-model}/sketch.js (77%) rename examples/{neuralNetwork-sequence-train-mouse-gesture RDP => neuralNetwork-sequence-mouse-gesture-rdp}/index.html (80%) rename examples/{neuralNetwork-sequence-train-mouse-gesture RDP => neuralNetwork-sequence-mouse-gesture-rdp}/sketch.js (93%) create mode 100644 examples/neuralNetwork-sequence-rdp-visualizer/index.html create mode 100644 examples/neuralNetwork-sequence-rdp-visualizer/sketch.js diff --git a/examples/neuralNetwork-sequence-hand-gestures/index.html b/examples/neuralNetwork-sequence-hand-gesture/index.html similarity index 55% rename from examples/neuralNetwork-sequence-hand-gestures/index.html rename to examples/neuralNetwork-sequence-hand-gesture/index.html index 1fdd80c3..afceb7f0 100644 --- a/examples/neuralNetwork-sequence-hand-gestures/index.html +++ b/examples/neuralNetwork-sequence-hand-gesture/index.html @@ -19,24 +19,5 @@
- -

- Instructions:
- 1.) To add data, press the "Add Gesture #1 Data" button
- 2.) Use one hand to do a gesture in front of the camera
- 3.) Once done with the gesture, take your hand out of the frame
- 4.) Repeat until there is a good amount of data
- 5.) Do the same for gesture #2 by pressing the "Add Gesture #2 Data" - button before doing the hand gesture in the frame
- 6.) Press the "Train and Save Model" button to train the model and save it - for future use.
-

- - diff --git a/examples/neuralNetwork-sequence-hand-gestures/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js similarity index 73% rename from examples/neuralNetwork-sequence-hand-gestures/sketch.js rename to examples/neuralNetwork-sequence-hand-gesture/sketch.js index d0e65f6c..1f2dd9bd 100644 --- a/examples/neuralNetwork-sequence-hand-gestures/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -13,17 +13,19 @@ let hands = []; let sequence = []; let targetLength = 30; -let gestures = ["Gesture #1", "Gesture #2"]; -let counts = { "Gesture #1": 0, "Gesture #2": 0 }; +let gestures = ["gesture #1", "gesture #2"]; +let counts = { "gesture #1": 0, "gesture #2": 0 }; -let state = "collection"; +let state = "collecting"; let currGesture = gestures[0]; // Set currGesture to gesture 1 by default +let predGesture = ""; function preload() { // Load the handPose model - handPose = ml5.handPose(); + // Set options to have data points flipped + handPose = ml5.handPose({ flipHorizontal: true }); - // Setup the timeseries neural network + // Setup the neural network using sequenceClassification let options = { outputs: ["label"], task: "sequenceClassificationConv", @@ -38,7 +40,7 @@ function setup() { canvas.parent("canvasDiv"); // Setup video capture - video = createCapture(VIDEO); + video = createCapture(VIDEO, { flipped: true }); video.size(640, 480); video.hide(); @@ -53,20 +55,28 @@ function draw() { // Draw video on frame image(video, 0, 0, width, height); - // Helpful tooltip to include - textSize(20); - stroke(255); - fill(0); - text(state + " : " + currGesture, 50, 50); - // If hand is detected in the frame, start recording gesture if (hands.length > 0) { handpoints = drawPoints(); sequence.push(handpoints); - // Add collected data to model once the hand is gone and state is collection + // Helpful text to signify recording + textSize(20); + stroke(255); + fill(0); + if (state == "collecting") { + text( + state + " : " + currGesture + ", put hand down once done with gesture", + 50, + 50 + ); + } else if (state == "prediction") { + text("predicting... put hand down once done with gesture", 50, 50); + } + + // Add collected data to model once the hand is gone and state is collecting } else if (hands.length <= 0 && sequence.length > 0) { - if (state == "collection") { + if (state == "collecting") { // Pad the length of the coordinates to targetLength let inputData = model.padCoordinates(sequence, targetLength); let outputData = { label: currGesture }; @@ -74,7 +84,7 @@ function draw() { // Add data to the model model.addData(inputData, outputData); - // Udate the counts for the UI + // Update the counts for the UI counts[currGesture]++; updateDataCountUI(); @@ -86,6 +96,29 @@ function draw() { // Reset the sequence sequence = []; + + // Tell users to put hand up to start recording + } else { + textSize(20); + stroke(255); + fill(0); + if (state == "collecting") { + text( + "put hand up in screen to start collecting for: " + currGesture, + 50, + 50 + ); + } else if (state == "prediction") { + if (!predGesture) { + text("do one of the trained gestures to predict", 50, 50); + } else { + text( + "prediction: " + predGesture + ", try again with another gesture!", + 50, + 50 + ); + } + } } } @@ -111,7 +144,7 @@ function finishedTraining() { // Callback for predict function gotResults(results) { - currGesture = results[0].label; + predGesture = results[0].label; } // Callback function for when handPose outputs data @@ -151,9 +184,9 @@ function UI() { "
Gesture 2 data: " + counts[gestures[0]] ); - rockButton = createButton("Add Gesture #1 Data"); + rockButton = createButton("Record Gesture #1"); rockButton.mousePressed(addGesture1); - paperButton = createButton("Add Gesture #2 Data"); + paperButton = createButton("Record Gesture #2"); paperButton.mousePressed(addGesture2); trainButton = createButton("Train and Save Model"); trainButton.mousePressed(train); diff --git a/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html b/examples/neuralNetwork-sequence-hand-gestures-load-model/index.html similarity index 74% rename from examples/neuralNetwork-sequence-load-model-hand-gestures/index.html rename to examples/neuralNetwork-sequence-hand-gestures-load-model/index.html index 0d8ae703..33bcf110 100644 --- a/examples/neuralNetwork-sequence-load-model-hand-gestures/index.html +++ b/examples/neuralNetwork-sequence-hand-gestures-load-model/index.html @@ -20,17 +20,6 @@

- This example loads a model that is trained with ASL hand gestures for - Hello and Goodbye.
-
- - Instructions:
- 1.) Use one hand to do a gesture in front of the camera
- 2.) Wait for the points to disappear or the prediction appears on - screen
- 3.) To predict again, remove your hands in the frame and do the gesture - again

- How to do gestures for Hello and Goodbye in ASL:
Hello: 0) { // Get coordinates from hands (21 points) handpoints = drawPoints(); sequence.push(handpoints); + // Helpful text to signify recording + textSize(20); + stroke(255); + fill(0); + text("predicting... put hand down once done with gesture", 50, 50); + // Pad the data and use for prediction } else if (hands.length <= 0 && sequence.length > 0) { let predictData = model.padCoordinates(sequence, targetLength); @@ -79,6 +83,21 @@ function draw() { // Reset the sequence sequence = []; + + // Tell users to put hand up to start recording + } else { + textSize(20); + stroke(255); + fill(0); + if (!predGesture) { + text("do one of the gestures below to predict", 50, 50); + } else { + text( + "prediction: " + predGesture + ", try again with another gesture!", + 50, + 50 + ); + } } } @@ -109,13 +128,5 @@ function gotHands(results) { // Call back for accessing the results function gotResults(results) { - predictedWord = results[0].label; - text(predictedWord, 100, 100); -} - -// For drawing text on screen -function placePredictedText() { - textSize(20); - fill(0); - text("Predicted Gesture : " + predictedWord, 50, 50); + predGesture = results[0].label; } diff --git a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html similarity index 80% rename from examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html rename to examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html index 0e99a9ee..7b7ba8cc 100644 --- a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/index.html +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html @@ -22,16 +22,9 @@

- - diff --git a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js similarity index 93% rename from examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js rename to examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index 30420b75..e61ec9a6 100644 --- a/examples/neuralNetwork-sequence-train-mouse-gesture RDP/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -8,7 +8,7 @@ let model; let currShape = "circle"; -let state = "collection"; +let state = "collecting"; let sequence = []; let targetLength = 30; @@ -45,11 +45,12 @@ function draw() { // Code to signify drawing can be done again function mouseReleased() { if (mouseY < height && mouseX < width) { - // If state is collection, add whole sequence as X, and shape as Y - if (state == "collection") { + // If state is collecting, add whole sequence as X, and shape as Y + if (state == "collecting") { let target = { label: currShape }; let paddedCoordinates = model.padCoordinates(sequence, targetLength); model.addData(paddedCoordinates, target); + clearScreen(); } else if (state == "prediction") { let paddedCoordinates = model.padCoordinates(sequence, targetLength); @@ -105,8 +106,10 @@ function UI() { recSquare.mouseClicked(recordSquare); trainBut.mouseClicked(trainModel); + text(state + " : " + currShape, 50, 50); + function recordCircle() { - state = "collection"; + state = "collecting"; currShape = "circle"; background(220); @@ -114,7 +117,7 @@ function UI() { } function recordSquare() { - state = "collection"; + state = "collecting"; currShape = "square"; background(220); diff --git a/examples/neuralNetwork-sequence-rdp-visualizer/index.html b/examples/neuralNetwork-sequence-rdp-visualizer/index.html new file mode 100644 index 00000000..c8077a2a --- /dev/null +++ b/examples/neuralNetwork-sequence-rdp-visualizer/index.html @@ -0,0 +1,24 @@ + + + + + + + + ml5.js Ramer–Douglas–Peucker algorithm Visualizer + + + + + +
+ +
+ + diff --git a/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js b/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js new file mode 100644 index 00000000..44739f71 --- /dev/null +++ b/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js @@ -0,0 +1,148 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates the Ramer–Douglas–Peucker algorithm to adjust points from a variable length to a target length. + */ +let allPoints = []; +const targetPointCount = 10; // Desired number of points +let maxEpsilon = 30; // Adjust as needed + +function setup() { + createCanvas(600, 400); + background(0); + clearButton = createButton("clear screen"); + clearButton.mousePressed(clearScreen); +} + +function draw() { + background(0); + + if (mouseIsPressed && mouseY < height && mouseX < width) { + allPoints.push(createVector(mouseX, mouseY)); + } + + if (allPoints.length > 10) { + rpdeia(); + } +} + +function rpdeia() { + const rdpPoints = []; + + const epsilon = findEpsilonForPointCount(allPoints, targetPointCount); + + const total = allPoints.length; + const start = allPoints[0]; + const end = allPoints[total - 1]; + rdpPoints.push(start); + rdp(0, total - 1, allPoints, rdpPoints, epsilon); + rdpPoints.push(end); + + stroke(255, 0, 255); + strokeWeight(4); + noFill(); + beginShape(); + for (let v of allPoints) { + vertex(v.x, v.y); + } + endShape(); + + stroke(255); + strokeWeight(2); + beginShape(); + for (let v of rdpPoints) { + vertex(v.x, v.y); + } + endShape(); + + fill(255); + noStroke(); + textSize(24); + text("mouse-created points: " + allPoints.length, 20, 25); + text("rdp-created points: " + rdpPoints.length, 20, 50); +} + +function findEpsilonForPointCount(points, targetCount) { + let low = 0; + let high = maxEpsilon; + let mid; + let simplifiedPointsCount = 0; + + while (high - low > 0.001) { + // Tolerance for approximation + mid = (low + high) / 2; + simplifiedPointsCount = getSimplifiedPointCount(points, mid); + if (simplifiedPointsCount > targetCount) { + low = mid; + } else { + high = mid; + } + } + + return mid; +} + +function getSimplifiedPointCount(points, epsilon) { + const rdpPoints = []; + const total = points.length; + const start = points[0]; + const end = points[total - 1]; + rdpPoints.push(start); + rdp(0, total - 1, points, rdpPoints, epsilon); + rdpPoints.push(end); + return rdpPoints.length; +} + +function rdp(startIndex, endIndex, allPoints, rdpPoints, epsilon) { + const nextIndex = findFurthest(allPoints, startIndex, endIndex, epsilon); + if (nextIndex > 0) { + if (startIndex != nextIndex) { + rdp(startIndex, nextIndex, allPoints, rdpPoints, epsilon); + } + rdpPoints.push(allPoints[nextIndex]); + if (endIndex != nextIndex) { + rdp(nextIndex, endIndex, allPoints, rdpPoints, epsilon); + } + } +} + +function findFurthest(points, a, b, epsilon) { + let recordDistance = -1; + const start = points[a]; + const end = points[b]; + let furthestIndex = -1; + for (let i = a + 1; i < b; i++) { + const currentPoint = points[i]; + const d = lineDist(currentPoint, start, end); + if (d > recordDistance) { + recordDistance = d; + furthestIndex = i; + } + } + if (recordDistance > epsilon) { + return furthestIndex; + } else { + return -1; + } +} + +function lineDist(c, a, b) { + const norm = scalarProjection(c, a, b); + return p5.Vector.dist(c, norm); +} + +function scalarProjection(p, a, b) { + const ap = p5.Vector.sub(p, a); + const ab = p5.Vector.sub(b, a); + ab.normalize(); // Normalize the line + ab.mult(ap.dot(ab)); + const normalPoint = p5.Vector.add(a, ab); + return normalPoint; +} + +function clearScreen() { + allPoints = []; + background(0); +} diff --git a/examples/neuralNetwork-sequence-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html index 353148c0..366dc283 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/index.html +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -19,15 +19,8 @@ -
+
- - diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index de3d5a10..9ba5316b 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -8,6 +8,7 @@ let model; let data; +let graphValues = []; let state = "training"; let precipitation = ""; @@ -28,7 +29,7 @@ let windowLength = 10; // Optional: define the size of the window for batch function preload() { json_data = loadJSON("weather_data.json"); - // Set the options to initialize timeSeries Neural Network + // Set the options to initialize Neural Network wit sequenceRegression Task let options = { task: "sequenceRegression", debug: "true", @@ -79,6 +80,7 @@ function draw() { text("🌧️", 320, 150); pop(); } + drawBarGraph(); } // Predict data @@ -94,16 +96,22 @@ function gotResults(results) { } // Code for adding new data to the dataset to be used for future prediction -function addNewData(results) { +function addNewData(newResults) { (new_values = { date: " for the next hour", - temperature: results[0].value, // Get string convert to float and round to 2 decimal points - humidity: results[1].value, - wind_speed: results[2].value, - pressure: results[3].value, - precipitation: results[4].value, + temperature: newResults[0].value, // Get string convert to float and round to 2 decimal points + humidity: newResults[1].value, + wind_speed: newResults[2].value, + pressure: newResults[3].value, + precipitation: newResults[4].value, }), data.push(new_values); + + // Add data to the bar graph + graphValues.push(newResults[4].value); + if (graphValues.length > maxBars) { + graphValues.shift(); // Remove first element + } } function finishedTraining() { @@ -116,4 +124,24 @@ function UI() { pred_but.mouseClicked(predictData); textAlign(CENTER); + + maxBars = 12; + barWidth = width / maxBars; + maxDataValue = 35; +} + +function drawBarGraph() { + for (let i = 0; i < graphValues.length && i < maxBars; i++) { + let barHeight = map(graphValues[i], 0, maxDataValue, 0, height - 180); + let x = i * barWidth; + let y = height - barHeight - 20; + + // Bar color gradient based on value + let barColor = map(graphValues[i], 0, maxDataValue, 0, 255); + fill(barColor, 100, 255 - barColor); + stroke(100); + + // Draw bar + rect(x + 5, y, barWidth - 10, barHeight); + } } diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 362634c7..69c99c16 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -69,6 +69,11 @@ class DIYSequential extends DiyNeuralNetwork { } createMetaData() { + // check if the data is empty + if (this.neuralNetworkData.data.raw.length <= 0) { + throw new Error("Must add data before training!"); + } + // this method does not get shape for images but instead for timesteps const { inputs } = this.options; diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index f8a54ae1..9dc03254 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -276,19 +276,77 @@ class SequentialUtils { // point simplification utilities - Ramer-Douglas-Peucker (RDP) algorithm padCoordinates(allPoints, targetPointCount, maxEpsilon = 50) { - const rdpPoints = []; + if (allPoints.length === 0) return []; + + // Check if it's an array of objects with .x and .y properties + if (this.isObjectFormat(allPoints[0])) { + // Original format: array of objects with .x and .y + return this.simplifyObjectCoordinates( + allPoints, + targetPointCount, + maxEpsilon + ); + } - const epsilon = this.findEpsilonForPointCount( + // Array format + const numColumns = allPoints[0].length; + + if (numColumns === 2) { + // Simple case: [n,2] - just one pair of coordinates + return this.simplifyCoordinatePair( + allPoints, + targetPointCount, + maxEpsilon + ); + } else if (numColumns % 2 === 0) { + // Multiple coordinate pairs: [n,42] -> 21 pairs + const numPairs = numColumns / 2; + const simplifiedPairs = []; + + // Process each coordinate pair separately + for (let pairIndex = 0; pairIndex < numPairs; pairIndex++) { + const coordinatePair = this.extractCoordinatePair(allPoints, pairIndex); + const simplified = this.simplifyCoordinatePair( + coordinatePair, + targetPointCount, + maxEpsilon + ); + simplifiedPairs.push(simplified); + } + + // Recombine all simplified pairs back into [targetPointCount, 42] format + return this.recombineCoordinatePairs(simplifiedPairs, targetPointCount); + } else { + throw new Error( + `Invalid array format: expected even number of columns, got ${numColumns}` + ); + } + } + + // Check if data is in object format (has .x and .y properties) + isObjectFormat(point) { + return ( + point && + typeof point === "object" && + typeof point.x === "number" && + typeof point.y === "number" + ); + } + + // Handle original object format - your exact original algorithm + simplifyObjectCoordinates(allPoints, targetPointCount, maxEpsilon) { + const rdpPoints = []; + const epsilon = this.findEpsilonForPointCountObjects( allPoints, targetPointCount, maxEpsilon ); - const total = allPoints.length; const start = allPoints[0]; const end = allPoints[total - 1]; + rdpPoints.push(start); - this.rdp(0, total - 1, allPoints, rdpPoints, epsilon); + this.rdpObjects(0, total - 1, allPoints, rdpPoints, epsilon); rdpPoints.push(end); if (rdpPoints.length > targetPointCount) { @@ -297,45 +355,249 @@ class SequentialUtils { const filler = new Array(targetPointCount - rdpPoints.length).fill( rdpPoints[rdpPoints.length - 1] ); - rdpPoints.push(...filler); return rdpPoints; } - return rdpPoints; } - findEpsilonForPointCount(points, targetCount, maxEpsilon) { + // Original object-based helper functions + findEpsilonForPointCountObjects(points, targetCount, maxEpsilon) { let low = 0; let high = maxEpsilon; let mid; let simplifiedPointsCount = 0; while (high - low > 0.001) { - // Tolerance for approximation mid = (low + high) / 2; - simplifiedPointsCount = this.getSimplifiedPointCount(points, mid); + simplifiedPointsCount = this.getSimplifiedPointCountObjects(points, mid); if (simplifiedPointsCount > targetCount) { low = mid; } else { high = mid; } } + return mid; + } + + getSimplifiedPointCountObjects(points, epsilon) { + const rdpPoints = []; + const total = points.length; + const start = points[0]; + const end = points[total - 1]; + rdpPoints.push(start); + this.rdpObjects(0, total - 1, points, rdpPoints, epsilon); + rdpPoints.push(end); + return rdpPoints.length; + } + + rdpObjects(startIndex, endIndex, allPoints, rdpPoints, epsilon) { + const nextIndex = this.findFurthestObjects( + allPoints, + startIndex, + endIndex, + epsilon + ); + if (nextIndex > 0) { + if (startIndex !== nextIndex) { + this.rdpObjects(startIndex, nextIndex, allPoints, rdpPoints, epsilon); + } + rdpPoints.push(allPoints[nextIndex]); + if (endIndex !== nextIndex) { + this.rdpObjects(nextIndex, endIndex, allPoints, rdpPoints, epsilon); + } + } + } + + findFurthestObjects(points, a, b, epsilon) { + let recordDistance = -1; + const start = points[a]; + const end = points[b]; + let furthestIndex = -1; + + for (let i = a + 1; i < b; i++) { + const currentPoint = points[i]; + const d = this.lineDistObjects(currentPoint, start, end); + if (d > recordDistance) { + recordDistance = d; + furthestIndex = i; + } + } + + if (recordDistance > epsilon) { + return furthestIndex; + } else { + return -1; + } + } + + lineDistObjects(c, a, b) { + const norm = this.scalarProjectionObjects(c, a, b); + return Math.sqrt(Math.pow(c.x - norm.x, 2) + Math.pow(c.y - norm.y, 2)); + } + + scalarProjectionObjects(p, a, b) { + const ap = { x: p.x - a.x, y: p.y - a.y }; + const ab = { x: b.x - a.x, y: b.y - a.y }; + const abMag = Math.sqrt(ab.x * ab.x + ab.y * ab.y); + + if (abMag === 0) { + return a; + } + + ab.x /= abMag; + ab.y /= abMag; + const dot = ap.x * ab.x + ap.y * ab.y; + return { x: a.x + ab.x * dot, y: a.y + ab.y * dot }; + } + + // Extract one coordinate pair from the data + extractCoordinatePair(allPoints, pairIndex) { + const coordinatePair = []; + const xIndex = pairIndex * 2; + const yIndex = pairIndex * 2 + 1; + + for (let i = 0; i < allPoints.length; i++) { + coordinatePair.push([allPoints[i][xIndex], allPoints[i][yIndex]]); + } + + return coordinatePair; + } + + // Simplify a single coordinate pair using RDP + simplifyCoordinatePair(coordinatePair, targetPointCount, maxEpsilon) { + const rdpPoints = []; + const epsilon = this.findEpsilonForPointCount( + coordinatePair, + targetPointCount, + maxEpsilon + ); + const total = coordinatePair.length; + + if (total <= 2) { + // If we have 2 or fewer points, just pad to target + return this.padToTarget(coordinatePair, targetPointCount); + } + + const start = coordinatePair[0]; + const end = coordinatePair[total - 1]; + + rdpPoints.push(start); + this.rdp(0, total - 1, coordinatePair, rdpPoints, epsilon); + rdpPoints.push(end); + + // Remove duplicates and sort by original order + const uniquePoints = this.removeDuplicatesAndSort( + rdpPoints, + coordinatePair + ); + + return this.padToTarget(uniquePoints, targetPointCount); + } + + // Recombine simplified coordinate pairs back into original format + recombineCoordinatePairs(simplifiedPairs, targetPointCount) { + const result = []; + + for (let i = 0; i < targetPointCount; i++) { + const row = []; + for (let pairIndex = 0; pairIndex < simplifiedPairs.length; pairIndex++) { + const pair = simplifiedPairs[pairIndex][i]; + row.push(pair[0], pair[1]); // x, y + } + result.push(row); + } + + return result; + } + + // Remove duplicates and maintain original order + removeDuplicatesAndSort(rdpPoints, originalPoints) { + const seen = new Set(); + const unique = []; + + // Create a map of original indices + const indexMap = new Map(); + for (let i = 0; i < originalPoints.length; i++) { + const key = `${originalPoints[i][0]},${originalPoints[i][1]}`; + if (!indexMap.has(key)) { + indexMap.set(key, i); + } + } + + // Sort by original index and remove duplicates + rdpPoints.sort((a, b) => { + const keyA = `${a[0]},${a[1]}`; + const keyB = `${b[0]},${b[1]}`; + return indexMap.get(keyA) - indexMap.get(keyB); + }); + + for (const point of rdpPoints) { + const key = `${point[0]},${point[1]}`; + if (!seen.has(key)) { + seen.add(key); + unique.push(point); + } + } + + return unique; + } + + // Pad array to target length + padToTarget(points, targetPointCount) { + if (points.length > targetPointCount) { + return points.slice(0, targetPointCount); + } else if (points.length < targetPointCount) { + const filler = new Array(targetPointCount - points.length).fill( + points[points.length - 1] + ); + return [...points, ...filler]; + } + return points; + } + + // Find epsilon that gives approximately the target point count + findEpsilonForPointCount(points, targetCount, maxEpsilon) { + if (points.length <= targetCount) { + return 0; // No simplification needed + } + + let low = 0; + let high = maxEpsilon; + let mid; + + while (high - low > 0.001) { + mid = (low + high) / 2; + const simplifiedCount = this.getSimplifiedPointCount(points, mid); + + if (simplifiedCount > targetCount) { + low = mid; + } else { + high = mid; + } + } return mid; } + // Get count of points after simplification with given epsilon getSimplifiedPointCount(points, epsilon) { const rdpPoints = []; const total = points.length; + + if (total <= 2) return total; + const start = points[0]; const end = points[total - 1]; + rdpPoints.push(start); this.rdp(0, total - 1, points, rdpPoints, epsilon); rdpPoints.push(end); - return rdpPoints.length; + + return new Set(rdpPoints.map((p) => `${p[0]},${p[1]}`)).size; } + // Core RDP algorithm rdp(startIndex, endIndex, allPoints, rdpPoints, epsilon) { const nextIndex = this.findFurthest( allPoints, @@ -343,30 +605,35 @@ class SequentialUtils { endIndex, epsilon ); + if (nextIndex > 0) { - if (startIndex != nextIndex) { + if (startIndex !== nextIndex) { this.rdp(startIndex, nextIndex, allPoints, rdpPoints, epsilon); } rdpPoints.push(allPoints[nextIndex]); - if (endIndex != nextIndex) { + if (endIndex !== nextIndex) { this.rdp(nextIndex, endIndex, allPoints, rdpPoints, epsilon); } } } + // Find furthest point from line segment findFurthest(points, a, b, epsilon) { let recordDistance = -1; const start = points[a]; const end = points[b]; let furthestIndex = -1; + for (let i = a + 1; i < b; i++) { const currentPoint = points[i]; const d = this.lineDist(currentPoint, start, end); + if (d > recordDistance) { recordDistance = d; furthestIndex = i; } } + if (recordDistance > epsilon) { return furthestIndex; } else { @@ -374,19 +641,29 @@ class SequentialUtils { } } + // Calculate distance from point to line lineDist(c, a, b) { const norm = this.scalarProjection(c, a, b); - return dist(c.x, c.y, norm.x, norm.y); + return Math.sqrt(Math.pow(c[0] - norm[0], 2) + Math.pow(c[1] - norm[1], 2)); } + // Project point onto line segment scalarProjection(p, a, b) { - const ap = { x: p.x - a.x, y: p.y - a.y }; - const ab = { x: b.x - a.x, y: b.y - a.y }; - const abMag = Math.sqrt(ab.x * ab.x + ab.y * ab.y); - ab.x /= abMag; - ab.y /= abMag; - const dot = ap.x * ab.x + ap.y * ab.y; - return { x: a.x + ab.x * dot, y: a.y + ab.y * dot }; + const ap = [p[0] - a[0], p[1] - a[1]]; + const ab = [b[0] - a[0], b[1] - a[1]]; + + const abMag = Math.sqrt(ab[0] * ab[0] + ab[1] * ab[1]); + + if (abMag === 0) { + return a; // Start and end points are the same + } + + ab[0] /= abMag; + ab[1] /= abMag; + + const dot = ap[0] * ab[0] + ap[1] * ab[1]; + + return [a[0] + ab[0] * dot, a[1] + ab[1] * dot]; } /** diff --git a/src/NeuralNetwork/index.js b/src/NeuralNetwork/index.js index 40ed5ec3..334523c1 100644 --- a/src/NeuralNetwork/index.js +++ b/src/NeuralNetwork/index.js @@ -1249,5 +1249,5 @@ class DiyNeuralNetwork { } } -export { DiyNeuralNetwork }; // Named export for extending in DIY TimeSeries +export { DiyNeuralNetwork }; // Named export for extending for Sequential Neural Network export default DiyNeuralNetwork; //export for taskSelection.js diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js index 74ecc78a..6dbea15f 100644 --- a/src/NeuralNetwork/taskSelection.js +++ b/src/NeuralNetwork/taskSelection.js @@ -2,14 +2,14 @@ import DiyNeuralNetwork from "./index.js"; import DIYSequential from "./Sequential/index.js"; // helper function to check if tasks follows specified convention -const isTimeSeriesTask = (task) => { - const timeSeriesTasks = [ +const isSequenceTask = (task) => { + const sequenceTask = [ "sequenceClassification", "sequenceRegression", "sequenceClassificationConv", "sequenceRegressionConv", ]; - return timeSeriesTasks.includes(task); + return sequenceTask.includes(task); }; // factory function for DIY Neural Network @@ -32,7 +32,7 @@ const createNeuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { return instance; }; -// factory function for DIY Timeseries +// factory function for DIY Sequential const createSequential = (inputsOrOptions, outputsOrCallback, callback) => { let options; let cb; @@ -52,7 +52,7 @@ const createSequential = (inputsOrOptions, outputsOrCallback, callback) => { return instance; }; -// Selection logic for either NeuralNetwork or TimeSeries +// Selection logic for either NeuralNetwork or Sequential const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { let options; @@ -67,7 +67,7 @@ const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { } // Choose which factory function to call based on task - if (isTimeSeriesTask(options.task)) { + if (isSequenceTask(options.task)) { return createSequential(inputsOrOptions, outputsOrCallback, callback); } else { return createNeuralNetwork(inputsOrOptions, outputsOrCallback, callback); From 3fc035a13fd3e8443e798bbe05874a28703665ac Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 10 Jul 2025 10:34:31 -0400 Subject: [PATCH 15/47] chore: changed gestures to gesture --- .../index.html | 0 .../model/model.json | 0 .../model/model.weights.bin | Bin .../model/model_meta.json | 0 .../sketch.js | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename examples/{neuralNetwork-sequence-hand-gestures-load-model => neuralNetwork-sequence-hand-gesture-load-model}/index.html (100%) rename examples/{neuralNetwork-sequence-hand-gestures-load-model => neuralNetwork-sequence-hand-gesture-load-model}/model/model.json (100%) rename examples/{neuralNetwork-sequence-hand-gestures-load-model => neuralNetwork-sequence-hand-gesture-load-model}/model/model.weights.bin (100%) rename examples/{neuralNetwork-sequence-hand-gestures-load-model => neuralNetwork-sequence-hand-gesture-load-model}/model/model_meta.json (100%) rename examples/{neuralNetwork-sequence-hand-gestures-load-model => neuralNetwork-sequence-hand-gesture-load-model}/sketch.js (100%) diff --git a/examples/neuralNetwork-sequence-hand-gestures-load-model/index.html b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html similarity index 100% rename from examples/neuralNetwork-sequence-hand-gestures-load-model/index.html rename to examples/neuralNetwork-sequence-hand-gesture-load-model/index.html diff --git a/examples/neuralNetwork-sequence-hand-gestures-load-model/model/model.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json similarity index 100% rename from examples/neuralNetwork-sequence-hand-gestures-load-model/model/model.json rename to examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json diff --git a/examples/neuralNetwork-sequence-hand-gestures-load-model/model/model.weights.bin b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin similarity index 100% rename from examples/neuralNetwork-sequence-hand-gestures-load-model/model/model.weights.bin rename to examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin diff --git a/examples/neuralNetwork-sequence-hand-gestures-load-model/model/model_meta.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json similarity index 100% rename from examples/neuralNetwork-sequence-hand-gestures-load-model/model/model_meta.json rename to examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json diff --git a/examples/neuralNetwork-sequence-hand-gestures-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js similarity index 100% rename from examples/neuralNetwork-sequence-hand-gestures-load-model/sketch.js rename to examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js From e70ab0e603e1c090bf7944c63b38b795a7ad13c9 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 10 Jul 2025 10:41:59 -0400 Subject: [PATCH 16/47] chore: replace Conv to withCNN --- .../sketch.js | 2 +- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 2 +- .../neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 2 +- src/NeuralNetwork/Sequential/index.js | 8 ++++---- src/NeuralNetwork/Sequential/seqLayers.js | 4 ++-- src/NeuralNetwork/taskSelection.js | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js index 4d8bd0ed..96f6aef1 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -27,7 +27,7 @@ function preload() { // Setup the neural network using sequenceClassification let options = { - task: "sequenceClassificationConv", + task: "sequenceClassificationWithCNN", }; model = ml5.neuralNetwork(options); diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 1f2dd9bd..d1df5f8f 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -28,7 +28,7 @@ function preload() { // Setup the neural network using sequenceClassification let options = { outputs: ["label"], - task: "sequenceClassificationConv", + task: "sequenceClassificationWithCNN", debug: "true", learningRate: 0.001, // The default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update) }; diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index e61ec9a6..a3b08163 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -17,7 +17,7 @@ function preload() { let options = { inputs: ["x", "y"], outputs: ["label"], - task: "sequenceClassificationConv", + task: "sequenceClassificationWithCNN", debug: true, learningRate: 0.005, // Learning rate decreased for better convergence }; diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 69c99c16..aefa7626 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -107,11 +107,11 @@ class DIYSequential extends DiyNeuralNetwork { layers = tsLayers.classification; return this.createNetworkLayers(layers); - case "sequenceClassificationConv": + case "sequenceClassificationWithCNN": layers = tsLayers.classificationConv; return this.createNetworkLayers(layers); - case "sequenceRegressionConv": + case "sequenceRegressionWithCNN": layers = tsLayers.regressionConv; return this.createNetworkLayers(layers); @@ -130,7 +130,7 @@ class DIYSequential extends DiyNeuralNetwork { if ( this.options.task === "sequenceClassification" || - this.options.task === "sequenceClassificationConv" + this.options.task === "sequenceClassificationWithCNN" ) { options = { loss: "categoricalCrossentropy", @@ -139,7 +139,7 @@ class DIYSequential extends DiyNeuralNetwork { }; } else if ( this.options.task === "sequenceRegression" || - this.options.task === "sequenceRegressionConv" + this.options.task === "sequenceRegressionWithCNN" ) { options = { loss: "meanSquaredError", diff --git a/src/NeuralNetwork/Sequential/seqLayers.js b/src/NeuralNetwork/Sequential/seqLayers.js index aa053f93..b3b30d54 100644 --- a/src/NeuralNetwork/Sequential/seqLayers.js +++ b/src/NeuralNetwork/Sequential/seqLayers.js @@ -4,7 +4,7 @@ export const createSeqLayers = ( outputUnits = null ) => { return { - classificationConv: [ + classificationWithCNN: [ { type: "conv1d", filters: 8, @@ -65,7 +65,7 @@ export const createSeqLayers = ( activation: "softmax", }, ], - regressionConv: [ + regressionWithCNN: [ { type: "conv1d", filters: 8, diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js index 6dbea15f..8db8b41e 100644 --- a/src/NeuralNetwork/taskSelection.js +++ b/src/NeuralNetwork/taskSelection.js @@ -6,8 +6,8 @@ const isSequenceTask = (task) => { const sequenceTask = [ "sequenceClassification", "sequenceRegression", - "sequenceClassificationConv", - "sequenceRegressionConv", + "sequenceClassificationWithCNN", + "sequenceRegressionWithCNN", ]; return sequenceTask.includes(task); }; From 8caffd3dea16cfa1bc03943880009c4f5bd765f0 Mon Sep 17 00:00:00 2001 From: gohai Date: Tue, 15 Jul 2025 16:03:24 +0800 Subject: [PATCH 17/47] Tweaks to neuralNetwork-sequence-weather-prediction example --- .../index.html | 7 +- .../sketch.js | 146 +++++++++--------- 2 files changed, 81 insertions(+), 72 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html index 366dc283..ad67986c 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/index.html +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -11,7 +11,7 @@ - ml5.js Neural Network Weather Prediction Example + ml5.js neuralNetwork Weather Prediction Example @@ -19,8 +19,9 @@ -
- +
+ +
diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 9ba5316b..f768626b 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -3,16 +3,13 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates Loading JSON data and Training a Weather Predictor through ml5.neuralNetwork with sequeceRegression Task. + * This example demonstrates loading JSON data and training a Weather Predictor + * through ml5.neuralNetwork with the sequeceRegression task. + * XXX: let us know what this dataset is from/about */ let model; let data; -let graphValues = []; - -let state = "training"; -let precipitation = ""; - let features = [ "temperature", "humidity", @@ -21,127 +18,138 @@ let features = [ "precipitation", ]; -let targets = features; // Must be the same to add predicted values back to data -let windowLength = 10; // Optional: define the size of the window for batch +let state = "training"; +let predictedRain = 0; +let maxBars = 12; +let graphValues = []; -// Load JSON data with same formatting from the internet, this means -// LoadData() cannot yet be used as it is formatted differently function preload() { - json_data = loadJSON("weather_data.json"); + data = loadJSON("weather_data.json"); - // Set the options to initialize Neural Network wit sequenceRegression Task let options = { task: "sequenceRegression", - debug: "true", - learningRate: 0.0075, // A smaller learning rate used for more stable training + debug: true, + learningRate: 0.0075, // smaller learning rate helps here inputs: features, - outputs: targets, + outputs: features, }; model = ml5.neuralNetwork(options); } function setup() { - data = json_data.data; - createCanvas(640, 400); - background(220); - - // Run a sliding window algorithm for time based data - let batchData = model.slidingWindow(data, features, targets, windowLength); + let canvas = createCanvas(640, 400); + canvas.parent("container"); + + // the JSON file has the actual data in a "data" property + data = data.data; + + // XXX: the following is quite unclear + // XXX: - try naming the function with an action word (verb) + // XXX: - explain why we do this + // XXX: - the second and the third parameter need to be the same - just pass one then? + // XXX: - the last parameter is 10, but the result has 14 entries 🤷‍♂️ (data has 24) + // XXX: - we use the result for inputs and outputs, how about naming those + // XXX: properties with those words directly (rather than "sequences", "targets") + + // run a sliding window algorithm for time based data + let batchData = model.slidingWindow(data, features, features, 10); let inputs = batchData.sequences; let outputs = batchData.targets; - // Feed data into the model + // add the training data for (let i = 0; i < inputs.length; i++) { model.addData(inputs[i], outputs[i]); } - // Normalize the data after adding everything model.normalizeData(); + // train right away let options = { epochs: 70, }; model.train(options, finishedTraining); - UI(); + let predictBtn = select("#predictBtn"); + predictBtn.center(); + predictBtn.mouseClicked(predictData); } function draw() { background(220); + noStroke(); textSize(20); + textAlign(CENTER); + if (state == "training") { - text("Training", 320, 200); + text("Training...", width / 2, 160); } else if (state == "prediction") { - text("Predicted Precipitation", 320, 200); - text(precipitation, 320, 250); - - // Helpful visual based on predicted value + // XXX: let's add the unit here (mm? inches?) + text("Predicted rain: " + nf(predictedRain, 0, 1) + "", 320, 160); push(); - textSize(precipitation * 5 + 10); - text("🌧️", 320, 150); + textSize(predictedRain * 5 + 10); + text("🌧️", width / 2, 100); pop(); } drawBarGraph(); } -// Predict data +function finishedTraining() { + state = "prediction"; +} + function predictData() { - seq = model.sampleWindow(data); //Helper function paired with the slidingWindow to get sample from data - model.predict(seq, gotResults); + // XXX: the following is similarly unclear + // XXX: - try naming the function with an action word (verb) + // XXX: - explain why we do this + // XXX: - (does this do more than return the last 10 samples from data?) + + // Helper function paired with the slidingWindow to get sample from data + let inputs = model.sampleWindow(data); + model.predict(inputs, gotResults); } -// Put the new data in the dataset so this will be considered for any new predictions function gotResults(results) { - precipitation = results[4].value; - addNewData(results); // Optional but will be helpful in using new prediction as part of dataset -} + predictedRain = results[4].value; -// Code for adding new data to the dataset to be used for future prediction -function addNewData(newResults) { - (new_values = { - date: " for the next hour", - temperature: newResults[0].value, // Get string convert to float and round to 2 decimal points - humidity: newResults[1].value, - wind_speed: newResults[2].value, - pressure: newResults[3].value, - precipitation: newResults[4].value, - }), - data.push(new_values); - - // Add data to the bar graph - graphValues.push(newResults[4].value); + // add result to the bar graph + graphValues.push(results[4].value); if (graphValues.length > maxBars) { - graphValues.shift(); // Remove first element + graphValues.shift(); } -} -function finishedTraining() { - state = "prediction"; + // optional: add predicted result to the dataset, so that it will be + // considered in further predictions (going forward in time) + addNewData(results); } -// Get buttons and assign functions (UI) -function UI() { - pred_but = select("#pred_but"); - pred_but.mouseClicked(predictData); - - textAlign(CENTER); - - maxBars = 12; - barWidth = width / maxBars; - maxDataValue = 35; +function addNewData(newValues) { + data.push({ + date: "One more (synthetic) hour", + temperature: newValues[0].value, + humidity: newValues[1].value, + wind_speed: newValues[2].value, + pressure: newValues[3].value, + precipitation: newValues[4].value, + }); } function drawBarGraph() { + let barWidth = width / maxBars; + let maxDataValue = 35; + for (let i = 0; i < graphValues.length && i < maxBars; i++) { let barHeight = map(graphValues[i], 0, maxDataValue, 0, height - 180); let x = i * barWidth; let y = height - barHeight - 20; - // Bar color gradient based on value + // XXX: the colors are a bit unintuitive (blue is normally associated + // with more rain rather than less) - maybe you could do some research + // and use lerpColor() here? + let barColor = map(graphValues[i], 0, maxDataValue, 0, 255); + push(); fill(barColor, 100, 255 - barColor); stroke(100); - - // Draw bar rect(x + 5, y, barWidth - 10, barHeight); + pop(); } } From 03d3a18fabeabf022393dac45de3bf2c8e33c27d Mon Sep 17 00:00:00 2001 From: gohai Date: Tue, 15 Jul 2025 16:07:08 +0800 Subject: [PATCH 18/47] Fix typo --- examples/neuralNetwork-sequence-weather-prediction/index.html | 3 ++- examples/neuralNetwork-sequence-weather-prediction/sketch.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html index ad67986c..bfc3e353 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/index.html +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -3,7 +3,8 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates Training a Weather Predictor through ml5.neuralNetwork with sequeceRegression Task. + This example demonstrates loading JSON data and training a Weather Predictor + through ml5.neuralNetwork with the sequenceRegression task. --> diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index f768626b..bd4dbf8c 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -4,7 +4,7 @@ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * * This example demonstrates loading JSON data and training a Weather Predictor - * through ml5.neuralNetwork with the sequeceRegression task. + * through ml5.neuralNetwork with the sequenceRegression task. * XXX: let us know what this dataset is from/about */ From 790a1627a8b12160c75a3098b6d05712e29cb431 Mon Sep 17 00:00:00 2001 From: gohai Date: Fri, 18 Jul 2025 07:23:47 +0800 Subject: [PATCH 19/47] Open-code sliding window in example --- .../sketch.js | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index bd4dbf8c..d8d6f890 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -10,6 +10,7 @@ let model; let data; +let sequenceLength = 10; let features = [ "temperature", "humidity", @@ -43,22 +44,16 @@ function setup() { // the JSON file has the actual data in a "data" property data = data.data; - // XXX: the following is quite unclear - // XXX: - try naming the function with an action word (verb) - // XXX: - explain why we do this - // XXX: - the second and the third parameter need to be the same - just pass one then? - // XXX: - the last parameter is 10, but the result has 14 entries 🤷‍♂️ (data has 24) - // XXX: - we use the result for inputs and outputs, how about naming those - // XXX: properties with those words directly (rather than "sequences", "targets") - - // run a sliding window algorithm for time based data - let batchData = model.slidingWindow(data, features, features, 10); - let inputs = batchData.sequences; - let outputs = batchData.targets; - - // add the training data - for (let i = 0; i < inputs.length; i++) { - model.addData(inputs[i], outputs[i]); + // step through the data (in a "sliding window" way) and break it into + // sequences of 10 data points, which is what the NN will see at a time + for (let i = 0; i <= data.length-sequenceLength; i++) { + let inputs = data.slice(i, i+sequenceLength); + let outputs = data.slice(i, i+sequenceLength); + for (let j=0; j < inputs.length; j++) { + delete inputs[j].date; + } + console.log("Sequence for training", inputs, outputs); + model.addData(inputs, outputs); } model.normalizeData(); @@ -97,13 +92,9 @@ function finishedTraining() { } function predictData() { - // XXX: the following is similarly unclear - // XXX: - try naming the function with an action word (verb) - // XXX: - explain why we do this - // XXX: - (does this do more than return the last 10 samples from data?) - - // Helper function paired with the slidingWindow to get sample from data - let inputs = model.sampleWindow(data); + // take the last 10 data points to predict the next + let inputs = data.slice(-sequenceLength); + console.log("Sequence for prediction", inputs); model.predict(inputs, gotResults); } From f702700cc00941a7422f000cea398b71920402ef Mon Sep 17 00:00:00 2001 From: gohai Date: Fri, 18 Jul 2025 07:42:18 +0800 Subject: [PATCH 20/47] Tweak variable placement and add comment --- examples/neuralNetwork-sequence-weather-prediction/sketch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index d8d6f890..19262df2 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -10,7 +10,6 @@ let model; let data; -let sequenceLength = 10; let features = [ "temperature", "humidity", @@ -18,6 +17,7 @@ let features = [ "pressure", "precipitation", ]; +let sequenceLength = 10; // NN looks at 10 data points at a time let state = "training"; let predictedRain = 0; From 4c9594e4770b95909af5a4c5497d4a2cd3979936 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 22 Jul 2025 03:03:20 -0400 Subject: [PATCH 21/47] fix: sliding window bug --- .../sketch.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 19262df2..454679a7 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -46,12 +46,13 @@ function setup() { // step through the data (in a "sliding window" way) and break it into // sequences of 10 data points, which is what the NN will see at a time - for (let i = 0; i <= data.length-sequenceLength; i++) { - let inputs = data.slice(i, i+sequenceLength); - let outputs = data.slice(i, i+sequenceLength); - for (let j=0; j < inputs.length; j++) { + for (let i = 0; i <= data.length - sequenceLength - 1; i++) { + let inputs = data.slice(i, i + sequenceLength); + let outputs = data[i + sequenceLength]; + for (let j = 0; j < inputs.length; j++) { delete inputs[j].date; } + delete outputs.date; console.log("Sequence for training", inputs, outputs); model.addData(inputs, outputs); } @@ -127,16 +128,18 @@ function drawBarGraph() { let barWidth = width / maxBars; let maxDataValue = 35; + let dryColor = color(235, 242, 255); + let wetColor = color(0, 80, 255); + for (let i = 0; i < graphValues.length && i < maxBars; i++) { let barHeight = map(graphValues[i], 0, maxDataValue, 0, height - 180); let x = i * barWidth; let y = height - barHeight - 20; - // XXX: the colors are a bit unintuitive (blue is normally associated - // with more rain rather than less) - maybe you could do some research - // and use lerpColor() here? + // interpolate color based on rainfall amount + let lerpAmt = constrain(graphValues[i] / maxDataValue, 0, 1); + let barColor = lerpColor(dryColor, wetColor, lerpAmt); - let barColor = map(graphValues[i], 0, maxDataValue, 0, 255); push(); fill(barColor, 100, 255 - barColor); stroke(100); From e3d95f229f875c005252ccdafbf7cb4019f179e3 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 22 Jul 2025 03:25:48 -0400 Subject: [PATCH 22/47] chore: slidingwindow to verb and rename samplewindow outputs --- src/NeuralNetwork/Sequential/index.js | 4 ++-- src/NeuralNetwork/Sequential/sequentialUtils.js | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index aefa7626..91f9ba38 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -181,7 +181,7 @@ class DIYSequential extends DiyNeuralNetwork { return seqUtils.padCoordinates(coordinates, targetPointCount, maxEpsilon); } - slidingWindow(data, featureKeys, targetKeys, batchLength = null) { + getSlidingWindow(data, featureKeys, targetKeys, batchLength = null) { this.featureKeys = featureKeys; if (batchLength == null) { @@ -200,7 +200,7 @@ class DIYSequential extends DiyNeuralNetwork { ); } - sampleWindow(data) { + getSampleWindow(data) { if (!this.batchLength || !this.featureKeys) { throw new Error( "Your data must be formated through the slidingWindow method first!" diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index 9dc03254..f40c531b 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -682,8 +682,8 @@ class SequentialUtils { ); } - const sequences = []; - const targets = []; + const inputs = []; + const outputs = []; // Start from the first possible complete sequence for ( @@ -715,11 +715,11 @@ class SequentialUtils { output[key] = target[key]; }); - sequences.push(seq); - targets.push(output); + inputs.push(seq); + outputs.push(output); } - return { sequences, targets }; + return { inputs, outputs }; } /** From 002213a81f1813f053d4cda2b6fd4ef63ff56b44 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 22 Jul 2025 14:20:46 -0400 Subject: [PATCH 23/47] chore: change padCoordinate name to setFixedLength --- .../sketch.js | 2 +- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 4 ++-- .../neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 8 ++++---- src/NeuralNetwork/Sequential/index.js | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js index 96f6aef1..237b4543 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -78,7 +78,7 @@ function draw() { // Pad the data and use for prediction } else if (hands.length <= 0 && sequence.length > 0) { - let predictData = model.padCoordinates(sequence, targetLength); + let predictData = model.setFixedLength(sequence, targetLength); model.classify(predictData, gotResults); // Reset the sequence diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index d1df5f8f..48c57aff 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -78,7 +78,7 @@ function draw() { } else if (hands.length <= 0 && sequence.length > 0) { if (state == "collecting") { // Pad the length of the coordinates to targetLength - let inputData = model.padCoordinates(sequence, targetLength); + let inputData = model.setFixedLength(sequence, targetLength); let outputData = { label: currGesture }; // Add data to the model @@ -90,7 +90,7 @@ function draw() { // Pad the data and use for prediction if state is prediction } else if (state == "prediction") { - let predictData = model.padCoordinates(sequence, targetLength); + let predictData = model.setFixedLength(sequence, targetLength); model.classify(predictData, gotResults); } diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index a3b08163..7dffed93 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -48,13 +48,13 @@ function mouseReleased() { // If state is collecting, add whole sequence as X, and shape as Y if (state == "collecting") { let target = { label: currShape }; - let paddedCoordinates = model.padCoordinates(sequence, targetLength); - model.addData(paddedCoordinates, target); + let fixedCoordinates = model.setFixedLength(sequence, targetLength); + model.addData(fixedCoordinates, target); clearScreen(); } else if (state == "prediction") { - let paddedCoordinates = model.padCoordinates(sequence, targetLength); - model.classify(paddedCoordinates, gotResults); + let fixedCoordinates = model.setFixedLength(sequence, targetLength); + model.classify(fixedCoordinates, gotResults); clearScreen(); } } diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 91f9ba38..4525a435 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -176,9 +176,9 @@ class DIYSequential extends DiyNeuralNetwork { } // RDP algorithm - padCoordinates(coordinates, targetPointCount) { + setFixedLength(coordinates, targetPointCount) { const maxEpsilon = int(coordinates.length / 2); - return seqUtils.padCoordinates(coordinates, targetPointCount, maxEpsilon); + return seqUtils.setFixedLength(coordinates, targetPointCount, maxEpsilon); } getSlidingWindow(data, featureKeys, targetKeys, batchLength = null) { From b0c968c6693350164923f5d1718e18a755f34752 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 22 Jul 2025 14:36:58 -0400 Subject: [PATCH 24/47] fix: tslayers naming bug --- src/NeuralNetwork/Sequential/index.js | 4 ++-- src/NeuralNetwork/Sequential/sequentialUtils.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 4525a435..ad3efc32 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -108,11 +108,11 @@ class DIYSequential extends DiyNeuralNetwork { return this.createNetworkLayers(layers); case "sequenceClassificationWithCNN": - layers = tsLayers.classificationConv; + layers = tsLayers.classificationWithCNN; return this.createNetworkLayers(layers); case "sequenceRegressionWithCNN": - layers = tsLayers.regressionConv; + layers = tsLayers.regressionWithCNN; return this.createNetworkLayers(layers); default: diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index f40c531b..743a2cc8 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -275,7 +275,7 @@ class SequentialUtils { } // point simplification utilities - Ramer-Douglas-Peucker (RDP) algorithm - padCoordinates(allPoints, targetPointCount, maxEpsilon = 50) { + setFixedLength(allPoints, targetPointCount, maxEpsilon = 50) { if (allPoints.length === 0) return []; // Check if it's an array of objects with .x and .y properties From b5adac38b369b17fa3a504accec356f096ec77fe Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Tue, 22 Jul 2025 14:44:35 -0400 Subject: [PATCH 25/47] chore: clarify error when task is unknown --- src/NeuralNetwork/Sequential/index.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index ad3efc32..5a046fcd 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -90,7 +90,7 @@ class DIYSequential extends DiyNeuralNetwork { addDefaultLayers() { let layers; - const tsLayers = createSeqLayers( + const seqLayers = createSeqLayers( this.neuralNetworkData.meta.seriesShape, this.options.hiddenUnits, this.numberOfClasses // For output units if needed @@ -100,24 +100,26 @@ class DIYSequential extends DiyNeuralNetwork { let taskConditions = task; switch (taskConditions) { case "sequenceRegression": - layers = tsLayers.regression; + layers = seqLayers.regression; return this.createNetworkLayers(layers); case "sequenceClassification": - layers = tsLayers.classification; + layers = seqLayers.classification; return this.createNetworkLayers(layers); case "sequenceClassificationWithCNN": - layers = tsLayers.classificationWithCNN; + layers = seqLayers.classificationWithCNN; return this.createNetworkLayers(layers); case "sequenceRegressionWithCNN": - layers = tsLayers.regressionWithCNN; + layers = seqLayers.regressionWithCNN; return this.createNetworkLayers(layers); default: - console.warn("Task is undefined or no inputUnits/outputUnits defined"); - layers = tsLayers.default; + console.error( + "Error: Task is unknown. Check documentation for a list of available sequence tasks. Defaulting to sequenceRegression Tasks" + ); + layers = seqLayers.default; return this.createNetworkLayers(layers); } } From c083bebd61afaf5754c5c5a6d036dcf0ecc168bd Mon Sep 17 00:00:00 2001 From: gohai Date: Wed, 23 Jul 2025 17:45:52 +0800 Subject: [PATCH 26/47] Rework the neuralNetwork-sequence-mouse-gesture-rdp example --- .../index.html | 11 +- .../sketch.js | 156 +++++++++--------- 2 files changed, 80 insertions(+), 87 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html index 7b7ba8cc..66e0f141 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html @@ -3,7 +3,8 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates How to train your own quickdraw classifier through ml5.neuralNetwork with sequeceClassification Task. + This example demonstrates how to train your own mouse gesture classifier + through ml5.neuralNetwork with the sequenceClassificationWithCNN task. --> @@ -11,7 +12,7 @@ - ml5.js Neural Network Train Mouse Gesture classifier Example + ml5.js neuralNetwork Mouse Gesture Classifier Example @@ -22,9 +23,9 @@
- - - + + +
diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index 7dffed93..95d3f2b3 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -3,132 +3,124 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates How to train your own mouse gesture classifier through ml5.neuralNetwork with sequeceClassification Task. + * This example demonstrates how to train your own mouse gesture classifier + * through ml5.neuralNetwork with the sequenceClassificationWithCNN task. */ let model; -let currShape = "circle"; -let state = "collecting"; +let state = "collecting"; +let curShape = "circle"; let sequence = []; let targetLength = 30; -function preload() { +function setup() { + let canvas = createCanvas(600, 400); + canvas.parent("canvasDiv"); + let options = { inputs: ["x", "y"], outputs: ["label"], task: "sequenceClassificationWithCNN", debug: true, - learningRate: 0.005, // Learning rate decreased for better convergence + learningRate: 0.005, // smaller learning rate converged better }; - model = ml5.neuralNetwork(options); + + select("#collCirclesBtn").mouseClicked(collectCircles); + select("#collSquaresBtn").mouseClicked(collectSquares); + select("#trainBtn").mouseClicked(trainModel); } -function setup() { - let canvas = createCanvas(600, 400); - canvas.parent("canvasDiv"); +function draw() { background(220); - UI(); + + for (let i = 0; i < sequence.length - 1; i++) { + line(sequence[i].x, sequence[i].y, sequence[i + 1].x, sequence[i + 1].y); + } + + // This sketch uses the RDP line simplification algorithm + // to make each inputs to the neural network have the same + // number of points. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA + + let rdp = model.setFixedLength(sequence, targetLength); + for (let i = 0; i < rdp.length; i++) { + fill(255); + rect(rdp[i].x - 3, rdp[i].y - 3, 6, 6); + } + + // display current state + textSize(20); + fill(0); + if (state == "collecting") { + text("Now collecting " + curShape + "s", 50, 50); + } else if (state == "training") { + text("Training...", 50, 50); + } else if (state == "predicting" && curShape == null) { + text("Training finished. Draw again to predict.", 50, 50); + } else if (state == "predicting") { + text("Saw a " + curShape, 50, 50); + } } -function draw() { - // Record data when the mouse is pressed inside the canvas - if (mouseIsPressed) { - // Draw lines through coordinates - line(pmouseX, pmouseY, mouseX, mouseY); - let inputs = { x: mouseX, y: mouseY }; - sequence.push(inputs); +function mousePressed() { + if (mouseX >= 0 && mouseX < width && mouseY >= 0 && mouseY < height) { + sequence.push({ x: mouseX, y: mouseY }); + } +} + +function mouseDragged() { + if (mouseX >= 0 && mouseX < width && mouseY >= 0 && mouseY < height) { + sequence.push({ x: mouseX, y: mouseY }); } } -// Code to signify drawing can be done again function mouseReleased() { - if (mouseY < height && mouseX < width) { - // If state is collecting, add whole sequence as X, and shape as Y - if (state == "collecting") { - let target = { label: currShape }; - let fixedCoordinates = model.setFixedLength(sequence, targetLength); - model.addData(fixedCoordinates, target); - - clearScreen(); - } else if (state == "prediction") { - let fixedCoordinates = model.setFixedLength(sequence, targetLength); - model.classify(fixedCoordinates, gotResults); - clearScreen(); - } + if (sequence.length == 0) return; + + if (state == "collecting") { + let inputs = model.setFixedLength(sequence, targetLength); + let outputs = { label: curShape }; + model.addData(inputs, outputs); + } else if (state == "predicting") { + let inputs = model.setFixedLength(sequence, targetLength); + model.classify(inputs, gotResults); } - // Reset the sequence + // reset the sequence sequence = []; } function trainModel() { - // Normalize Data first before Training model.normalizeData(); - // Set the number of epochs for training let options = { epochs: 40, }; model.train(options, finishedTraining); - background(220); state = "training"; - text("Training...", 50, 50); - - recCircle.attribute("disabled", true); - recSquare.attribute("disabled", true); - trainBut.attribute("disabled", true); + select("#collCirclesBtn").attribute("disabled", true); + select("#collSquaresBtn").attribute("disabled", true); + select("#trainBtn").attribute("disabled", true); } function finishedTraining() { - background(220); - text("Training Finished, Draw again to predict", 50, 50); - state = "prediction"; + state = "predicting"; + curShape = null; } function gotResults(results) { - let label = results[0].label; - currShape = label; + curShape = results[0].label; } -// UI Elements -let recCircle, recSquare, trainBut; - -function UI() { - textSize(20); - - recCircle = select("#recCircle"); - recSquare = select("#recSquare"); - trainBut = select("#trainBut"); - - recCircle.mouseClicked(recordCircle); - recSquare.mouseClicked(recordSquare); - trainBut.mouseClicked(trainModel); - - text(state + " : " + currShape, 50, 50); - - function recordCircle() { - state = "collecting"; - currShape = "circle"; - - background(220); - text(state + " : " + currShape, 50, 50); - } - - function recordSquare() { - state = "collecting"; - currShape = "square"; - - background(220); - text(state + " : " + currShape, 50, 50); - } +function collectCircles() { + state = "collecting"; + curShape = "circle"; } -// Cleanup screen and removed drawn elements, add helpful text -function clearScreen() { - background(220); - textSize(20); - fill(0); - text(state + " : " + currShape, 50, 50); +function collectSquares() { + state = "collecting"; + curShape = "square"; } From ec08ad9f283a4b76043c547f5f93b64b31c5cf2e Mon Sep 17 00:00:00 2001 From: gohai Date: Wed, 23 Jul 2025 17:46:37 +0800 Subject: [PATCH 27/47] Construct NN inside setup() in weather-prediction example --- .../sketch.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 454679a7..6da31032 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -26,6 +26,11 @@ let graphValues = []; function preload() { data = loadJSON("weather_data.json"); +} + +function setup() { + let canvas = createCanvas(640, 400); + canvas.parent("container"); let options = { task: "sequenceRegression", @@ -35,11 +40,6 @@ function preload() { outputs: features, }; model = ml5.neuralNetwork(options); -} - -function setup() { - let canvas = createCanvas(640, 400); - canvas.parent("container"); // the JSON file has the actual data in a "data" property data = data.data; From e8f6958a6b821391055bb83014ba15421628d715 Mon Sep 17 00:00:00 2001 From: gohai Date: Wed, 23 Jul 2025 18:58:30 +0800 Subject: [PATCH 28/47] Shave off two unneeded lines --- examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index 95d3f2b3..c12498aa 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -116,11 +116,9 @@ function gotResults(results) { } function collectCircles() { - state = "collecting"; curShape = "circle"; } function collectSquares() { - state = "collecting"; curShape = "square"; } From f580e642f0fd27b697e541463b9c6e0ad76a9323 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 24 Jul 2025 16:09:40 -0400 Subject: [PATCH 29/47] fix: error capitalization from seqUtils --- src/NeuralNetwork/Sequential/sequentialUtils.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index 743a2cc8..8b30fbe7 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -31,7 +31,7 @@ class SequentialUtils { checkInputStructure(xInputs, options = null) { if (!Array.isArray(xInputs)) { - throw new error("Syntax Error: Data Should be in an Array"); + throw new Error("Syntax Error: Data Should be in an Array"); } let isObjects = true; let isArrays = true; @@ -47,7 +47,7 @@ class SequentialUtils { Object.keys(xInputs[i]).length || nnUtils.getDataType(xInputs[i - 1]) === "object" ) { - throw new error("Data format is inconsistent"); + throw new Error("Data format is inconsistent"); } } } else if (Array.isArray(xInputs[i])) { @@ -58,7 +58,7 @@ class SequentialUtils { xInputs[i - 1].length !== xInputs[i].length || !Array.isArray(xInputs[i - 1]) ) { - throw new error("Data format is inconsistent"); + throw new Error("Data format is inconsistent"); } } } else { @@ -66,7 +66,7 @@ class SequentialUtils { isObjects = false; isArrays = false; } else { - throw new error("inputLabels is needed for 1D array inputs"); + throw new Error("inputLabels is needed for 1D array inputs"); } } @@ -77,7 +77,7 @@ class SequentialUtils { } else if (isValues) { return "ValueSequence"; } else { - throw new error("Syntax Error: Input Structure is unknown"); + throw new Error("Syntax Error: Input Structure is unknown"); } } } @@ -91,7 +91,7 @@ class SequentialUtils { case "ValueSequence": return this.convertValueSequence(xInputs, options); default: - throw new error("Input Data Structure is unknown"); + throw new Error("Input Data Structure is unknown"); } } @@ -127,7 +127,7 @@ class SequentialUtils { convertValueSequence(xInputs, options = null) { const { inputLabels } = options; if (xInputs.length % inputLabels.length !== 0) { - throw new error( + throw new Error( "Invalid Input: Number of Labels don't match amount of values" ); } From 0998b5b63d382f4c8a4c0af7dc65ae503ca3b987 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 24 Jul 2025 16:12:05 -0400 Subject: [PATCH 30/47] chore: remove rdp visualizer --- .../index.html | 24 --- .../sketch.js | 148 ------------------ 2 files changed, 172 deletions(-) delete mode 100644 examples/neuralNetwork-sequence-rdp-visualizer/index.html delete mode 100644 examples/neuralNetwork-sequence-rdp-visualizer/sketch.js diff --git a/examples/neuralNetwork-sequence-rdp-visualizer/index.html b/examples/neuralNetwork-sequence-rdp-visualizer/index.html deleted file mode 100644 index c8077a2a..00000000 --- a/examples/neuralNetwork-sequence-rdp-visualizer/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - ml5.js Ramer–Douglas–Peucker algorithm Visualizer - - - - - -
- -
- - diff --git a/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js b/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js deleted file mode 100644 index 44739f71..00000000 --- a/examples/neuralNetwork-sequence-rdp-visualizer/sketch.js +++ /dev/null @@ -1,148 +0,0 @@ -/* - * 👋 Hello! This is an ml5.js example made and shared with ❤️. - * Learn more about the ml5.js project: https://ml5js.org/ - * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - * - * This example demonstrates the Ramer–Douglas–Peucker algorithm to adjust points from a variable length to a target length. - */ -let allPoints = []; -const targetPointCount = 10; // Desired number of points -let maxEpsilon = 30; // Adjust as needed - -function setup() { - createCanvas(600, 400); - background(0); - clearButton = createButton("clear screen"); - clearButton.mousePressed(clearScreen); -} - -function draw() { - background(0); - - if (mouseIsPressed && mouseY < height && mouseX < width) { - allPoints.push(createVector(mouseX, mouseY)); - } - - if (allPoints.length > 10) { - rpdeia(); - } -} - -function rpdeia() { - const rdpPoints = []; - - const epsilon = findEpsilonForPointCount(allPoints, targetPointCount); - - const total = allPoints.length; - const start = allPoints[0]; - const end = allPoints[total - 1]; - rdpPoints.push(start); - rdp(0, total - 1, allPoints, rdpPoints, epsilon); - rdpPoints.push(end); - - stroke(255, 0, 255); - strokeWeight(4); - noFill(); - beginShape(); - for (let v of allPoints) { - vertex(v.x, v.y); - } - endShape(); - - stroke(255); - strokeWeight(2); - beginShape(); - for (let v of rdpPoints) { - vertex(v.x, v.y); - } - endShape(); - - fill(255); - noStroke(); - textSize(24); - text("mouse-created points: " + allPoints.length, 20, 25); - text("rdp-created points: " + rdpPoints.length, 20, 50); -} - -function findEpsilonForPointCount(points, targetCount) { - let low = 0; - let high = maxEpsilon; - let mid; - let simplifiedPointsCount = 0; - - while (high - low > 0.001) { - // Tolerance for approximation - mid = (low + high) / 2; - simplifiedPointsCount = getSimplifiedPointCount(points, mid); - if (simplifiedPointsCount > targetCount) { - low = mid; - } else { - high = mid; - } - } - - return mid; -} - -function getSimplifiedPointCount(points, epsilon) { - const rdpPoints = []; - const total = points.length; - const start = points[0]; - const end = points[total - 1]; - rdpPoints.push(start); - rdp(0, total - 1, points, rdpPoints, epsilon); - rdpPoints.push(end); - return rdpPoints.length; -} - -function rdp(startIndex, endIndex, allPoints, rdpPoints, epsilon) { - const nextIndex = findFurthest(allPoints, startIndex, endIndex, epsilon); - if (nextIndex > 0) { - if (startIndex != nextIndex) { - rdp(startIndex, nextIndex, allPoints, rdpPoints, epsilon); - } - rdpPoints.push(allPoints[nextIndex]); - if (endIndex != nextIndex) { - rdp(nextIndex, endIndex, allPoints, rdpPoints, epsilon); - } - } -} - -function findFurthest(points, a, b, epsilon) { - let recordDistance = -1; - const start = points[a]; - const end = points[b]; - let furthestIndex = -1; - for (let i = a + 1; i < b; i++) { - const currentPoint = points[i]; - const d = lineDist(currentPoint, start, end); - if (d > recordDistance) { - recordDistance = d; - furthestIndex = i; - } - } - if (recordDistance > epsilon) { - return furthestIndex; - } else { - return -1; - } -} - -function lineDist(c, a, b) { - const norm = scalarProjection(c, a, b); - return p5.Vector.dist(c, norm); -} - -function scalarProjection(p, a, b) { - const ap = p5.Vector.sub(p, a); - const ab = p5.Vector.sub(b, a); - ab.normalize(); // Normalize the line - ab.mult(ap.dot(ab)); - const normalPoint = p5.Vector.add(a, ab); - return normalPoint; -} - -function clearScreen() { - allPoints = []; - background(0); -} From 0febf59448048fdafdbd6c76612873ec5567f26d Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:01:25 +0800 Subject: [PATCH 31/47] Rework the neuralNetwork-sequence-hand-gesture example --- .../index.html | 5 +- .../sketch.js | 251 ++++++++---------- 2 files changed, 121 insertions(+), 135 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture/index.html b/examples/neuralNetwork-sequence-hand-gesture/index.html index afceb7f0..c6bb1918 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/index.html +++ b/examples/neuralNetwork-sequence-hand-gesture/index.html @@ -3,7 +3,8 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates training a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task. + This example demonstrates training a hand gesture classifier through + ml5.neuralNetwork with the sequenceClassificationWithCNN task. --> @@ -11,7 +12,7 @@ - ml5.js Neural Network Hand Gesture Train and Save + ml5.js neuralNetwork Hand Gesture Train and Save diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 48c57aff..8ebde8f7 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -3,211 +3,196 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates training a Hand Gesture classifier through ml5.neuralNetwork with sequeceClassification Task. + * This example demonstrates training a hand gesture classifier through + * ml5.neuralNetwork with the sequenceClassificationWithCNN task. */ let video; let handPose; let hands = []; +let model; +let state = "training"; let sequence = []; let targetLength = 30; -let gestures = ["gesture #1", "gesture #2"]; -let counts = { "gesture #1": 0, "gesture #2": 0 }; +let gestures = ["Gesture #1", "Gesture #2"]; +let counts = { "Gesture #1": 0, "Gesture #2": 0 }; +let curGesture = gestures[0]; -let state = "collecting"; -let currGesture = gestures[0]; // Set currGesture to gesture 1 by default -let predGesture = ""; +let gesture1Button; +let gesture2Button; +let trainButton; function preload() { - // Load the handPose model - // Set options to have data points flipped + // load the handPose model handPose = ml5.handPose({ flipHorizontal: true }); - - // Setup the neural network using sequenceClassification - let options = { - outputs: ["label"], - task: "sequenceClassificationWithCNN", - debug: "true", - learningRate: 0.001, // The default learning rate of 0.01 didn't converge for this usecase, thus a learning rate of 0.001 is used (make smaller steps of parameters each update) - }; - model = ml5.neuralNetwork(options); } function setup() { let canvas = createCanvas(640, 480); canvas.parent("canvasDiv"); - // Setup video capture video = createCapture(VIDEO, { flipped: true }); video.size(640, 480); video.hide(); + handPose.detectStart(video, gotHands); - // Setup the UI buttons for training - UI(); + let options = { + outputs: ["label"], + task: "sequenceClassificationWithCNN", + debug: true, + learningRate: 0.001, // the default learning rate of 0.01 didn't converge for this use case (0.001 makes smaller steps each epoch) + }; + model = ml5.neuralNetwork(options); - // Use handpose model on video - handPose.detectStart(video, gotHands); + // setup the UI buttons for training + gesture1Button = createButton("Start recording " + gestures[0]); + gesture1Button.mousePressed(recordGesture1); + gesture2Button = createButton("Start recording " + gestures[1]); + gesture2Button.mousePressed(recordGesture2); + trainButton = createButton("Train and Save Model"); + trainButton.mousePressed(trainModel); } function draw() { - // Draw video on frame image(video, 0, 0, width, height); + drawHands(); - // If hand is detected in the frame, start recording gesture if (hands.length > 0) { - handpoints = drawPoints(); + // hands in frame, add to sequence + let handpoints = getKeypoints(["Left", "Right"]); sequence.push(handpoints); - // Helpful text to signify recording - textSize(20); - stroke(255); - fill(0); - if (state == "collecting") { - text( - state + " : " + currGesture + ", put hand down once done with gesture", - 50, - 50 - ); - } else if (state == "prediction") { - text("predicting... put hand down once done with gesture", 50, 50); - } - - // Add collected data to model once the hand is gone and state is collecting - } else if (hands.length <= 0 && sequence.length > 0) { - if (state == "collecting") { - // Pad the length of the coordinates to targetLength - let inputData = model.setFixedLength(sequence, targetLength); - let outputData = { label: currGesture }; - - // Add data to the model - model.addData(inputData, outputData); + // This uses the RDP line simplification algorithm to make sure each + // input to the neural network has the same number of points. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA - // Update the counts for the UI - counts[currGesture]++; - updateDataCountUI(); - - // Pad the data and use for prediction if state is prediction - } else if (state == "prediction") { - let predictData = model.setFixedLength(sequence, targetLength); - model.classify(predictData, gotResults); + let rdp = model.setFixedLength(sequence, targetLength); + for (let i = 0; i < rdp.length - 1; i++) { + for (let j = 0; j < rdp[i].length; j += 2) { + stroke(255, 0, 0); + line(rdp[i][j], rdp[i][j + 1], rdp[i + 1][j], rdp[i + 1][j + 1]); + } } - - // Reset the sequence + } else if (sequence.length > 0) { + // hands moved out of the frame, end of sequence + let inputs = model.setFixedLength(sequence, targetLength); + + if (state == "training") { + let outputs = { label: curGesture }; + model.addData(inputs, outputs); + } else if (state == "predicting") { + model.classify(inputs, gotResults); + } + counts[curGesture]++; + // reset the sequence sequence = []; + } - // Tell users to put hand up to start recording - } else { - textSize(20); - stroke(255); - fill(0); - if (state == "collecting") { + // display current state + textSize(16); + fill(255); + noStroke(); + if (state == "training" && sequence.length == 0) { + text("Move your hand(s) into the frame to record " + curGesture, 50, 50); + } else if (state == "training") { + text("Move your hand(s) out of the frame to finish " + curGesture, 50, 50); + } else if (state == "predicting" && curGesture == null) { + text("Try a trained gesture to see the prediction", 50, 50); + } else if (state == "predicting" && curGesture) { + text("Saw " + curGesture, 50, 50); + } + + // show how many times each gesture was recorded + if (state == "training") { + for (let i = gestures.length - 1; i >= 0; i--) { text( - "put hand up in screen to start collecting for: " + currGesture, + gestures[i] + ": " + counts[gestures[i]], 50, - 50 + height - 50 - (gestures.length - i - 1) * 20 ); - } else if (state == "prediction") { - if (!predGesture) { - text("do one of the trained gestures to predict", 50, 50); - } else { - text( - "prediction: " + predGesture + ", try again with another gesture!", - 50, - 50 - ); - } } } } -// Train the data when 'Train abd Save Model' button is pressed -function train() { - // The data should be normalized before training +function trainModel() { model.normalizeData(); - - currGesture = ""; - - // Train the model - let trainingOptions = { + let options = { epochs: 50, }; - model.train(trainingOptions, finishedTraining); + model.train(options, finishedTraining); + + gesture1Button.attribute("disabled", true); + gesture2Button.attribute("disabled", true); + trainButton.attribute("disabled", true); } -// When the model is trained, save the model function finishedTraining() { - state = "prediction"; + state = "predicting"; model.save(); + curGesture = null; } -// Callback for predict +// callback function for when the classification fininished function gotResults(results) { - predGesture = results[0].label; + curGesture = results[0].label; } -// Callback function for when handPose outputs data +// callback function for when handPose outputs data function gotHands(results) { hands = results; } -// Draw visuals for hand points and flatten values into an array -function drawPoints() { - let handpoints = []; - // Iterate through both hands +function drawHands() { for (let i = 0; i < hands.length; i++) { let hand = hands[i]; for (let j = 0; j < hand.keypoints.length; j++) { - // Access the keypoints in the hand let keypoint = hand.keypoints[j]; - handpoints.push(keypoint.x, keypoint.y); - fill(0, 255, 0); noStroke(); circle(keypoint.x, keypoint.y, 5); } } - - // Assign to a different variable before clearing - let output = handpoints; - handpoints = []; - - return output; -} - -// UI Elements -function UI() { - dataCountsP = createP( - "Gesture 1 data: " + - counts[gestures[0]] + - "
Gesture 2 data: " + - counts[gestures[0]] - ); - rockButton = createButton("Record Gesture #1"); - rockButton.mousePressed(addGesture1); - paperButton = createButton("Record Gesture #2"); - paperButton.mousePressed(addGesture2); - trainButton = createButton("Train and Save Model"); - trainButton.mousePressed(train); } -// Set the current handPose data to the model as "Gesture #1" -function addGesture1() { - currGesture = gestures[0]; +// Return the tracked hand points as flattened array of 84 numbers +// for use as input to the neural network + +function getKeypoints(whichHands = ["Left", "Right"]) { + let keypoints = []; + // look for the left and right hand + for (let handedness of whichHands) { + let found = false; + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + if (hand.handedness == handedness) { + // and add the x and y numbers of each tracked keypoint + // to the array + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + keypoints.push(keypoint.x, keypoint.y); + } + found = true; + break; + } + } + if (!found) { + // if we don't find a right or a left hand, add 42 zeros + // to the keypoints array instead + for (let j = 0; j < 42; j++) { + keypoints.push(0); + } + } + } + return keypoints; } -// Set the current handPose data to the model as "Gesture #2" -function addGesture2() { - currGesture = gestures[1]; +function recordGesture1() { + curGesture = gestures[0]; } -// Update the HTML UI with the current data counts -function updateDataCountUI() { - dataCountsP.html( - "Gesture 1 data: " + - counts[gestures[0]] + - "
Gesture 2 data: " + - counts[gestures[1]] - ); +function recordGesture2() { + curGesture = gestures[1]; } From 17a1cc28d3a76ab20341dc7e1c6501ead9cf3eab Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:13:33 +0800 Subject: [PATCH 32/47] Fix grammar in comment --- examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index c12498aa..c651fce4 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -39,9 +39,8 @@ function draw() { line(sequence[i].x, sequence[i].y, sequence[i + 1].x, sequence[i + 1].y); } - // This sketch uses the RDP line simplification algorithm - // to make each inputs to the neural network have the same - // number of points. + // This uses the RDP line simplification algorithm to make sure each + // input to the neural network has the same number of points. // For more information about RDP, see: // https://www.youtube.com/watch?v=ZCXkvwLxBrA From 21d3054c64f6bd3983ed1017818aa4cc18b1e96f Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:13:59 +0800 Subject: [PATCH 33/47] Use canvasDiv everywhere --- examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html | 3 +-- examples/neuralNetwork-sequence-weather-prediction/index.html | 3 +-- examples/neuralNetwork-sequence-weather-prediction/sketch.js | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html index 66e0f141..1d8f933c 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html @@ -18,10 +18,9 @@ +
-
-
diff --git a/examples/neuralNetwork-sequence-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html index bfc3e353..402d054e 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/index.html +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -19,8 +19,7 @@ - -
+
diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 6da31032..5077ae31 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -30,7 +30,7 @@ function preload() { function setup() { let canvas = createCanvas(640, 400); - canvas.parent("container"); + canvas.parent("canvasDiv"); let options = { task: "sequenceRegression", From 155fde6eb25eb9a8813a2a99261b142baf647cfd Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:17:24 +0800 Subject: [PATCH 34/47] Use training/predicting in examples --- .../neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 9 +++------ .../neuralNetwork-sequence-weather-prediction/sketch.js | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index c651fce4..29fcb88b 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -9,7 +9,7 @@ let model; -let state = "collecting"; +let state = "training"; let curShape = "circle"; let sequence = []; let targetLength = 30; @@ -53,10 +53,8 @@ function draw() { // display current state textSize(20); fill(0); - if (state == "collecting") { + if (state == "training") { text("Now collecting " + curShape + "s", 50, 50); - } else if (state == "training") { - text("Training...", 50, 50); } else if (state == "predicting" && curShape == null) { text("Training finished. Draw again to predict.", 50, 50); } else if (state == "predicting") { @@ -79,7 +77,7 @@ function mouseDragged() { function mouseReleased() { if (sequence.length == 0) return; - if (state == "collecting") { + if (state == "training") { let inputs = model.setFixedLength(sequence, targetLength); let outputs = { label: curShape }; model.addData(inputs, outputs); @@ -99,7 +97,6 @@ function trainModel() { }; model.train(options, finishedTraining); - state = "training"; select("#collCirclesBtn").attribute("disabled", true); select("#collSquaresBtn").attribute("disabled", true); select("#trainBtn").attribute("disabled", true); diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 5077ae31..dabaeb03 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -77,7 +77,7 @@ function draw() { if (state == "training") { text("Training...", width / 2, 160); - } else if (state == "prediction") { + } else if (state == "predicting") { // XXX: let's add the unit here (mm? inches?) text("Predicted rain: " + nf(predictedRain, 0, 1) + "", 320, 160); push(); @@ -89,7 +89,7 @@ function draw() { } function finishedTraining() { - state = "prediction"; + state = "predicting"; } function predictData() { From 9e53b9795580b7f04a178d7b0a08e6363f7bee16 Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:21:40 +0800 Subject: [PATCH 35/47] Whitespace change --- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 8ebde8f7..49f4d9b1 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -15,7 +15,6 @@ let model; let state = "training"; let sequence = []; let targetLength = 30; - let gestures = ["Gesture #1", "Gesture #2"]; let counts = { "Gesture #1": 0, "Gesture #2": 0 }; let curGesture = gestures[0]; From d4a26a48861bdd4d0d625b8e6679613a2dc49ce8 Mon Sep 17 00:00:00 2001 From: gohai Date: Sat, 26 Jul 2025 15:26:53 +0800 Subject: [PATCH 36/47] Flip for order --- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 49f4d9b1..4345cf2a 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -106,11 +106,11 @@ function draw() { // show how many times each gesture was recorded if (state == "training") { - for (let i = gestures.length - 1; i >= 0; i--) { + for (let i = 0; i < gestures.length; i++) { text( gestures[i] + ": " + counts[gestures[i]], 50, - height - 50 - (gestures.length - i - 1) * 20 + height - 50 - (gestures.length - i) * 20 ); } } From 0e71849d3e7df47975a51ccca97dfe1cebb2aae5 Mon Sep 17 00:00:00 2001 From: gohai Date: Sun, 27 Jul 2025 09:01:35 +0800 Subject: [PATCH 37/47] Rework neuralNetwork-sequence-hand-gesture-load-model example --- .../index.html | 16 +- .../sketch.js | 165 ++++++++++-------- 2 files changed, 99 insertions(+), 82 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html index 33bcf110..ec78e95b 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html @@ -3,7 +3,9 @@ Learn more about the ml5.js project: https://ml5js.org/ ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md - This example demonstrates loading a Sign Language classifier through ml5.neuralNetwork with sequeceClassification Task. + This example demonstrates loading a hand gesture classifier through + ml5.neuralNetwork with the sequenceClassificationWithCNN task. + This example has been trained with the ASL gestures for Hello and Goodbye. --> @@ -11,7 +13,7 @@ - ml5.js Time Series Hand Gesture load model + ml5.js neuralNetwork Hand Gesture Loading Pre-trained Model Example @@ -20,15 +22,7 @@

- How to do gestures for Hello and Goodbye in ASL:
- Hello: - https://babysignlanguage.com/dictionary/hello/
- Goodbye: - https://babysignlanguage.com/dictionary/goodbye/
+ How to sign: Hello & Goodbye in ASL.

diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js index 237b4543..022e0c2a 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -3,107 +3,114 @@ * Learn more about the ml5.js project: https://ml5js.org/ * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md * - * This example demonstrates loading a Hand Gesture classifier through ml5.neuralNetwork with sequeceClassification Task. - * This example is trained with the ASL gestures for Hello and Goodbye + * This example demonstrates loading a hand gesture classifier through + * ml5.neuralNetwork with the sequenceClassificationWithCNN task. + * This example has been trained with the ASL gestures for Hello and Goodbye. * * Reference to sign hello and goodbye in ASL: - * Hello: https://babysignlanguage.com/dictionary/hello/ - * Goodbye: https://babysignlanguage.com/dictionary/goodbye/ + * Hello: https://www.signasl.org/sign/hello + * Goodbye: https://www.signasl.org/sign/goodbye */ -let handPose; let video; +let handPose; let hands = []; +let model; +let isModelLoaded = false; let sequence = []; -let targetLength = 50; - -let predGesture = ""; +let sequenceLength = 50; +let curGesture; function preload() { - // Load the handPose model - // Set options to have data points flipped + // load the handPose model handPose = ml5.handPose({ flipHorizontal: true }); - - // Setup the neural network using sequenceClassification - let options = { - task: "sequenceClassificationWithCNN", - }; - - model = ml5.neuralNetwork(options); } function setup() { let canvas = createCanvas(640, 480); canvas.parent("canvasDiv"); - // Create video capture video = createCapture(VIDEO, { flipped: true }); video.size(640, 480); video.hide(); - handPose.detectStart(video, gotHands); - // Setup the model files to load + let options = { + task: "sequenceClassificationWithCNN", + }; + model = ml5.neuralNetwork(options); + + // setup the model files to load let modelDetails = { model: "model/model.json", metadata: "model/model_meta.json", weights: "model/model.weights.bin", }; - // Load the model and call modelLoaded once finished + // load the model and call modelLoaded once finished model.load(modelDetails, modelLoaded); } -// Callback for load model function modelLoaded() { - console.log("model loaded!"); + console.log("Model loaded"); + isModelLoaded = true; } function draw() { - // Draw video on the canvas image(video, 0, 0, width, height); + drawHands(); + textSize(16); + stroke(0); + fill(255); - // If hands are found then start recording if (hands.length > 0) { - // Get coordinates from hands (21 points) - handpoints = drawPoints(); + // hands in frame, add their keypoints the sequence (input) XXX + let handpoints = getKeypoints(["Left", "Right"]); sequence.push(handpoints); - - // Helpful text to signify recording - textSize(20); - stroke(255); - fill(0); - text("predicting... put hand down once done with gesture", 50, 50); - - // Pad the data and use for prediction - } else if (hands.length <= 0 && sequence.length > 0) { - let predictData = model.setFixedLength(sequence, targetLength); - model.classify(predictData, gotResults); - - // Reset the sequence + text( + "Move your hand(s) out of the frame after finishing the gesture", + 50, + 50 + ); + } else if (sequence.length > 0) { + // hands moved out of the frame, end of sequence + + // sequence will have varying length at this point, depending on + // how long the hands were in frame - a line simplification algorithm + // (RDP) turns it into the fixed length the NN can work with + let inputs = model.setFixedLength(sequence, sequenceLength); + // start the classification + if (isModelLoaded) { + model.classify(inputs, gotResults); + } + // reset the sequence sequence = []; - - // Tell users to put hand up to start recording + text("Classifying...", 50, 50); + } else if (curGesture == null) { + // on program start + text("Move your hand(s) into the frame to sign a gesture", 50, 50); } else { - textSize(20); - stroke(255); - fill(0); - if (!predGesture) { - text("do one of the gestures below to predict", 50, 50); - } else { - text( - "prediction: " + predGesture + ", try again with another gesture!", - 50, - 50 - ); - } + // after receiving a classification + text('Saw "' + curGesture + '"', 50, 50); } } -// Draw the points on the hands -function drawPoints() { - let handpoints = []; +// callback function for when the classification fininished +function gotResults(results, error) { + if (error) { + console.error(error); + return; + } + curGesture = results[0].label; +} + +// callback function for when handPose outputs data +function gotHands(results) { + hands = results; +} + +function drawHands() { for (let i = 0; i < hands.length; i++) { let hand = hands[i]; for (let j = 0; j < hand.keypoints.length; j++) { @@ -111,22 +118,38 @@ function drawPoints() { fill(0, 255, 0); noStroke(); circle(keypoint.x, keypoint.y, 5); - handpoints.push(keypoint.x, keypoint.y); } } - let output = handpoints; - handpoints = []; - - return output; } -// Callback function for when handPose outputs data -function gotHands(results) { - // Save the output to the hands variable - hands = results; -} - -// Call back for accessing the results -function gotResults(results) { - predGesture = results[0].label; +// Return the tracked hand points as flattened array of 84 numbers +// for use as input to the neural network + +function getKeypoints(whichHands = ["Left", "Right"]) { + let keypoints = []; + // look for the left and right hand + for (let whichHand of whichHands) { + let found = false; + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + if (hand.handedness == whichHand) { + // and add the x and y numbers of each tracked keypoint + // to the array + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + keypoints.push(keypoint.x, keypoint.y); + } + found = true; + break; + } + } + if (!found) { + // if we don't find a right or a left hand, add 42 zeros + // to the keypoints array instead + for (let j = 0; j < 42; j++) { + keypoints.push(0); + } + } + } + return keypoints; } From 452ed2d2351e3c482cf6135ad797580319498fcf Mon Sep 17 00:00:00 2001 From: gohai Date: Sun, 27 Jul 2025 09:06:04 +0800 Subject: [PATCH 38/47] Tweaks to language --- .../sketch.js | 9 ++++++--- examples/neuralNetwork-sequence-hand-gesture/index.html | 2 +- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 2 +- .../neuralNetwork-sequence-mouse-gesture-rdp/index.html | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js index 022e0c2a..9239292d 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -65,7 +65,7 @@ function draw() { fill(255); if (hands.length > 0) { - // hands in frame, add their keypoints the sequence (input) XXX + // hands in frame, add their keypoints to the sequence (input) let handpoints = getKeypoints(["Left", "Right"]); sequence.push(handpoints); text( @@ -76,10 +76,13 @@ function draw() { } else if (sequence.length > 0) { // hands moved out of the frame, end of sequence - // sequence will have varying length at this point, depending on + // Sequence will have varying length at this point, depending on // how long the hands were in frame - a line simplification algorithm - // (RDP) turns it into the fixed length the NN can work with + // (RDP) turns it into the fixed length the NN can work with. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA let inputs = model.setFixedLength(sequence, sequenceLength); + // start the classification if (isModelLoaded) { model.classify(inputs, gotResults); diff --git a/examples/neuralNetwork-sequence-hand-gesture/index.html b/examples/neuralNetwork-sequence-hand-gesture/index.html index c6bb1918..8602322c 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/index.html +++ b/examples/neuralNetwork-sequence-hand-gesture/index.html @@ -12,7 +12,7 @@ - ml5.js neuralNetwork Hand Gesture Train and Save + ml5.js neuralNetwork Hand Gesture Training and Saving Example diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 4345cf2a..55c30fbf 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -59,7 +59,7 @@ function draw() { drawHands(); if (hands.length > 0) { - // hands in frame, add to sequence + // hands in frame, add their keypoints to the sequence let handpoints = getKeypoints(["Left", "Right"]); sequence.push(handpoints); diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html index 1d8f933c..aa7b2a8c 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html @@ -12,7 +12,7 @@ - ml5.js neuralNetwork Mouse Gesture Classifier Example + ml5.js neuralNetwork Gesture Classifier Example with sequenceClassificationWithCNN From 46924ba65b15449a06f687aa8b13c6b07b0b1b8a Mon Sep 17 00:00:00 2001 From: gohai Date: Sun, 27 Jul 2025 09:08:16 +0800 Subject: [PATCH 39/47] Rename targetLength to sequenceLength and other tweaks --- .../sketch.js | 21 +++++++++++-------- .../sketch.js | 18 +++++++++------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 55c30fbf..3093a379 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -14,7 +14,7 @@ let model; let state = "training"; let sequence = []; -let targetLength = 30; +let sequenceLength = 30; let gestures = ["Gesture #1", "Gesture #2"]; let counts = { "Gesture #1": 0, "Gesture #2": 0 }; let curGesture = gestures[0]; @@ -68,32 +68,35 @@ function draw() { // For more information about RDP, see: // https://www.youtube.com/watch?v=ZCXkvwLxBrA - let rdp = model.setFixedLength(sequence, targetLength); + let rdp = model.setFixedLength(sequence, sequenceLength); for (let i = 0; i < rdp.length - 1; i++) { for (let j = 0; j < rdp[i].length; j += 2) { stroke(255, 0, 0); - line(rdp[i][j], rdp[i][j + 1], rdp[i + 1][j], rdp[i + 1][j + 1]); + // hide lines to 0,0 + if (rdp[i][j] != 0 && rdp[i + 1][j] != 0) { + line(rdp[i][j], rdp[i][j + 1], rdp[i + 1][j], rdp[i + 1][j + 1]); + } } } } else if (sequence.length > 0) { // hands moved out of the frame, end of sequence - let inputs = model.setFixedLength(sequence, targetLength); + let inputs = model.setFixedLength(sequence, sequenceLength); if (state == "training") { let outputs = { label: curGesture }; model.addData(inputs, outputs); + counts[curGesture]++; } else if (state == "predicting") { model.classify(inputs, gotResults); } - counts[curGesture]++; // reset the sequence sequence = []; } // display current state textSize(16); + stroke(0); fill(255); - noStroke(); if (state == "training" && sequence.length == 0) { text("Move your hand(s) into the frame to record " + curGesture, 50, 50); } else if (state == "training") { @@ -110,7 +113,7 @@ function draw() { text( gestures[i] + ": " + counts[gestures[i]], 50, - height - 50 - (gestures.length - i) * 20 + height - 50 - (gestures.length - i - 1) * 20 ); } } @@ -162,11 +165,11 @@ function drawHands() { function getKeypoints(whichHands = ["Left", "Right"]) { let keypoints = []; // look for the left and right hand - for (let handedness of whichHands) { + for (let whichHand of whichHands) { let found = false; for (let i = 0; i < hands.length; i++) { let hand = hands[i]; - if (hand.handedness == handedness) { + if (hand.handedness == whichHand) { // and add the x and y numbers of each tracked keypoint // to the array for (let j = 0; j < hand.keypoints.length; j++) { diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index 29fcb88b..87a982be 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -12,7 +12,7 @@ let model; let state = "training"; let curShape = "circle"; let sequence = []; -let targetLength = 30; +let sequenceLength = 30; function setup() { let canvas = createCanvas(600, 400); @@ -39,20 +39,22 @@ function draw() { line(sequence[i].x, sequence[i].y, sequence[i + 1].x, sequence[i + 1].y); } - // This uses the RDP line simplification algorithm to make sure each - // input to the neural network has the same number of points. + // Sequence will have varying length at this point, depending on + // how long the hands were in frame - a line simplification algorithm + // (RDP) turns it into the fixed length the NN can work with. // For more information about RDP, see: // https://www.youtube.com/watch?v=ZCXkvwLxBrA - let rdp = model.setFixedLength(sequence, targetLength); + let rdp = model.setFixedLength(sequence, sequenceLength); for (let i = 0; i < rdp.length; i++) { fill(255); rect(rdp[i].x - 3, rdp[i].y - 3, 6, 6); } // display current state - textSize(20); - fill(0); + textSize(16); + stroke(0); + fill(255); if (state == "training") { text("Now collecting " + curShape + "s", 50, 50); } else if (state == "predicting" && curShape == null) { @@ -78,11 +80,11 @@ function mouseReleased() { if (sequence.length == 0) return; if (state == "training") { - let inputs = model.setFixedLength(sequence, targetLength); + let inputs = model.setFixedLength(sequence, sequenceLength); let outputs = { label: curShape }; model.addData(inputs, outputs); } else if (state == "predicting") { - let inputs = model.setFixedLength(sequence, targetLength); + let inputs = model.setFixedLength(sequence, sequenceLength); model.classify(inputs, gotResults); } // reset the sequence From 1308865946ea3f4828e74e347a03dd7d29e8326a Mon Sep 17 00:00:00 2001 From: gohai Date: Sun, 27 Jul 2025 09:29:52 +0800 Subject: [PATCH 40/47] Revert part of last commit --- examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index 87a982be..d954f1e0 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -53,8 +53,7 @@ function draw() { // display current state textSize(16); - stroke(0); - fill(255); + fill(0); if (state == "training") { text("Now collecting " + curShape + "s", 50, 50); } else if (state == "predicting" && curShape == null) { From 024d91f6d471830be353911800859cabb7a38d48 Mon Sep 17 00:00:00 2001 From: gohai Date: Sun, 27 Jul 2025 16:30:28 +0800 Subject: [PATCH 41/47] Rename "predicting" to "classifiying" in classification tasks --- examples/neuralNetwork-sequence-hand-gesture/sketch.js | 10 +++++----- .../neuralNetwork-sequence-mouse-gesture-rdp/sketch.js | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js index 3093a379..c9a162be 100644 --- a/examples/neuralNetwork-sequence-hand-gesture/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -86,7 +86,7 @@ function draw() { let outputs = { label: curGesture }; model.addData(inputs, outputs); counts[curGesture]++; - } else if (state == "predicting") { + } else if (state == "classifying") { model.classify(inputs, gotResults); } // reset the sequence @@ -101,9 +101,9 @@ function draw() { text("Move your hand(s) into the frame to record " + curGesture, 50, 50); } else if (state == "training") { text("Move your hand(s) out of the frame to finish " + curGesture, 50, 50); - } else if (state == "predicting" && curGesture == null) { - text("Try a trained gesture to see the prediction", 50, 50); - } else if (state == "predicting" && curGesture) { + } else if (state == "classifying" && curGesture == null) { + text("Try a trained gesture", 50, 50); + } else if (state == "classifying" && curGesture) { text("Saw " + curGesture, 50, 50); } @@ -132,7 +132,7 @@ function trainModel() { } function finishedTraining() { - state = "predicting"; + state = "classifying"; model.save(); curGesture = null; } diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js index d954f1e0..069a620c 100644 --- a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -56,9 +56,9 @@ function draw() { fill(0); if (state == "training") { text("Now collecting " + curShape + "s", 50, 50); - } else if (state == "predicting" && curShape == null) { - text("Training finished. Draw again to predict.", 50, 50); - } else if (state == "predicting") { + } else if (state == "classifying" && curShape == null) { + text("Training finished. Draw again to see the classification in action.", 50, 50); + } else if (state == "classifying") { text("Saw a " + curShape, 50, 50); } } @@ -82,7 +82,7 @@ function mouseReleased() { let inputs = model.setFixedLength(sequence, sequenceLength); let outputs = { label: curShape }; model.addData(inputs, outputs); - } else if (state == "predicting") { + } else if (state == "classifying") { let inputs = model.setFixedLength(sequence, sequenceLength); model.classify(inputs, gotResults); } @@ -104,7 +104,7 @@ function trainModel() { } function finishedTraining() { - state = "predicting"; + state = "classifying"; curShape = null; } From 43b9198122fd3ef52f52108966d622bad039c57f Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Mon, 28 Jul 2025 13:50:42 -0400 Subject: [PATCH 42/47] feat: filter input/output by labels --- .../sketch.js | 5 +- src/NeuralNetwork/Sequential/index.js | 15 +++ .../Sequential/sequentialUtils.js | 94 ++++++++++++++++++- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index dabaeb03..6015133a 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -49,10 +49,7 @@ function setup() { for (let i = 0; i <= data.length - sequenceLength - 1; i++) { let inputs = data.slice(i, i + sequenceLength); let outputs = data[i + sequenceLength]; - for (let j = 0; j < inputs.length; j++) { - delete inputs[j].date; - } - delete outputs.date; + console.log("Sequence for training", inputs, outputs); model.addData(inputs, outputs); } diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 5a046fcd..6e276c1a 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -210,6 +210,21 @@ class DIYSequential extends DiyNeuralNetwork { } return seqUtils.getLatestSequence(data, this.batchLength, this.featureKeys); } + + /** + * getData + * Returns the raw data that was added through addData + * @returns {Object} - Object with xs, ys, and raw properties + */ + getData() { + const rawData = this.neuralNetworkData.data.raw; + + return { + xs: rawData.map((item) => item.xs), + ys: rawData.map((item) => item.ys), + raw: rawData, + }; + } } export default DIYSequential; //export for taskSelection diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index 8b30fbe7..04a3e1cd 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -21,14 +21,65 @@ class SequentialUtils { */ verifyAndFormatInputs(xInputs, options = null, classOptions) { const dataFormat = this.checkInputStructure(xInputs, options); + + // Apply input filtering if we have object sequences and input labels are provided + let filteredInputs = xInputs; + if (dataFormat === "ObjectSequence") { + filteredInputs = this.filterInputsByLabels( + xInputs, + options, + classOptions + ); + } + return this.formatInputsToObjects( - xInputs, + filteredInputs, options, classOptions, dataFormat ); } + /** + * filterInputsByLabels + * Filters input objects to keep only the keys specified in input labels + * @param {Array} xInputs - Array of input objects + * @param {Object} options - Options object that may contain inputLabels + * @param {Object} classOptions - Class options that may contain inputs + * @returns {Array} - Filtered array of objects + */ + filterInputsByLabels(xInputs, options = null, classOptions) { + // Get input labels from options or classOptions + let inputLabels = null; + + if (options && options.inputLabels && Array.isArray(options.inputLabels)) { + inputLabels = options.inputLabels; + } else if ( + classOptions && + classOptions.inputs && + Array.isArray(classOptions.inputs) + ) { + inputLabels = classOptions.inputs; + } + + // If no input labels provided or not an array, return original data + if (!inputLabels || !Array.isArray(inputLabels)) { + console.log("did nothing"); + return xInputs; + } + + // Filter each object to keep only the specified keys + return xInputs.map((obj) => { + const filteredObj = {}; + inputLabels.forEach((key) => { + if (obj.hasOwnProperty(key)) { + filteredObj[key] = obj[key]; + } + }); + return filteredObj; + }); + } + checkInputStructure(xInputs, options = null) { if (!Array.isArray(xInputs)) { throw new Error("Syntax Error: Data Should be in an Array"); @@ -170,7 +221,46 @@ class SequentialUtils { throw new Error("outputLabels must be an array"); } - return nnUtils.formatDataAsObject(yInputs, outputLabels); + // Apply output filtering if we have object outputs and output labels are provided + let filteredOutputs = yInputs; + if (typeof yInputs === "object" && !Array.isArray(yInputs)) { + filteredOutputs = this.filterOutputsByLabels(yInputs, options, classOptions); + } + + return nnUtils.formatDataAsObject(filteredOutputs, outputLabels); + } + + /** + * filterOutputsByLabels + * Filters output objects to keep only the keys specified in output labels + * @param {Object} yInputs - Output object + * @param {Object} options - Options object that may contain outputLabels + * @param {Object} classOptions - Class options that may contain outputs + * @returns {Object} - Filtered output object + */ + filterOutputsByLabels(yInputs, options = null, classOptions) { + // Get output labels from options or classOptions + let outputLabels = null; + + if (options && options.outputLabels && Array.isArray(options.outputLabels)) { + outputLabels = options.outputLabels; + } else if (classOptions && classOptions.outputs && Array.isArray(classOptions.outputs)) { + outputLabels = classOptions.outputs; + } + + // If no output labels provided or not an array, return original data + if (!outputLabels || !Array.isArray(outputLabels)) { + return yInputs; + } + + // Filter the object to keep only the specified keys + const filteredObj = {}; + outputLabels.forEach(key => { + if (yInputs.hasOwnProperty(key)) { + filteredObj[key] = yInputs[key]; + } + }); + return filteredObj; } prepareLabels(xInputs, yInputs, options = null, classOptions) { From 0d50690cd3aa76ae7d076126118f6650d89c562c Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Wed, 30 Jul 2025 14:10:22 -0400 Subject: [PATCH 43/47] chore: add new pre-trained models --- .../model/model.json | 176 +--------- .../model/model.weights.bin | Bin 17128 -> 16040 bytes .../model/model_meta.json | 302 ++++++------------ .../sketch.js | 2 +- 4 files changed, 92 insertions(+), 388 deletions(-) diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json index 98820b95..136bcf90 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json @@ -1,175 +1 @@ -{ - "modelTopology": { - "class_name": "Sequential", - "config": { - "name": "sequential_1", - "layers": [ - { - "class_name": "Conv1D", - "config": { - "filters": 8, - "kernel_initializer": { - "class_name": "VarianceScaling", - "config": { - "scale": 1, - "mode": "fan_avg", - "distribution": "normal", - "seed": null - } - }, - "kernel_regularizer": null, - "kernel_constraint": null, - "kernel_size": [3], - "strides": [1], - "padding": "valid", - "dilation_rate": [1], - "activation": "relu", - "use_bias": true, - "bias_initializer": { "class_name": "Zeros", "config": {} }, - "bias_regularizer": null, - "activity_regularizer": null, - "bias_constraint": null, - "name": "conv1d_Conv1D1", - "trainable": true, - "batch_input_shape": [null, 50, 42], - "dtype": "float32" - } - }, - { - "class_name": "MaxPooling1D", - "config": { - "pool_size": [2], - "padding": "valid", - "strides": [2], - "name": "max_pooling1d_MaxPooling1D1", - "trainable": true - } - }, - { - "class_name": "Conv1D", - "config": { - "filters": 16, - "kernel_initializer": { - "class_name": "VarianceScaling", - "config": { - "scale": 1, - "mode": "fan_avg", - "distribution": "normal", - "seed": null - } - }, - "kernel_regularizer": null, - "kernel_constraint": null, - "kernel_size": [3], - "strides": [1], - "padding": "valid", - "dilation_rate": [1], - "activation": "relu", - "use_bias": true, - "bias_initializer": { "class_name": "Zeros", "config": {} }, - "bias_regularizer": null, - "activity_regularizer": null, - "bias_constraint": null, - "name": "conv1d_Conv1D2", - "trainable": true, - "batch_input_shape": [null, 50, 42], - "dtype": "float32" - } - }, - { - "class_name": "MaxPooling1D", - "config": { - "pool_size": [2], - "padding": "valid", - "strides": [2], - "name": "max_pooling1d_MaxPooling1D2", - "trainable": true - } - }, - { - "class_name": "Flatten", - "config": { "name": "flatten_Flatten1", "trainable": true } - }, - { - "class_name": "Dense", - "config": { - "units": 16, - "activation": "relu", - "use_bias": true, - "kernel_initializer": { - "class_name": "VarianceScaling", - "config": { - "scale": 1, - "mode": "fan_avg", - "distribution": "normal", - "seed": null - } - }, - "bias_initializer": { "class_name": "Zeros", "config": {} }, - "kernel_regularizer": null, - "bias_regularizer": null, - "activity_regularizer": null, - "kernel_constraint": null, - "bias_constraint": null, - "name": "dense_Dense1", - "trainable": true - } - }, - { - "class_name": "Dense", - "config": { - "units": 2, - "activation": "softmax", - "use_bias": true, - "kernel_initializer": { - "class_name": "VarianceScaling", - "config": { - "scale": 1, - "mode": "fan_avg", - "distribution": "normal", - "seed": null - } - }, - "bias_initializer": { "class_name": "Zeros", "config": {} }, - "kernel_regularizer": null, - "bias_regularizer": null, - "activity_regularizer": null, - "kernel_constraint": null, - "bias_constraint": null, - "name": "dense_Dense2", - "trainable": true - } - } - ] - }, - "keras_version": "tfjs-layers 4.8.0", - "backend": "tensor_flow.js" - }, - "weightsManifest": [ - { - "paths": ["./hello.weights.bin"], - "weights": [ - { - "name": "conv1d_Conv1D1/kernel", - "shape": [3, 42, 8], - "dtype": "float32" - }, - { "name": "conv1d_Conv1D1/bias", "shape": [8], "dtype": "float32" }, - { - "name": "conv1d_Conv1D2/kernel", - "shape": [3, 8, 16], - "dtype": "float32" - }, - { "name": "conv1d_Conv1D2/bias", "shape": [16], "dtype": "float32" }, - { - "name": "dense_Dense1/kernel", - "shape": [176, 16], - "dtype": "float32" - }, - { "name": "dense_Dense1/bias", "shape": [16], "dtype": "float32" }, - { "name": "dense_Dense2/kernel", "shape": [16, 2], "dtype": "float32" }, - { "name": "dense_Dense2/bias", "shape": [2], "dtype": "float32" } - ] - } - ] -} +{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,30,84],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.22.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,84,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[96,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin index e57f181601f525897b348ac9ca8a03203e2c4420..1b38b79eb92efece857ab11b57d137d7240c4a93 100644 GIT binary patch literal 16040 zcmWlgc{EmE7st&r86#v?QXwKdXP-1^&|oM{BBFUfnkSh`=Bb3DltPj;@|=A{i8LW8 zjZ%tIn)D0F>wVU`YdwG6b?>?R?6ddh``t4{LtxtAC0v`L$e&B;;fG9uo9+>KzH=_5 z&M@GM*E}cho?Is}=hbcuMmP5BP}NRTl6}}5zcs9csm7DZ ze-ldB5s1df1Knh%xe0AgGvdb|kB7a|pV>frIjNf3kAjbX%K5w=xHeOXT86*J-}@fp zrBgC=!i+RHF)J6n|4hWxy>C!l%9 zsY^LaUe^Yq-Dzp4T5Zg#_dUYDA<>u?JQA}GM&a-cxhQcV5L)iF;mI!xustaaP7bKk zBkCh?LR|)#ePJcIjtpUs`~Kp?zBK5YWtf*e24v2J^@f}st_^fX(wqG^^XA@)WaX3w^Qbkzz zR%vSWdJ)d|4~E_?OVQ=vX(&7S2p{?6ljT0{G`;FBp8Qq>YN5wr#5F0pB}N%D<{W}r z5|0Jv7c+$`m)L_IG5X!;7aq-@DA4?@#3PFwp<&x`thw9+3v9mOs_`jUxDY|*#$Do9 z`54XSigJen9c(-jjvw>?;$8JqxM3iM4W<^O$g2*pc@a-Y-ZiLrUnw+@d5(&`XAwH| zS<-*2@oGsVSv0tpjBlJuTp#?w&XhXndpiM_zM6q?4`t~o2QywIvIy3jhmuuQ7huA@ z6mWT-OjOupXu2rJ5|z)u+u(dM>~uR019NWrvBj}q_(eEBw*nN+H$ih{AD+$<$NXWf zP<)oc@ScSr@gNL*nwJvU%2nXeiLl$lf?v6&$a_T;$>S|SU>c)L7B%SMhUmwzLb3>_ z>}Vnx9?~>&_El1J$%ti%M++ML3^-1az>qEOFe@(-G8e6c8Lt!Im~ebL5Dv+{4k)3&1LXE3v4WKISath7PLlYBUItI#^L{HVE=#}@xhYJgs!{0p zw-jEz7sI{czu22ka^R}%&I}a_AnVa0kS(?)4JD~~ysa7CuXW%wg?0F1fkJuRBSSuE zvpp$Sm{JL=?S;0P z?E=*~C1|`uhToYRh7*=Yq5@`;)fx-g&!z(GE&qqh>W5Rwq_LC+C7_ID4~E1Qv+0Q~ zDASP#LotIN;w|}Jz2#)m)^fqrhwHHBs1}h5O~&pPBaF1W1XD;O{A(BvAH-wWjnjU@ zR9hv?5qVB-YEQ%s!Kzf9QJ&M10->T`So@v8wPnsj7lU{lG!>)HljUKLITtp` zR1kyU!w_&%h?g&(!qLZWqQb3SSbgOXJ9S%@jLoSOWOaAI!J8fsIwGz7lE!NI?UV-V z=KErqRxM6dIRf(QUX%Z{XJVmV8`MlIgJ+}8I8N&sDjT)_wt4oWeA}{aEAahn73uTvA zcMV+CmEum#+XY&`&w=v7B+^|!40ZP=m1oKu!jhl2NZ_-%$*yG~SnSgVx_&Bf+Cok6ciua8&JyzCfsrq<$~ zCu`B~R}=0()I^+rgbE*w2qeA(wcv3w9jYuO>02*pdaFd6Z*nTbGq;QJXRZ?cL`7iK z1Bw&(y~VF{`XPH@3BGDiWY>2-0Bq_ZE`N&zDM11Z;9dB~+>n|Md=%s=FNDSr4VbUF zo_WuE!_HN{Cjq51N%6w{xYlybiD zJFc8R8U8a)6OP<-9W~Rl;24V3b+lMy|2R|4KMuZ?2R)9 z0`OCJF*Gl(M%91rVCMaSS&3HQ`%5Kg+VT^pu2mzSMy(Wtc6egPQhx~h`3;J;RzY^8 z0UFqk;$tdDw|T$K6?Po!wm%;F&S>begcdE~_7D~MyGdoi>soxshdiFnRaij8be zh1j%uP@Eq|;&qp>s0eRdZM6k%;d4m+)J9%w%W_{^Mfm$&j(6F2;LKGi_^?KZTaF)P zTf|$);OHA*cc~61y|kf@uM*jvZE|oyBb!+-84f0%;rOTXI$A_&(UIgE{2QUf!ne-F zU8Y9#{snosbL}P`*RWwH?nuM8ZX2q0)gCJLyTJTMX#)KNnL-eYBVVPuAgfUo&W@2m zR;5A}U*|w@dp%=0RcvF+IO_Ack2t&jA{TZX1eg6;M4>7L4tfR>`YejI3IuTQSpuu= z76|(P_JX}jJDYMT59TERng3-pDrP-}i=Ti>++2>D>&MZqH-*B(8!~aGVj`#p`!kO~ z3UzW~kmxdWmiZ`h!QxO@buR#oo{I2$y+Ta7+a)6E}&D2NFPixuF#BQ?RY%J7| zzlF_}lgQ=48W1y%!}Lq}_{yQb+~>zpc>VAiR$D8=_=O+gaNJp7`tO~hD7IcTPBn2NR)*XQ|Tc!E&Gu@c)s|zKP$l^whfSA0$Q1ss#7%i0N zKP&ElM)x;i(IPt(Mm!Uqm8c_&<-W1J+z9YFp~!t=q-fFk<4ia|!*S#r1^RpU54K>Y zIn@anPhMw@l(~Sm>Ab6{8nvlA{0WVBC*ZLB>$e@vI>e zEia2Rnft?u$iY4`&Uq8Ye=dg#J16ctBN3%O%@!1ntA56>Ij{Nj6(sW8(q? z$}|^{f&V1<&niWzUz?2vBM!rh7l$$AZy`+DQv>G?Ut>xIDzG4ZIvoqu@XA^U_cl$& z83JQIxhn}uv)&+BR>0jVS6F*P3UtafX?@ZffuUy{Ii7ceJflUZvBwL3AG=F_bgUpw zhr8ka#sK_2UzN(e${;C07oqCgVmSS2GP!cvf|;!l$JxKCK=a2b_K)l%CmyWCfk%5t zSV1k>6&nP{%v06al)3OT4H$6gj%Qcv)wW?1X)Kk>FeJ8bK;PjkTc(&)4Ps=%?S8h%&z62s3MVEgZ2d^4|$6}N}LUqxlX)~GnJlRAkb z?wnvFE!@bIXQk}^0q3~f)cy1ILtg@lUo?23A z6D|6y;k^*T^90KzY;aHRHdu821P*Mv$u@*v!hSH1t&s!vNhe@Z@qEltVW=a_1V$(OCgjG3xZyG*$Scr4I)!V({bHr_lFq zHU3_m1z%l9a*;?GXn&moj{{smtiuJ=tfN43oB=rA>VdKM-?CmFO}-dhg)&J=zUr(E zxpPjEhYg+M>i@DaV9^v3q@s*l&s_(@&*AvST9#irDh7d}ub{@W29MAqU^2xR&wZSY zfsfvj&Cjoaa%dLVO|VC2yp(Hb{~-Lg$-@_;`5* z{u)|;KaY5Ji@qQ>tcRFRdc%%=)Zl^M?quYHTF|%Crj5R9!8y%|=L^s}bd; zn)g*P?oa? zIgik>Ul#ZKPl1Dt8G`V;c{qA=2QJZ4rf*lLv3t9Zqr`+oV5~P8>@`NSh8LoEVP7(y zHk&~={JX#w1Q_Fqdyd5P&@w(p77S@%RIz+ISFeSrTg7 z^P5N;9>L8^kFy_}7QjZy$)xCB8$0EvO?9G;hz@0_mJ^3Z7M~>^zh4rA6T1BC&y{#= z#6{3Emd3|{11yETAW?$`;5Yda%Fp}?5p!H2tm7o+jg{oRem7v-S$T+U(WL65?Xl-e zE_ytXq3?~W;q8VVxNql5ZkFrQF(Dhk>boqzowL-@HvBCvRIdm5%j?Lq^Mk^)i=nVn zbr6pa3xKVzet5Wl1Y~`>!WM6+0waS!)LGC4jpe@thllci<-V`Nlu*LIt~dS1{U8{MTerrc>i&yw> z{uxKvVad!NcH{i_pUO;LtV56c{lqF(2OpZAh0bGHX#GkZZv?J(%xKGp-L_%u(Pj%g zy=E%>c5X%0nV*SVdJ1_fUgFp|?G-z)))>uP=ivSO)tD}I9i~oF;n%E231VE%;BAq5 zfxW&Cci(>$evvzvdC~w+YSqA@);fH)x{H-xSPn0ZL%?WHHVKU%$7Wr43s#L{eCyW# z;MQY#9y~jrRBZVPvHGDnCdd?T%zHyF)Wx!QFQ1SjQ(uw6zHg*|<_}c(nhdf3uA}ar zc*y!GN)^K5AbhMj>V+2Kf&Ze}kpLz9v*-_TGrA=VJx8c)`!t+~Pl(*MC?a8^&r-Lw zk*R4uw`5vfq3-y?jU5esP%q40F^Y+6*V20H2YG5@6l zIeD~=++v-Al`ZAM7wcL;-$Rd8T+W0$diALGSc>Ty$kO2zE1-Hq8WSmgiJML@qh&`4 zcFg@twlvofRmBn*?%avD`?aX!#uO}ZJPu1oQ@qkKf`;fT!J1$xZu??7k6C{ZmTDen z_di@CKVy}+NaHxv-)W2uewFy+=x^p!a}KU2%tptN(U@`Y6La!jj4fyF1=E&_k&y}N z^vU@WvUN=kF>}4oqRg|PKUJ4n4HKa+MLTi-#hI+!Ruh`t#kgEq9g)5<9Wx(liFNl2-uewi_iUY;g3};@M!l%H=m~7(@5enu!Zg&bCn)m_|=5L2Z-zPKM5&fY0C>&=N zX!5a(o$(5PMNh&N(>0-OhYGb^6;FbG z>(e^>5VY=b!&Q@eVDW`)xPP<}a(8FL!~djd;?71qAUhmoe?EmM`OVlw-9YWHBCK6= z2ra(M!P`$yfJnwg2KnyTIAR_o#NQsOrzW#w6TETY`c}}pTtX5Grc$R#;?(!dOF_Zx zXmaV~}Lq1kbYQR0EZJ-wZSRh)hOaAk|hpUxu!3ez_sMe8%M~{BPy|uHz zzTz4dzR7k}IC+iy^RCB_9~ns7JjIUlmjpjfT)@(HYsa>Bp*w@ICO85vh$-{;ZjTucIF4!eBFT;kH=)JtH+t${WTT5Dd5Q1> z+t(crIZ-$8k<}?d-J(^nxu=XQwfV)&?v8}8)v|2)yovD1Hw~kgp9bqMQZ!k$olIC* zjOu&3;Izg8blaB)yV@EYy;%%Qy|R5^D!harzu=@5hV!M$+ z)0n?dd|ZwBPcFvSn`L?Dlv#L>u=zJ&HIh2-e> zTw(I6+eGtM0Bm2ejHTPW0G4uq9TLwJ*f~BEOprK$8@xQBBXl8~qM3_=){$^MRv%_Q z&4XcL&Ft-eD6|`+37)n=_)9MZkKI+EQ(l`>`dXgn#5=IK3!-%9@e+_|6X(VsLf}vK zecadj9CuC)!--p#!3UkESiiCWZ<7JAC=Drp*)K~M{Tt$*dkJ_I{Q+kkRY>>V42wr; zVckGI%dIbhQbF=i-zLW|E1GfRS)G_NG7AE~&d2;gPkf_R0~N+QptEluKD#-Lm9-UP zPOkzg?k6~MXzg-}G*Kxh1)o_d(QcQ!m^A{~v-uSmV_Y+o=exnNJOu2I<)g$`IqGz0 zF}6*Ak3H#8Ak{h*%`8mddF!Udmddm>}--9-PXb z3+@juVp-WY1U+lg!Nk-N6~r6i+?Qi`xk;3^LNn_MU&N*-jfbzJAHkm%87iM1%=BjM z#BXCxBW->Ff8xiXNc4NwGk97U;XaJ&AGnLndG|$nGMEH65{lqD?-J_TzHoHO7>>I`vdMXc6x_K(j@@1U zoV}m%nt8>mI_O1p^-j~8eBwf*YydwAZ73J%?;)(0*QM_o`Xt?sv5u{UI zklwTH0{;`@{6j}S`F`j$8>o0MSZq~*m6O$Rt(*@1Z>Ky4|2c`Xw{~G(=WY=F+Cj`q z4WPx}0!Z9K;q{n07S!U4x3-7Ef?47;>S+wji0U1Bb~AWnt&Qr`5ep4};1AV@xLzR$ z+wR3+;JS-oy3r2I6LK*`+6{x&T*F~c*TC_vYmgOt3XaJPr=c;UXlPIkcI``sum4nV zeEXjA+2^BhjM--5u40Q@PoBrxHHV1Gb}ymFM-_B!+Q)vy%F-z-kFdb9@^a;kK2R4s z0n0k~LhglYW(Nem5}+ctSuhST(BaOx4H}Z2ej!Kt@~Jd=L?9gUjb!htI67P>G

Ykn`}Mq?6!^zM%90Gc{Qf#VFE!*x=4im3iDIZ0tqvG&8791#cN}+HJHXu;QrLDk2DWW`j`Pzr zu**}IUR)rLQv%&l^`bs1Zodr?(`S-Dg$DTU8f6Eg>WGcY7Hs(xg)y>5n7>DauTxrv zK`o8sxy)5;yDZHuHtX?4wNsg4^?4RNO^e=}oQy}rwdg#(eQ3T+mgXema|O9 z={J7KS0Q3|(y(G#lOUzFo_LUA7T=0zWa) zWObgt`69S@+rkMMIX=7g8YXMyGKc6^vdLDPk6NC9DwAgutC^8(iGx5eI<^-?zMG

%Ve7<)+6;-vSn&%3!zW4=psOeJm{2;zntsRwR{=(heE<`G7H+WtY!%aWph^bvL z)S113`lkgX&^n%${`^f^?@Q3nsoAva<4AgJ%|2Q-!=LNe+i+{C$L06-9N_mKsp5@w z9n5RT7kKu|3ZE;xQusH4(qAjUvOJdVeKn4Mbs0-t4eq0#;dB;oa~j|6vz=n$32NRE zz<)Tq(`l+&RHUl_-dnuDgPR3qh}f!)E3Uum z&73;Z;e}WwnYsng9G;Hgd*hKI3&NZng6-i>?JoG4R4Kx(r1& z9ir6<9 z^H~0R+>frV+o!Qh}~4zxMnQe9IG|79QVG-u(oX zwK+@a1tTi7{zxPo5{SEo6f5hsrepQmNzwalP`D}?{m+Gy^6}Gnx_vV3e|MbD_F6!u zh`nZgW5)2ugCclp-Cj?cs`F__K{D5I+QT=ROu)l= zGW5_8`{Ym7V5eXVRe$RbOPauc%zTJfzKbh9>j+L=+Roz= z)?)0qDtanGht_Sgp{FJik1e51OP?-=Dkap+aN&A}9xh$C}uM2M^$soeTZDwgLZ2?Wa$}CK9MV z&OH@w5~bPAAl0P?CVqj`&((yUnfwzb%iqN7DxvJ6X&C=`Hi{-qTFR}Qv-y_Tm+bUI z1^z%*l-`eA3*xd9xqqN9KfmyspeW=O>v$G}Mn)Q3+Rz;y^rsN*eN(AK-$WYv_6C}Y z6=Q<&S}HR;1@OiXn7?EyKW3VY@z*G={pm(m8NUVBm4{)^EG>F%qOYJ@DW0Dz_2&BI z2w!ZaPS32&M8TC1K3p!J9vwZKUQ{K*nfs04-o{^$cDa|-rS0eTXQ$H$Te(;xAU_@d41mU zOqBSuiBGhRr{7ZV4)Kk1c<;j@7NKoQ14Z2&?^}A~%J)ZbLQ1b7d7FSA(UyWkVe#DU zNPoG_kpCy(Sth2P-;N|>HcvF-ShxAPu=2_ba#bmgpETChKp!VgvgI1dsitxb(9|%>A4JlT@;x_3c>db7vb}oR*IX zEoVR>Qi>jw&c(qvcRGJ?y6{8vJZ}B<*icS;F1$AALcf*<(!@I{=>B0Q^_aH{M(eo2 zsEKl*z0{1I_DB`vs?948YBMCQ|B{GWj5Of4VS*`YyM{bRuB`p)Rl!a*c{B67&xZ_!NW#U)r^zlaYA&!OJpf5Of^ zLe^fhooxL<`2K}sc=ya>uuJqS#xEEK)zZUwIdkIz@df<6)em-si}J5i4A}d1wtUI% zlVJSC6u!Kb8Ol+AAd=XjrJou0If)8ZZo7yBK0`XDpAPp^l*WteOUc5R6Eyqo6}I50 z5jTD)N!634bJfb*Rk>cCiSAcx+err$I3!~@)u(L z>=#rV{SUW^r_ya#D@fc1GhVwmhRQfi!*sX_j}Bb{jb#FA;S|6RbbrLPhL7NJGNq#r z4Wq%1VK|~eV!A zffB8qeijt!b!gHA0~*s>2~VVKVB6ulO#ek7#Lkw$Uq5#9KErwNXUjR3Etdd0X*>&< zkxu&kZsVz`J86(<4{N%ypRPTi3U3a-z=pl4yz-+X{B08FwxS8lLcA2_$DW|+*O%bT zlAUC$^L6++$B>_~b>rq6TiG_FIC^JGJQ^+Of@=}te0_m2eb;>)&F6^FR~sVOzn?$x zoY+=6afJcb3u?m95&K|Ul{`&ua^mtku7IcNCzv6f#12h44sXvUknmnjy4NBdGf&6z z9S#%m{oRL5_#LRAEdsKp9>mYH>)BEVZ92FhfoR$LGxGv}{^46JjAMFSWTra3lxf3@ z_Bimk;|A2$P>#zU4kmkbSAwaXEKf>$h$*3QTy=c}|F_hRi?pb4vHj`v(%^QyTf3Qy zg{i~Qk)rgj%~jNhb>-7`j9?NmHl$uA5N!wK*gQF1y0piT4@N0cmTAcM+Z{)pE*tVt zej2->G{m=dETIEl_Eg;u=*AjL)||ebx)kWq{%L2Kon{_=|G|;Ag+`Jw;|@a3y<+lh zye=Nf>3~U2@ic@^N2@s#Xr0-5T)o__*Cw~pcct$m==xR8Ie?1xS3 z?6|@zNf;O&hhCazaP;npG-z22W`@_J=&oW?U^ow_J{-;?k4&JqRlUiBEi36G!^On* z%S;p=v*v12O*qc;B64mBucGb~u^>HkUov!GNiCM4l!fBg#cTAbVY4lT^gtrJ*O(^tj;C3TnL@?z+hj%S zkltmpiHdztq9WhUgI?%G^svtaW361e=Rb4;U*ow$bQyd%v!_ccZ{m{B zkvwbNR}8I~#&3bxrDj+@4#BUZejAl`PK)?kN&3X;)9Gf6U;9YuCW$hcl?VSO#L$ zYI=5&5_cNvy-Qy|hj*=7V7M`dHto@**E$k;(Rm~ub0^~OCL?+{PJx@uH{t@z&76M> zB`+;9pwyY8l>aE=_(qx@-#rK;x4Fa3VIq94>>Rpp;2h{Kkf$F-rRlE9x8xfugi<{Z zHez6i|5e-YyPdoENaudo@^CSZ-8mkL{k^$Mmi5Zeyru+#}fXx&gk$EvNsQ-hqdG0{?G( z6|8w+LM^Tjhw|z@G=AD!a?{P5ri54E9#H1TE?CfgdsS&=@&*)7k>%>6mQw!@JL%Og ze^KoIZ^7#u;Y@wH5&f#K!W&1a&}G|o>7*wk=qW98>b`I+O-RUuv;=oSY2H5)mwuLn zk2b?&bt32%6NWD$uaMu5|G|4v8UFRu!JA$!;g35!p>*sRu=I}*q`kR^TR*z7 zhIQevK;b7ob)G?#NI8xk<%%KE3^7s)w*Se-!0>f+)J=2vw9|(EQZ~RFFH>oQbR;A- zmyvO$g#P#Uk3j5B2no88B!IJL!KZ7n;P+K=8YE?e6;tcrj-w>~uX;W|l_sET?OI^! z)h~is`wrs;(O`PoT!$Qbpw8d*JtHLsXRtDN9J_u|7Dd)8;9R!~R1lnE5uc)mw0&Q6 zYLBEVGsg4hvH^Iz){%NG-w1m&Ou68?G1>a{C$ugU!DHEkc&^&T)9GA#J$VCZTCxK6 zJb4E}?M7UyFrFTY+DWEa7C_65!_-IHi{CUdppDJuROH_&yuIKHo8sWg<%+9WV#EPd z?*r~^iLQ&VCT5?nhQ~ukGgC zP0b(2zy1Ygi?bo&R2=*A=m2cE7fqWB+tKxB1UM$82n=PBKMf%2w&_NqM+cMNes`FL;E+FXJaS4Q)k#2Xl^ zuL72m5olk#o9}+|X2|zd0c%dbz`P;kOGfL`mLzB4<{{5L|2>n3^OHECJOM%_r-1ak zIdswe?R4dj*Lb-gmgKyNgTR#?Ag4J33a^>bp0Ufo>`(5H{(O!7`KAP?U#63LVM1^+ zoR90n8*ucs&-gBJNS7*!WUF!#K)%t0N_;xNSBB@)Hi=*5+YV7^TelJ`o=9<-%OByz zR2%ww$4!`5P{^~~?qXr!a(-H4E#4oIS*EKp5m$&jV}G8h@#j|xpyKHcsxd>I=Cz%| zhp)t;Z*(up7tDgw8`8kD{jgxtPH}GXtqL2ZhVhph* zj=`5IQEH|gb!d%)?y0eS-|AhMVP{C+NjG6|k2UXGxQMQF(8XOV_34EbnRxhT7^>Jh zKy|+x`x7>ums#3?FiC}bOUTh2%ZH?Li3~L8$Ow+!mcYLMt_T-8R>PD#-(ih+D+$ZF zE_`uisNZnP0I}l;we{af%9_WdPtTs4cf{a(wh(g@XYjkzMZwyv2en>|pi`Ed#DqVo zEasmm_c)?OgLJLw_WJ1pk%=~Rx6D&gHKfaWZqLTCr$%zocV`5)hVs{Z8UeAQ$}ol~#Fqogn}@zUYGg2$|4a1~o+Bu6&P`3qV@Uh`rx9nRFusKOy<@C{1C zyoF0K_mDHK6+M7vnyPp#VxPkr-w3)RCk!Th+)WGQx8U2{GsLr_9Ea(A0sX75aiwDi z9?WbPHWYlq1cP?cqJ=`Wl3-YpVD8w{{0!6F?h0f4#?o=eY-nCcJ?xJ8z}6q{!2z`n ze4RUnPTzG2`zI)va`|j~KO%b>Kg*CX?Ky)#&E1mzqzUPY0H~f#WAjQ2ORL zI@?od5A=%qyMo4fG+auwQdpN!5cOz4;ke{hG*kcL}$o2~h`l;&>J zquDFIu*_5?y0OI?l7}0~;S@d;NF%0unpvu$S>5-p@ zF#5LRNaPF%2U~TEfMvdGA2fdTP zTqnOg?QkI6uPwoQqQ-p6;32rC8A$Vo`t;&0L-nbN1+|~q0TVV$(7b08NdY{^Czws3KTC|A8FNi+5Dqf(bH-P5fAi4S{V z-XWkXpFKeR>Azw7j1EEFY8P-)H|H8!C*kGW6W9*=cz*n4cp~u_CHH0Fj65&MYJG`c z9;s3c{sE0cKCr%V@%Y(sI#2JBfrAybFvesPRo4pT{b}3rq+2vfPFO_)-P z(gpNnvYv25f(k5kvxAtvJv4mRG<>^Y6s}nA3sq~>_|*|RuqafU@6I1dJ?~e6xA+2j zbZF1*bq}F@XlBdq$9eeqX)JnV?uHFbw$#MQk)E*gh9sE^oU)`7S{?SF@r;|$d0(5l zCmC|_S)=JqcUxRhj975*2AYK(N0D3?nwee<7UK_+qs=ueLt`m3R!XJGk{?kZnh(7* z75S|f!{`L1bb97dCJdwUZ1a$=5!UBLcL)u+;$H$j8ojV>j0Huf1HA36DQ}(q27f=0 zC;R`HQ_~|i;T}(gd2?cU^9Ms7xgrN_<@fQzl*RD6{4J#Q7~}Aj8nh;MB>Cds03zcr z3W=dOSRIYPjnhxS#F0LaHssNfq6sYrdC-}7b+EoG`zHk{8He*^B_GUfIj_I!f!DOl8MPH#!y5hk1V<7j3_4>d^e z=$ut_&j<&YQ{ha@{v_jyZ4P`yr9M8c8phWL==0hIVuJHC1iUvi5@bXl;_zcDVS~Vg z-&?Ot&)>6#?VD}6=L<@9SWV|HmW$B3P>kj&w82KJDU=<~fSTOma-E~vFtxB0t0hl? z$)tV!r%VLfw|_6VJvFD|!-n?v9Tk}31AO?V!zd;h$;A4iu_tmRm-HF(MyiNW<6+_S z$E9Yd_BaWF&x#?T+na_&mbdN-R|$432twbJ3%JhixlHoG0r1IpPN|0Q54yK!xVBH%Nd^wneCK~r3 zBS8{Z>12V-_&q$_s{}4cz50+SqXT4C7m4Rz>kj$posxt zaP<61rsMh+bHbfSeN!pyo@PQbpUXJ5SpGwg+fT>}2{k%FFP3iTzX6%EWT=Y5BUbDy z#h(Xc3M>*oK@I6-ozpWw)Mpdl*fX1^Rh2{axO&0ru%iH7l62RS$M|Z{i;o$aF;RC| zNn_H2;4vS^wO6z|7U})Rn!_nbr-{<+s#c7MQ{ub#d&3Sdpn)zL{9t3-(41@-82rwK z)3@6}I_?(QYsw2}YrG+U-`0S!mmdYqb7a~LZ9cuB32toKL-Pv8K}*9Hq2er0__*q$ zVB>m6$ha#4%L-4j89&CsIE{D0UAJq6CtAaWy*?KO6Gx6_bNqS;ZLM*H}Eie~)|I_o3>U|Ir=4E>ZpM_xZbRUEF!4NIQmh)iSC@3iGK|5v7w9n59Tpws{jB1 literal 17128 zcmW)nc{r8d*TxNH9ztX&MI=IKFr2+MDWnXkR2np@Xdp$?w<1Z>V8~FS(1a9`;_S6a zQc{MZD20+n(ws=|`Mvwk^T)Za=bZiAYpr{I?xH1@@F>s_eE;Ud7`T9&Y7f)#z1{JZ*iy5Yac_;Qpo zOb=G2GJV!`pHnW`nGuOw9LMte{a%79xga3p-?3ln=`ekLCs}aT7y3?7IOIPT{91Hq zTksnks?diPQTg!kBp30IQZjdtg3sDhLo z)PGfjl*kO6@xqNxel3Y}D%7CWI09F(Jg~pJ1(tN>vrPxhaOoC9Fs{_&5zeXb)*)N4+OSh@UJ} z-7XI|btK5B50mL&ZVFVq%7NZ`6+X1!0JyBuqMyEfspCRQesUzqf*K zxobICQ<~qi?gCqtW1zBBhh|?egW$y#P$IFFj#w$n6W2aq&I+6Hk#Q5c zE9ugOdxvrK(}Qr;MVM7mjStpn;=h`Sv@(7V_*FabvUBd#h1o!<>oK-qt_zg>+sXzW z=u?xp?ReVjF=|Oy32N2Cp?}z7n3^2MK93GW*}a2sQc4VQ=tWqyCYns_yN*Y@W-^27 zDKvKUT6(hVJsT6Z5c)Oz$t0CDxOmNoKe=d&DspyIDl#3`>v2d5xd-jB-I#moD_f*y zz`MSlW!^K#;wj|`JbJ-o=>03thXrn@*PxY+y}trqeQLqL|E@u>`wuvE-x!wvO~41- zfvWv?gnkwDvc%W7V02m!(Q2@P|Ew=SbMq&5XVG}r)%TsD{2y4GJc36~`HA2Et)BBf#2?H)m)Rt)+* zI!@Y7jN%g|W7+i?OZYaswfHdh3^>G%;U8><6Ve`z`+w}F&nvhv;DQMswpfq)SH$Cm zPajz7%Oc2-n?ap-%w+Taj;Bss8~^i^;)na6!Vr}rw(sH=TK-`V|8@NYIbbl6j(clM z{@G^2kDnUcGdGq@RXBn#pU8sa?p&5XVG{i}_6{V1armYX=m+pP2 zMfJm{bD#BEyi3~{Hu4U9w6~9_@>Te5d=M@!3&B6W`h3f}1X7SZ30!pL`P{7%Bx~eN z;OpDiL@!ldUfaNC|8|3tQPucM;TX*D$tRYP68uwr2Rs_B&DTgo!qd)z+^C+HOoAh76~@-w-YqEMUr)V`0@NeY(G=4X!?%jU~6v3+p;wzz~^t|1IEZZVTo-8jLJMlWZW*nm%hi~ zIaUWMe-%Q)t5mE@ih^xX@1V-PPPkzA4Q!rbLC+SA zJ$a86$Hw8g1X;cyyAu*?KY%gUK@Ybi(%|Zf3yxS3SJxosc=RIr`=1j`KV(KXSwF;} znNi@MGn&2jR_AUTr1<-Ndi-*qBJ~RDhWnvK=p#RgE`B|Qp4MMS2hI6yzVD&i6?+;vwref%o!N^xD^C+`D!VE59R5;p8&7SHDIvY9zW*J;1-^rz#~4D#Hg+1g=_TLlD0b-x92R;7*~Uz1=`qr z_*H4ZIt?26>m_>2d2sV!Yd){$1TNRG!RLKTg&pa~pvKgT_tpB~2CEXBw_k!bH@grG zzs2;KwFDb%52Ag~w$c@j37VMCHS#<5oGDHE@Ij68uo}O(VL4Lh|#sP!l=D^bmHBEV5pQ0v)FHVqMl3^Neba} zR|G5ze9eS=-2}7W&qJRZ(J+!+gO`q~yymS0J=If#Sxvocy4w=il`#P(&hCK|3;&j<=;opmKZ}T zUz~)jnfIY>b27Z%w-JVKj%GtvQ=D#VMKwRR;=u3*_;7eCTK8Urko0Qd=EZTS6Lbvk zyt~D`U%o}XtFEAKSw$ZIn8PE2efgVp$!L)y&)4iwg=N*J;Xik2+A~jZw6i==--nqe3$2qKgT&Vh}R5o`~y!iP8X;KBD%`0)&(k6O)z z)3#m3&sR6llUo{DZsJXLrLmZNYu4r;`|O$Is`*?)SDZIIw&KQq+922Jh(7l<>8Mxv zD8G9@X8#$=`_f*+q*>XhJ<}Vj<<1DTQuIk+fH%IW$-@tmMfJe%Ck}}nLoTN`lK!7E z%=2?UHg1;&qs(gjG50vZ)4F zTfc$H&o7aG!A@-G?XN^;*KnSs6NdUNCeYSnL=VV)A=yKUpmC%kqtC9nz%c$E0d)BmE6OY+$;8Q-#;hWU|fZ3Mibe@bH zu!qfd?0-1T7Y{m&W3X5TiCwu4@l}HfV7-0Rdcm~r>zH= zg-H(V=r-p6D!V{s7soB>hA8#=3_d@vOSdc-PIDw{SZ_=|SZ%%{7%^fKNWYY&vksaN z@#jhKFX@`_>J=rpkQ9x#p3Q7t++D%~Qw0xx&f#G@ zRj_)xE|of3fPMZWI6u}0*Y?k%MO9xg8{UKEw<$0x{V44sQI37iT!%Z3LZrMvF!)(4gnP?1pt|9Q8hAqPF`*TsM3N8$$9?qv!dlLx@MyZwaQG2os~|@2)=uE-m7{UXZ5!GXwO^&4{0)>|8$ ztc6&AY8kEnmx6a1WVyRHg|0FS2&?{r&rY4gq0W=&liD)!x4I8vuGvtz8~H>+iEzU` zX9XH7r||C?+O*2N8QqsH!;ovoVEHW}l;_cSz&n3=MJ{5*nuZi z51_cSBQyDZ6OUKyf)4j(ur%@yJFgf`F3TIy^&dURX}SVlLJwQBNSUwba^b$u-{QFs zlGx_I8n&)m3Cv3o+5T2Axqb$fU(e+?jq*{OiBtL64zMRpoTf?=`uIf_98NkZoZxg1 z3kMsS*Ul>3*?$G(77T|VYh7+JQj)s7zk(rUs&HoVTCyvR@F&Awpv_hn+HEi%vqAz$ z)(mle*X$gu%N_y}&7av1{sLY;)WBQA>X;*cTRJ%VBF0ZLfL>uVTh!nW=F083xI%~j zcSDbQ?DN3e5gcN7+0w^30gyC*9LMkG6~4xxc*&U1=rFQ^%(fzt!7`gRlpl_QeZg7cb=R zcgoTWC#tc0q$(}pGx!cC8UAA9DE>5hAa zWU#jZl=t}KHND>`tEmcXv8aCfj-{JN^uqmP6RBH8GiGM1Q-eF%DEDbRJl?+w8$4Ii z`T-LzKl>Nt4~pjP^d#`<&lSF(v4V(+&dmP3Wc$T>LbQ5IPDBeymx~R1I%OhJJ}t?m zhQ1}UZrrD;Hh~!W;Sluoc~Qe(v+?AMw;&cNAVi zx+T%r$xm6z$IrN{WG;6*A;s$@XF^EA6!@d84qv|2!Vs5fe6~^sDe@M=P!)HsF!g{y zPPmIt#)-TjK$ce|W#Xmzt6_V^6Vfrbik61hz;f>->V7B=7sri&H7}RK#BI5-rRq5> zzUD@!*on0EhZ9Y`wH^ZIzJ|MY>a??SI`|Jt(vXiFb-ko`VBJaTeDn}kUgyJ~-`L8` zK3WQj9j)j~?I_4ni=k-VLoEMUaKnY}RAG~lY#!~xGmKOrM{F}S_BEv!bj8r8+lS70 zdKcrq1dHz2LT++04a&|)@$#2S{KCZqXgt@=O1~|jkMc$d6n~85r^4iU&heG_H82~~ z|Lvj1PG@0me>A=G)e$}0|DpODMecSjhjxvANss(~1Y;i+<3G8tq|yHqT+XZ$3Xhcu z&jqaEw>C%fpI_Wi+h8mmqF9M5Kj)#t;v(UMz4Oqc<~SWb=QUIGzJisuDI_6cA8zp5 zz}tTvf@^{I$obeQkXOBfF8y_dHcRah_F7HG2BXzb8kvf=Y1RS>w_!v{Pt+GU&*yQO zMxa&p4xBHaV%xEe^l0uUD5qmY zUO{NlQDQnlK&Rj4urAS#q^^0z*zFWjdwe|xIV_|F9}xRvtT9w@6PMQIv9ogy!PRhW zK74{XFc`@y62{ZIOE%+1iF=a7**CovU0s}i;O*QVN=Q?(V>$=XW}{IuE-{J8QuocK1ECazr1Usn#JyM}M2Ys8n+$=j?+po0YU z%U8$gMm9JiW(DsvEFzzBg}BMjm&?Qt<^4Ad@fdF?J;Vmlw|^+TSMv=YK03lZeRPH1 zGfwmRvT*8@F^Lvs^uyV3A3kqk9A8_x64$qffq8Q8yr>F$+1VOYwbF!&$-0QzYDojuCzgnS-+sU86CV^#J`v2FC z|9$qqQ&px$}3m8=@wJDZ#I2{w@E2J=)6md+S9YNwQsbNWAEVB!a7gjueL!7;{ zpy#2I@Vk65GfDXg#_k_r&f#>lHjtzrGy`#I`4=+Yx*s;0_>rpLB5x{pEKBZ>$4nJt z@^)-kYETFt}$fgl3c-~g8isulF6dqWU$KXA8?Y+Ez}&5f^%jp03J7t2Odg<)qV23 zEl>v&)(i?~ty&1T_Gr?P>PTd7_OO3#pD^>D5q)L_M#UDT^_4*D5ebKBUl)KJS4hmH}$tw&5T#cUrKBtF2a*DG+6@<@Kd z-hn}$BDGSyg3(@?OlLXe#$)91k{Dw@W!kaW$_cAydjsCxgQf+U5PoYn47?}@y@!$b z=GAPFoM%m6XR4431^!SfI}2Ph58#NQov2o}0LN}Of_p|9T>5G^l2@v9jWdUaja|&f zuoSHt6|i-_6cC>!xV^p>Vm`RS%&9JLYS=lDld$H6jY2T7?#3NaM@Vtm2+VDog$sK2 z!`tn>f+Np7Ag)Xsm83QC_D3Nkm?W{3!%y)@R0Q$I0zqAsESk8O(3?M^Kqp3;RzEF= z0l#Z3GQg1h?tg|pc88$QZwiljr-0LHPQe2^d4A>Tb*2)bz{|y_<3GzJv|N4^G`1VV zJqI18AsGew?=O&Vi`sBwt2G@MBSt$OInbp`AHq1H4QUkh_v1c@iM+P)eCft!U_O#C z%6AG4?3hU9b31X7Mk>z8ccD2CwQ1VYI_$Yt32Ssxprqpt$c~ZbE7Wt~zvX6BeRL%e zq(s2LH!)Gqb_-(Hzd*fkYgilWjekJr2?jSX^PCR9Odhl8c?jY0U+}kX zI2rFQ#mmAa@nhCBx+qqSdgq_O(pwVrq|GNXkRii*T$>=ITasG7I!&s2Zm=as^Mvya7EvD*#3iEm$xwo&`G`C)4(+lVd$HeA5UU z9A~Wv%TKvMMv5-%4>X|K3*InQ6;ry3+yl!EWso&b6P2#~hJaxOWK)k9@qN6KZMSOx z$2B|Q%*L%;-oUr-+qM^94=zJ&YXl<3zy)gX9vOU%T9Wo*@9uz%I^orUY<(r-hGFLX`79GhryfJ7-E)oSLBsd3i4+>M%C^J z;-RaAi(A~#YrYfR)7n6O#wU}e8*wOmM2Wi|a9|#}?+{#f;M2%gB<86k3N&L-^;k6a zFIa}{odm3#tHAT+Buts2L*oyxgNE|wQ1WCkG<}MN+s3-MOW_I@-87>6f|?nInDcvc zDslTq9ipAHzvTGTSa!HOpB4Iw=HY)!1ksvSQOywEy&g_7j zX(wRo)~v*8|=nKqIsg0(D~G9P;+{S<*^g7OiqFuf2bmbmm=7pffXLv zcn8kD$YzJv#z9HT0Bn#s4`!7+arW2ASf+82bhgC_qKkD{@6HFf{hlV+-hBoRmoo9P z+Yr{!XU4alsb#}%IpYWUConJ63pzS(!TQ_2m?Zxbj;;9v5ARiDnP{Fh%bia3&Gw>H zk_|VNm8ae{pOE~N!Y^~@&|e)vczo#%>{l@&2b|>a&G{aDG37EbeJ3f@zIc)SJog0? zw~R$;|5-To&3`by+nYAeA^2{kJoMzxpa~b}gVwp-croif>IJCKp|jn=qIDdf@a!Rk zi8P>Tk}H?KF%*xk5+l-H14PyKBiPvvh4?*cR8Ogmw9Yk!?tM1|hw{$i*{6SDP(hbE zor=J+CGT*sHC1qNi5wkOHV+=joWa$Pn=vN-GqlVQ&9=Ma$kd0$*qJbvd~`Noflv?X z=Mv$q?H=LBp$f1da|e2V?ZYL0)?g9j1Cz}nF?mM?YIYfU;L)y4b|f?G$Vq< zJb48wp~HFED@(fn=UYe}qeo9#MU&MEJ*3CeABBFcc7A{Cv9QIG@1Gb9N1V*z-%mq2 zJKG+=+v)MXFlX-k{63bJcwv3A1n(ba3!Oj3AVTdETlD=TTm1GI&L~??OdZn6e@9m{ z{i#yW{5zFq)q-jA4)L6{$*45T0X*1Ck@@v_C2|Vxuh_=sSuMhmEk| zd@Oqrug(pJDL}WcBNZz@0m?)4>36l+_|8w6U6ye{^RE$jdQUTx&9}u{+v}NJyE>h5 zqYW$O$HHfyonSF=5wBZR3LeLPg4Sy@@$d2qoc>=Jj2o}ZYdWvO;nM)-yBcB5cz3FK zJ_$|Y=LpA3zGePz&FIppWBDVaKz6fOh9BRvn|Qcc!Fxdh#4s&x(-A}N{Pd(vlBe+d z9s}-hU7dz&wvkya$q>G&6wW^R0jJYcc}fc4SB(*UN{}FfjJ(#BsiEl zAJ-}#f{}y0(6#P4c69>`xqgfMIKK^4FX`X`@krdEF$;Da_JhmABxzpc5-@R<ll{bE?a4+X4zUCz3Sd!cUpR5qvT8m4^9CnLAo^BK1z zh^y^*D*5Xth|hlln;rbwP4#$qeasQIwcm!P<>u^~sVPm3ae;#1nV`5CK*RJ9e7NL9 zkH=NwMuY2cNMj7_*om0q5mXu%Y)d9@&%~Dh9O+$KRW@h8G7VqRKsNnlLJz}lf-N7F zc<;dwHrj47ymN_1I4@DUiE8pI)6(JcdmFy}Z8eh^B2Hrr$Dp}LL+z?_;nS1=wrSS^ z8a<)_w@oWz>G72Q@xBFx^KzKvS~)w5CnI?7<3Kd?c`AHjdj?-VI|XBPnEG8=~?1pDg%LEe)TFN>Ra>vl(;Ou(wqQ;rG!>5@IkBgYDDt zbnJ7oM7n^O|0%^p+cywBH2uPr&z&7L_@cL(&|c5V`w$)OSuEcDcX9fJ!my zdw(SveS8EjwqFN#D$kMCK6zv+Q{t!33?usc&fw7d6kV^MfKzpvaC#BK*I^#Kxy=YD5E;%elan5oGEr?{226G|XFd0r}kK@K(i+IYwGxi{M68`?~1*-of zP@*D%_?bG>ac4U4>1_&*m!0XlWLMDJ?SjcI&8+R_Awfg)IIbuw&fm3^lD`vvvmASW z_Dswc_Q&;sHhqX0qf+2-M+BL?{~ywIvta9)D98}|PV|?af^|oy(_g`5xVj(~KhKqc z+9moh?`{w({Cz}TTCAo~c8dY081SNcb8ZkRN$V}-x%6<#Jv*alp+Os@Xx3qs^<9jw zFUQRWk*vq<2%EC~3QRcYkFF|1_&|XwfA1qr#WK#p#yOfC9xTGCy5aC5_Mb4rCIlXu z09|Eu8F%L2!!jSnnhM9$|CCkex=9XveEVK3e>xo>+1CpLob2Jk1T_%6ljrlVUqaUn zZ$RAX8hRcXOJCm(6LhTGjhCwP@IrYZ%(s_;Xe%|^eP$eY8!ACVyDD*9-gU@T+Jd2r zuGoEgqCoTgCWAGd$RA7E;^wYcJe3-Tk_%Po%YQ!vhfMq6R+I)9AJ2i6@^9htRH0DH z!;S}8Xu;g;8Sr03G(JD~3FmHH0_=1Y-nUDG(cdIc@x^7R_!GvK&DkmJu#*LRunOCr zrvfY2IVFOut*-Vm`~(eV3SkDn|a19k_GgMsWE-1SF_7Wx-4C7BV}ys-h|8a-)g*$Cb# zBgJQSSVH9W%Mk8677qCyf?pSx!mNAR>|@<$JThoW(%p_g@&31Xx8WO%eiDmnqmszx z4fn|4#Y}8IFa;J&7)>U=FNQt-4=_pLJo8V{^iPo5Zi?$O4`YJTC#YH2 zfFIB8W77}zVDI*M^wmTiZj|+%EMK;Ox{e=5E4D~dq3%fV`uPFX*AxlzHnkBo!Bf0f zFp0jgoi3VL9O(E?JAg%vFzTQX^sgSkBToil#gd`4%W4o)W!(6}0}Q9Qeu6>^MmoM^ zuzkfsd^ut}HTCg@XZ>gKn65I5?oEK2pL@vF*A7C7fit+x)SV2S_XmcUB!P871zLUh z1LKmk`QXZ1EOn{|&7W`|%0zrrQtCwB{x1#O&Mty^$p@hPEQS5L#ZW%W1khz69VMkp z#6))Aq4B44_0I8V_UI(|-FbnY+A5H$FooPJP$owtN@0<|6wlC*=ZYui zK*c$ZKKd_kL(5}`J_s<}VFFY?pM-tuenIhuC^~*#B_6AJkK&11^y0j6up*{|L=^qT zEfH_Qv-2DFJh{(y7mLBJ(Q7f=J`ZxbHnTHhrlC!y7GJ;3fgW6ap6#4G6m+DA!JF=8 zw1)yTYf#~5!}>{TkRG~M?#DG#V%YUdTHw6nJ!uuqY2w#8@oR)n{KQ4G`7SUw@5LQ1N63S_dok{BB8%8PlO${zjfzdW z{F|LLE;wL@%UAuveGi8c6Q@{~wb_9d%~;PvO?8AHp0*Lzfi~gT5b9P+O2hicUnc;Q_y(qsN;5UZ_joIN0&QD;0S3 zvK@VzUJL_xpVTUa;Da1<5*xLau9p_j2gC_-t9|+B_x8~HbQ^tJbrd~TMj{??py|~% zaI{?;MvHpk^ogHvmuMH|EWLsSNbdx-*rP0A_H`njt3!_-y8=i4PUJ^*_o4j`J!*UC z05&;Gb5@is%qVrImkO3~|M+1)5wb zu>xFX>;t)wDln-o7HGQ{vb$qcxYy?m_;od~p!9_(D-_ z42c=Xv!nW;zCTjf=90;d`K|+@+Ix6&yN9gqyF=8TW}QF^0OiN+IrkHN}J5)rWm<-Bkb_eq#^^I ze0y025xdp+j;?deF)js1rAX7?6)T|nTRS=YV+^(3Yf6tAc=F$)o8ilrC_$3gJG|aL z9lkDnjB}*#f%b!5!Sc~LSmR#{iz?Nqe7!QiQ7b`d?q_%xbDjOyJcL%758=-zjpTyr zOQnHIJ6P-6pWvUWLd!(Gis`K*aA@lYy7<9Exc5$*F3hrq0h>xpJ-C-#v7U^+@*mOb z@)3+mx&tpC|Az~1MMJ4eK0eJa0ROsE(7NL`+%9wCFE_8n#0n2uYki4$`;Wxyv;`H_ z*1{W^vHZxJ1cAwASNd8#o+bA##I(KFaa7|aa;(q^T`q-X2oHKy&uY^ZX)Q`X(av%*`;!+O*lENk-HM>Uwn@>=Z{qPzi!L{E z*nrQ9>I9b}(#ZHuf97M93Wn{{7=s2o>{a{@NAVZkLZqHZ={?9xPZI<4T&esLaPV8GWW zn!}qCB`RsC$IFIyGvV2EREnqtYq+wr>>1_Xv*Z6NPL4n`Gj zp#P3ep;;Sj`RCE5ymOTm&41X>bS5;BJ>QbB!EZKQ^vW4(XAXmdZzZX_=@6QAIuebB zwc}2cC}@5*ifUY;DD&edZWEtM_dWjrHQQVzQ^)|j#+~pd$Z#3ku{i{(p7z>R?HX?r11J4DvquW7G zGA7Fs!fiD86=y?QyI8cpNi@MRsTEN7?04zWC(-zR@F;c!NMj~bL$h~&_}CX=jL}5) zIxrOdM{Xdmm$nLY-b?^L{{Y<8J`%+?P>``|0~K2_x;*Rxrn}t~EZC<;qbn!VPP1Eh zu52z}IZK6V70iOpZgXD1M9(-k4$^nY@Z6MM7#dAzUcqPfSKJe7vxnl!`BF@BPb^9P zDnlhbr_-@6r6Mk(sC1>9Cj@?yLz~;7@UJupPTm{C%+df{*QP_|8bS@q-op0{vp?ZJg~Q-BWCJz1?uzLO-55MsK%dX^;G5TH zgZC0`-lQ;@|1x_Hbea(UiF|_&#TZC9nF!;)t^%DXWpXF&04Be92Wt{!Xf{US(#4q& zm~#y*McUoHy?{)wNx8F1SW`isEG%9n>a#PR;Mu9;dFttD*!}MjR@^yE#wwapuPO_!I%g|Z8|MiQ$Vk&c zzf?HrcoThP$MCbFU1j**dt`6eaqzh*&z)E~c0w(ziH?K4vz*C`s3a)p%pa`?0&`f;Gd#&$bNB-9UOfdx}J`veo%pFt>z@)@k5Ze&|;g6 z7U8vE6WXP=0+XsLm~pN;S!25vex0uuJ{}TEO1`#=IJQctd;N@gPtfDqmp;Sd(1SQE zXB7EmB*AZ;h{F9>6!1!u3sv?K;1u(j=za#^OPr2inrtu`8I!`=LoNB6dr~xRTqLyY zie@Q!uKcfvpBNrK4RjaJe+E{a_A7_V* zf_@7((2yPr2Oqq{6AJ>^5TP2Kw=Ko)_ZSnr-xUCx!$(nPngj2Al=*or32NZ%NKgLx zhP~OwJo=O#)%`P;tFzza)NLn9U#F9giz2b1-jf*IiG-1wAK;MiyKs-jDi9Ox!g3x4 zf~SN6sBI5|-tf_QVaiV^?hIppo{R>Q-0d*zbrx*Q9!V2hH`D7at0)-PlP8s<`0WX@ zaP04E_`7@|L|ste8+TQ~1rR`)}pgTW$?hG`GFGpjAuW%@A zE;fptB693Gp3*ys3)`glcC|Ms^~#k#PHKYF31+Nlj{~;a9vA%nDM{iYKHF97Q^ON$ z(=cq&Axtz#BaXEh5LtH(*XXS*-8adT{D|zpo2^Lv=R3f%tU54j$`b6~E>HbBWvOL` z1I_+DmFARA$9QWma5F0uf|VMqs@7*`Pl@IW-?bPXZ3RJ&!Pxm#7vF@Z!JTbK$oRS# zjOeK*9S_AZXs#$j6r)7vZW+b*dY=ce1Y6LXaTLxgcQK9kpHM4oki5Dsz$3n`ur*PW z)&{m<+Okuqm~2R&`P_l5mof0gYaOoFI1J{=doj#82g+C2aJvl|!fJsr-M>Bq|Lu&& z_iHR+)5UDuE7EVJCBx~st6!j4e<~~tnF}Uq0?=Pt3iDT(;eV6Od70WbpjUOM?$dUm zc$hl`-~RwoMTWd&u?)~52&v_lP)m4)IO)6)Wtp_;=#+7AHPDgH4z3f(bRNK6TZ@Tb zT(@AAy&-jMRN@LH67=+JcY5mQM(X(54$p}G7uGKX=39@O!cx%e?P~h?s3{$JMV2MY zG+^w4Z)AeaV9EAnpUL=Z`D~Y~D>a%<`J5A7_$6r}6eu5pXcu>;t9=a@R^+qU3!V7i z?mukF+GS+beiQsYB9rCqy#OA^U8&VxNj^909)9yM6cm3;hoD!Y9xp{4?J~pZo#s*W zlbRT+4k-n5^(nA)qBif(egx5>btEy~n>P%MCJ?jAnmafFW35sa%IgB<8%+k z$Q@(Nh8lcTL?l(4GJtW5%JKSQSFAqgNu}x@;)Jzg#5iXe11D?j9N|k9r~9FM{9Qp@ z;RT%KQzhb97hsi^815V@OaG-!6Q*o!LE+jD0(B8zp}6x6Lw*#J7GDCvHYw;!J&hlt zw}YL6Hq8oZhYM;)aWJY@#OX%Dj}^7x5qul=w09zn`-v+84B@`+DkvHwfsVye=N#m1dFAIMh)8gzRofUA z?LW+YT^_@&H09FO>Bq5l@B?0XWJ%pGOvIt@7ja)l8+h?_8GSkL9(vx+!bc$*5caYG z_RmU(H*F$Tpk9|R+x`HG#Z%y$mnHQzpTgu1Z^53EYA_{C3A8r678Z6(@Vn)|!QIpw zdWT(xO*5it?(gw*bEgxFB>VW>$6?g|ts4Du(;M}SCh^yEhJ#jS6DtS_AufZttfyx+ z`y$E^Nj*1Uk2}Id*`QglPyG&@yRi+%-|~mmA5M}%aXotZo)~^~y3W=|%mssuzj1HG zY0$m!0O~~iw)uBiw(+|p>3n&eG|lqI?DR)?XT(;lsi=VsFaF>XWe2#iH5nW>T*Jrm z$~e9GDw%wzgGhCJCbjXNWaF7PEIqpsejJHs-SgF9#%^ae@2Mk;JbDAeM;r%z0VDH# zhtbP7^WnMVMbKF+3!5ziaQ%ubIG|@OSQ(mZ*Yp0V@DPM!`1PystME7ZcxR+=LhpAd zQ=d$N2gcJet0efPn;v92+X%Nd4v_rQKgdUOO`Kkkg)Ixu!g7%>(s%qQ_)EWsoX1nR z8%(0T&#sZnNq13h-#y!am~bj-IT36<6zExtH@MTOo)vIK@DV$KQ@mopQB0o{PN>D$ zapl;%KZzNrEhEWEFQ9+qXezG!1y9SRlHlA&ctYAz^XFH2`tw05*9njz?@^pQM)z^{wx1P4%v+p@xc$+6d8t> z-|LbR(VP)Au^yyp|sWv+rJkii|#(Y~=?(;}y8Qr6L`6un454 zP7r#S$FW5{4iGC7HQ?gUMNIyTiPUhqI#t{y4km9_(yV(7ibVe1Ugb25-<$zWXGM4T zt^`)VcFuF{Xh2vvm~LFjqx6Q;=Z}2p^Okw|B5?_nvL?Z6$>G#4ZUCO=-)C+IR|>VR znF@mLuEe1=A0hBU3GQ)QOS69JLXEf&&Xt|YU7B>+54AZ$&3GlcTDSydYwn`yk^!jI z&SxXd1E9-S3r$*e;FHWqJLRjfu&=%ddchQ2%p`?w&#$u(2`g?M>PB<*C-SX^FHl`e zj2ewiguF+8p?pa)3;lZ=hMMWoNmhoa?=8x5bd<9AnIoaF!keX(m&8 z1NW?tfjo;YxE$w2PCh%%3z;_eUB3!~a@FWFuW$JE=6Y@`Qx3K_pJC9MK<-!d9fzv@ zN0Z_npiAadI@Q9JDs(M}t`kYn@G2Z;@7a%iPezf;Ik{l@wF21#Y2=?MeOy*eTGax` z$J3gyCbAhO{Jtsj=i`NB>``d6z9G=L5CvTu&CtJEjPuO>^s$OP4Sm#z2FlMcOG=pr zq@08;ovZCaEquW^U?E+$<19aM+aKp|VIX}!2M;TY(*yNFa#A^oh-por#_}RYeXbQv z+8ux`V}FzNcW3gj+OweJdJU!aRg_kh%F`(_^B~mwJk$E3j>?L1B>dJay5!JF2wY?h z@?odJT$FFQw{irRgSU)mqi|uJuF%` zg3X@3j)@;#OT3NLXlY3(TQL}pf?y+5`!@~tP8)$YG?((_FUN}RcnA&zy@O4QR=|a9 zW$60T3dfCaV$#Ncr16^{n<}gZh8Ruug#PXmiG`Nn-du3JD%1k z)Z*fmmh`3FS$MJHFewm$9dlFRj7m* z=Ct0%Ka=KR<>_G}Hvcj%bWVc+qpPqvv=h3D=fXY78!+*~S9aE_5En$4QpHO@q5fbc z^A%r!nl3vbG^mzv;{w6?I#Ir*J&+3QXW-c*di=Ir8{70V7(#aidvzPpNQVtV%0JHjeiB!zhh7a%bu-naw{wG%q_O}I4 z>uf=VTXrzZ)uBu>%Yfd~yM;$c8xH&Og}4%V797+ES#2#iX~az4Z!m)TdoRUR1vS_p zu1X*AW7vBs34RFTap9z^_7kXT12T^){& zKO_0tlGCi^;TCH3@Un=PGbSG&&xB(g-on!l$7A)u4Dc*6p?w=W;mv9}{B+5RG&qlf z!w014C1o-26xGe9$Sne&#e2|Xf)ibG>mVAd=yR1nuV7)G7NlB!VA_+s`8LxIV)Is; zdcFR|zW54I*)$nk-$XNMgLZUWECq&Jvf=JCQ3kIng{;mtp`BG?lvOxT3F%jmn`S_V zS=O-oQ;pf-C2?S-kjtE=g$nk6un=TL9)q@obJ!H1z_WZ?$qNq?L5Oc8yuGBreZD7x z&p`_;5%G4R{lkgY_d(G45r{5-F2T#XldSDZ81}7oXTvg$h;VHVd9Bq3#d)Lol?xJ} zsV_%IU)v3S=e*(e0~_oPZ9%d9XN8*#wC#H5J%M|%_tCEI2{w*NhPvYS@F7r(OJ@CF z01^W2{ro#knUg*0=Muhb*XF%^!j3)K7hFAqH2FOkJ0!ljl@z`*+Uh;#b5_0I3q`#~ z5h%Vz9}m352GG4Aw2Zt6`vE?GfrdK;9`n758^FB;U#PuMtTsM<9@IVjO<2AhukO9z zgv34jih@2i&1}7V%D6q*kr%y^VXwSJ8rr=_OY}VrzE->wA@aS!y*Ip_8wS1~H`%)l zbCo^5;axo>pkzHyC7`|fy+yo!^(#J8!5lt=yo|kbK%YG*hwDAu|3tmMAp<^14s*S+ zGuFLeS@ylk4`n_{1(80hfh@jJI_bRt*sncQm^(hGQw6;p_!zz=uwy>Py^lS4Nj|<5 z_#8g^^U%DV{usWnh%Y_Uk9a+Xin6@yrjNZ2ln6e9&VfEDHXy!ige1KZT&X=3(5bzn zD-yoU!D&5R;pM$wNN>Km@~u4^yn8+;Fd95R#1K8+a0xtAFxxvPNB=vHnRh(t5{5fT zwvRk=@|-(P4yn9$WllT*0000S6h{^-70iy6GgpxnL*?oK}F zJ1o8TO~<~a8sk0+pNPJ?s$RZOYC%0FWdOh0bEQ70;2b{~e2hK(Ob|cFC$T;BAJM+s zoiROr!vDQBcLBbw8TUS#>?ps>0X4pIp-H}6tT4W`cBMXQ!Yn-tUG2VB?6W;p;i9~Y Sdi}j8ejY#HA~?I>A~-v7%Qid! diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json index 6f201552..528797af 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json @@ -1,227 +1,105 @@ { - "inputUnits": [42], + "inputUnits": [84], "outputUnits": 2, "inputs": { - "label_0": { - "dtype": "number", - "min": 4.151249399907168, - "max": 586.4725394909854 - }, - "label_1": { - "dtype": "number", - "min": 186.47223882383636, - "max": 496.34918695509003 - }, - "label_2": { - "dtype": "number", - "min": 12.818880217505907, - "max": 564.7860747522525 - }, - "label_3": { - "dtype": "number", - "min": 160.9460986889124, - "max": 478.89482602620234 - }, - "label_4": { - "dtype": "number", - "min": 20.681431005110262, - "max": 557.1173870582799 - }, - "label_5": { - "dtype": "number", - "min": 135.1274696802808, - "max": 454.0862355189599 - }, - "label_6": { - "dtype": "number", - "min": 29.375938053231934, - "max": 562.4826339023859 - }, - "label_7": { - "dtype": "number", - "min": 113.22511415628927, - "max": 455.15365538508894 - }, - "label_8": { - "dtype": "number", - "min": 37.27265551578051, - "max": 573.3838980891996 - }, - "label_9": { - "dtype": "number", - "min": 98.00531862273047, - "max": 473.4382341601794 - }, - "label_10": { - "dtype": "number", - "min": 2.706973037101564, - "max": 599.2858408346702 - }, - "label_11": { - "dtype": "number", - "min": 117.7350326456234, - "max": 453.76022921684716 - }, - "label_12": { - "dtype": "number", - "min": 11.635752695869659, - "max": 612.8243751678727 - }, - "label_13": { - "dtype": "number", - "min": 91.05094143918305, - "max": 481.6467136241304 - }, - "label_14": { - "dtype": "number", - "min": 22.9353041163117, - "max": 621.0127886598051 - }, - "label_15": { - "dtype": "number", - "min": 61.619264849841635, - "max": 499.63536096409143 - }, - "label_16": { - "dtype": "number", - "min": 33.53953084457643, - "max": 626.4181148091915 - }, - "label_17": { - "dtype": "number", - "min": 28.455718477478662, - "max": 512.7953875856006 - }, - "label_18": { - "dtype": "number", - "min": -2.8065139589559984, - "max": 617.7828981986556 - }, - "label_19": { - "dtype": "number", - "min": 117.6886729722432, - "max": 459.5357193516273 - }, - "label_20": { - "dtype": "number", - "min": 3.7782929928570064, - "max": 633.7038985044576 - }, - "label_21": { - "dtype": "number", - "min": 86.77279076496669, - "max": 486.0751342925063 - }, - "label_22": { - "dtype": "number", - "min": 16.177018651157255, - "max": 642.8366376068107 - }, - "label_23": { - "dtype": "number", - "min": 51.687144639081325, - "max": 502.64037741142846 - }, - "label_24": { - "dtype": "number", - "min": 28.1461509145229, - "max": 650.2419536370577 - }, - "label_25": { - "dtype": "number", - "min": 15.922382743702723, - "max": 516.9301399988833 - }, - "label_26": { - "dtype": "number", - "min": -6.382516546058305, - "max": 630.7077663350849 - }, - "label_27": { - "dtype": "number", - "min": 120.16376158664924, - "max": 461.0881814514869 - }, - "label_28": { - "dtype": "number", - "min": -1.4074379536407533, - "max": 647.5041251714117 - }, - "label_29": { - "dtype": "number", - "min": 90.58035685591811, - "max": 485.04491883378125 - }, - "label_30": { - "dtype": "number", - "min": 10.174906800459325, - "max": 658.4893875478738 - }, - "label_31": { - "dtype": "number", - "min": 71.76407331703523, - "max": 500.55112323964187 - }, - "label_32": { - "dtype": "number", - "min": 21.11718120932074, - "max": 668.566957655395 - }, - "label_33": { - "dtype": "number", - "min": 39.557348432978586, - "max": 514.4287318106208 - }, - "label_34": { - "dtype": "number", - "min": -7.9534800405596595, - "max": 641.3232619371444 - }, - "label_35": { - "dtype": "number", - "min": 126.31599791044414, - "max": 465.6320514399833 - }, - "label_36": { - "dtype": "number", - "min": -3.8369034650104927, - "max": 658.2044139172733 - }, - "label_37": { - "dtype": "number", - "min": 103.73604938021917, - "max": 481.03793223993495 - }, - "label_38": { - "dtype": "number", - "min": 3.7075645592075435, - "max": 668.8017566330357 - }, - "label_39": { - "dtype": "number", - "min": 88.76136006394765, - "max": 494.63688258092407 - }, + "label_0": { "dtype": "number", "min": 0, "max": 479.19925350185474 }, + "label_1": { "dtype": "number", "min": 0, "max": 428.55538772969044 }, + "label_2": { "dtype": "number", "min": 0, "max": 493.1563637363407 }, + "label_3": { "dtype": "number", "min": 0, "max": 427.56692302194773 }, + "label_4": { "dtype": "number", "min": 0, "max": 499.91689098126307 }, + "label_5": { "dtype": "number", "min": 0, "max": 430.02650947613336 }, + "label_6": { "dtype": "number", "min": 0, "max": 496.3208721418956 }, + "label_7": { "dtype": "number", "min": 0, "max": 441.4487064271474 }, + "label_8": { "dtype": "number", "min": 0, "max": 491.1078639783216 }, + "label_9": { "dtype": "number", "min": 0, "max": 449.2829368661615 }, + "label_10": { "dtype": "number", "min": 0, "max": 509.81763121840737 }, + "label_11": { "dtype": "number", "min": 0, "max": 427.9468311472154 }, + "label_12": { "dtype": "number", "min": 0, "max": 519.7167764450397 }, + "label_13": { "dtype": "number", "min": 0, "max": 449.2054737819238 }, + "label_14": { "dtype": "number", "min": 0, "max": 518.5199748098399 }, + "label_15": { "dtype": "number", "min": 0, "max": 463.8811725137893 }, + "label_16": { "dtype": "number", "min": 0, "max": 514.2732060805772 }, + "label_17": { "dtype": "number", "min": 0, "max": 475.8006861523057 }, + "label_18": { "dtype": "number", "min": 0, "max": 499.0736746691703 }, + "label_19": { "dtype": "number", "min": 0, "max": 435.18154671948156 }, + "label_20": { "dtype": "number", "min": 0, "max": 503.95906901514456 }, + "label_21": { "dtype": "number", "min": 0, "max": 453.42442120690896 }, + "label_22": { "dtype": "number", "min": 0, "max": 500.483162019238 }, + "label_23": { "dtype": "number", "min": 0, "max": 468.18940999515536 }, + "label_24": { "dtype": "number", "min": 0, "max": 495.75003969620434 }, + "label_25": { "dtype": "number", "min": 0, "max": 480.72524952103413 }, + "label_26": { "dtype": "number", "min": 0, "max": 483.45096761731435 }, + "label_27": { "dtype": "number", "min": 0, "max": 440.95834852166774 }, + "label_28": { "dtype": "number", "min": 0, "max": 473.19789907159645 }, + "label_29": { "dtype": "number", "min": 0, "max": 458.5087523020914 }, + "label_30": { "dtype": "number", "min": 0, "max": 467.93109224233444 }, + "label_31": { "dtype": "number", "min": 0, "max": 471.2808751940178 }, + "label_32": { "dtype": "number", "min": 0, "max": 465.7011008815248 }, + "label_33": { "dtype": "number", "min": 0, "max": 481.9448468282673 }, + "label_34": { "dtype": "number", "min": 0, "max": 464.5309672255035 }, + "label_35": { "dtype": "number", "min": 0, "max": 444.791971246973 }, + "label_36": { "dtype": "number", "min": 0, "max": 451.3378977170124 }, + "label_37": { "dtype": "number", "min": 0, "max": 459.57942119350326 }, + "label_38": { "dtype": "number", "min": 0, "max": 451.17016603053224 }, + "label_39": { "dtype": "number", "min": 0, "max": 470.1975592491152 }, "label_40": { "dtype": "number", - "min": 6.9609311353376135, - "max": 676.9525074586147 - }, - "label_41": { - "dtype": "number", - "min": 75.97401514052241, - "max": 506.7948506427954 - } + "min": -2.250544608200542, + "max": 455.08975496996834 + }, + "label_41": { "dtype": "number", "min": 0, "max": 478.7628390581024 }, + "label_42": { "dtype": "number", "min": 0, "max": 586.6608405854927 }, + "label_43": { "dtype": "number", "min": 0, "max": 443.59360893451384 }, + "label_44": { "dtype": "number", "min": 0, "max": 575.2940255551233 }, + "label_45": { "dtype": "number", "min": 0, "max": 439.4650023423489 }, + "label_46": { "dtype": "number", "min": 0, "max": 572.6978570905426 }, + "label_47": { "dtype": "number", "min": 0, "max": 445.77737640752645 }, + "label_48": { "dtype": "number", "min": 0, "max": 577.5610386619726 }, + "label_49": { "dtype": "number", "min": 0, "max": 455.3685328800494 }, + "label_50": { "dtype": "number", "min": 0, "max": 587.5799889243797 }, + "label_51": { "dtype": "number", "min": 0, "max": 460.6591159738472 }, + "label_52": { "dtype": "number", "min": 0, "max": 595.3738279672924 }, + "label_53": { "dtype": "number", "min": 0, "max": 435.3837420626638 }, + "label_54": { "dtype": "number", "min": 0, "max": 603.0984481232397 }, + "label_55": { "dtype": "number", "min": 0, "max": 458.47111655979 }, + "label_56": { "dtype": "number", "min": 0, "max": 610.7793182445314 }, + "label_57": { "dtype": "number", "min": 0, "max": 475.95085629845164 }, + "label_58": { "dtype": "number", "min": 0, "max": 618.5882608713695 }, + "label_59": { "dtype": "number", "min": 0, "max": 489.3053477098461 }, + "label_60": { "dtype": "number", "min": 0, "max": 610.2477703388294 }, + "label_61": { "dtype": "number", "min": 0, "max": 439.0911629675826 }, + "label_62": { "dtype": "number", "min": 0, "max": 620.111633978391 }, + "label_63": { "dtype": "number", "min": 0, "max": 465.4726266037314 }, + "label_64": { "dtype": "number", "min": 0, "max": 626.9405786817175 }, + "label_65": { "dtype": "number", "min": 0, "max": 484.92308751687654 }, + "label_66": { "dtype": "number", "min": 0, "max": 633.3462004686493 }, + "label_67": { "dtype": "number", "min": 0, "max": 497.7592120455144 }, + "label_68": { "dtype": "number", "min": 0, "max": 622.5964250264554 }, + "label_69": { "dtype": "number", "min": 0, "max": 443.22402036828703 }, + "label_70": { "dtype": "number", "min": 0, "max": 634.5536612109757 }, + "label_71": { "dtype": "number", "min": 0, "max": 470.2350331722456 }, + "label_72": { "dtype": "number", "min": 0, "max": 642.2073345193608 }, + "label_73": { "dtype": "number", "min": 0, "max": 487.5833155606672 }, + "label_74": { "dtype": "number", "min": 0, "max": 648.6144713693823 }, + "label_75": { "dtype": "number", "min": 0, "max": 498.5309450346654 }, + "label_76": { "dtype": "number", "min": 0, "max": 632.6189459192858 }, + "label_77": { "dtype": "number", "min": 0, "max": 447.8460008763578 }, + "label_78": { "dtype": "number", "min": 0, "max": 644.54236495545 }, + "label_79": { "dtype": "number", "min": 0, "max": 472.5026761828674 }, + "label_80": { "dtype": "number", "min": 0, "max": 651.8202599792932 }, + "label_81": { "dtype": "number", "min": 0, "max": 486.6449571135753 }, + "label_82": { "dtype": "number", "min": 0, "max": 656.9313671078949 }, + "label_83": { "dtype": "number", "min": 0, "max": 497.0423817040591 } }, "outputs": { "label": { "dtype": "string", "min": 0, "max": 1, - "uniqueValues": ["Hello", "Bye"], - "legend": { "Hello": [1, 0], "Bye": [0, 1] } + "uniqueValues": ["Hello", "Good Bye"], + "legend": { "Hello": [1, 0], "Good Bye": [0, 1] } } }, "isNormalized": true, - "seriesShape": [51, 42] + "seriesShape": [30, 84] } diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js index 9239292d..2d0dfd39 100644 --- a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -19,7 +19,7 @@ let model; let isModelLoaded = false; let sequence = []; -let sequenceLength = 50; +let sequenceLength = 30; let curGesture; function preload() { From a7f8eea2ed4b43ed7206ba59a4ff1ae391dbad09 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 31 Jul 2025 07:20:38 -0400 Subject: [PATCH 44/47] chore: trainOptions, mm, barHeight --- .../neuralNetwork-sequence-weather-prediction/sketch.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js index 6015133a..8d7833b7 100644 --- a/examples/neuralNetwork-sequence-weather-prediction/sketch.js +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -56,10 +56,10 @@ function setup() { model.normalizeData(); // train right away - let options = { + let trainOptions = { epochs: 70, }; - model.train(options, finishedTraining); + model.train(trainOptions, finishedTraining); let predictBtn = select("#predictBtn"); predictBtn.center(); @@ -75,8 +75,7 @@ function draw() { if (state == "training") { text("Training...", width / 2, 160); } else if (state == "predicting") { - // XXX: let's add the unit here (mm? inches?) - text("Predicted rain: " + nf(predictedRain, 0, 1) + "", 320, 160); + text("Predicted rain: " + nf(predictedRain, 0, 1) + "mm", 320, 160); push(); textSize(predictedRain * 5 + 10); text("🌧️", width / 2, 100); @@ -123,7 +122,7 @@ function addNewData(newValues) { function drawBarGraph() { let barWidth = width / maxBars; - let maxDataValue = 35; + let maxDataValue = 78; let dryColor = color(235, 242, 255); let wetColor = color(0, 80, 255); From 8fd81c268e563375e92b1d18798de14de0db468d Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 31 Jul 2025 07:43:55 -0400 Subject: [PATCH 45/47] chore: friendlier error messages --- src/NeuralNetwork/Sequential/index.js | 24 +++++++++++ .../Sequential/sequentialData.js | 41 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 6e276c1a..2741a54b 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -59,6 +59,30 @@ class DIYSequential extends DiyNeuralNetwork { null, this.options ); + + // Validate sequence length against trained model + if (this.neuralNetwork && this.neuralNetwork.model && formatted_inputs.length > 0) { + const inputSequenceLength = formatted_inputs.length; + const numFeatures = Object.keys(meta.inputs).length; + + try { + // Get the expected input shape from the model + const expectedShape = this.neuralNetwork.model.inputs[0].shape; + const expectedSequenceLength = expectedShape[1]; // [batch, sequence, features] + + if (expectedSequenceLength !== null && inputSequenceLength !== expectedSequenceLength) { + throw new Error( + `🟪 ml5.js sequence length mismatch: The model was trained with sequences of length ${expectedSequenceLength}, but you provided a sequence of length ${inputSequenceLength}. Please provide exactly ${expectedSequenceLength} data points for prediction.` + ); + } + } catch (err) { + // If we can't get the model shape, provide a more general error + if (err.message.includes('sequence length mismatch')) { + throw err; + } + } + } + const normalized_inputs = this.neuralNetworkData.normalizePredictData( formatted_inputs, meta.inputs diff --git a/src/NeuralNetwork/Sequential/sequentialData.js b/src/NeuralNetwork/Sequential/sequentialData.js index f100a241..f4799f24 100644 --- a/src/NeuralNetwork/Sequential/sequentialData.js +++ b/src/NeuralNetwork/Sequential/sequentialData.js @@ -111,6 +111,47 @@ class SequentialData extends NeuralNetworkData { outputArr.push(ys); }); + // Validate sequence data structure before creating tensors + if (inputArr.length === 0) { + throw new Error( + "🟪 ml5.js sequence training error: No training data provided. Please add sequence data using model.addData() before training." + ); + } + + // Check if we have proper sequence structure (array of arrays) + const firstInput = inputArr[0]; + if (!Array.isArray(firstInput)) { + throw new Error( + "🟪 ml5.js sequence training error: Input data must be sequences (arrays of data points). Each training example should be an array of time steps. For example: [[{temp: 20}, {temp: 21}], [{temp: 22}, {temp: 23}]]" + ); + } + + // Validate all sequences have the same length + const sequenceLength = firstInput.length; + if (sequenceLength === 0) { + throw new Error( + "🟪 ml5.js sequence training error: Empty sequences provided. Each sequence must contain at least one data point." + ); + } + + const inconsistentSequence = inputArr.find(seq => seq.length !== sequenceLength); + if (inconsistentSequence) { + throw new Error( + `🟪 ml5.js sequence training error: All sequences must have the same length. Found sequences with lengths ${sequenceLength} and ${inconsistentSequence.length}. Please ensure all training sequences have exactly the same number of time steps.` + ); + } + + // Validate we have proper feature structure + const firstTimeStep = firstInput[0]; + const numFeatures = Object.keys(meta.inputs).length; + const timeStepFeatures = Object.keys(firstTimeStep || {}).length; + + if (timeStepFeatures !== numFeatures) { + throw new Error( + `🟪 ml5.js sequence training error: Feature count mismatch. Expected ${numFeatures} features per time step but found ${timeStepFeatures}. Make sure each time step in your sequences has the same features as defined in your model inputs.` + ); + } + const inputs = tf.tensor(inputArr); const outputs = tf.tensor(outputArr.flat(), [ From e84709fc6369f42dd3a460a3d2ea376e3e0cb2a3 Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 31 Jul 2025 08:11:25 -0400 Subject: [PATCH 46/47] fix: friendly error message normalize --- src/NeuralNetwork/Sequential/index.js | 21 +++++++++++++------ .../Sequential/sequentialData.js | 8 ++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js index 2741a54b..0f57d469 100644 --- a/src/NeuralNetwork/Sequential/index.js +++ b/src/NeuralNetwork/Sequential/index.js @@ -61,23 +61,30 @@ class DIYSequential extends DiyNeuralNetwork { ); // Validate sequence length against trained model - if (this.neuralNetwork && this.neuralNetwork.model && formatted_inputs.length > 0) { + if ( + this.neuralNetwork && + this.neuralNetwork.model && + formatted_inputs.length > 0 + ) { const inputSequenceLength = formatted_inputs.length; const numFeatures = Object.keys(meta.inputs).length; - + try { // Get the expected input shape from the model const expectedShape = this.neuralNetwork.model.inputs[0].shape; const expectedSequenceLength = expectedShape[1]; // [batch, sequence, features] - - if (expectedSequenceLength !== null && inputSequenceLength !== expectedSequenceLength) { + + if ( + expectedSequenceLength !== null && + inputSequenceLength !== expectedSequenceLength + ) { throw new Error( `🟪 ml5.js sequence length mismatch: The model was trained with sequences of length ${expectedSequenceLength}, but you provided a sequence of length ${inputSequenceLength}. Please provide exactly ${expectedSequenceLength} data points for prediction.` ); } } catch (err) { // If we can't get the model shape, provide a more general error - if (err.message.includes('sequence length mismatch')) { + if (err.message.includes("sequence length mismatch")) { throw err; } } @@ -95,7 +102,9 @@ class DIYSequential extends DiyNeuralNetwork { createMetaData() { // check if the data is empty if (this.neuralNetworkData.data.raw.length <= 0) { - throw new Error("Must add data before training!"); + throw new Error( + "🟪 ml5.js sequence training error: Must add data before training!" + ); } // this method does not get shape for images but instead for timesteps diff --git a/src/NeuralNetwork/Sequential/sequentialData.js b/src/NeuralNetwork/Sequential/sequentialData.js index f4799f24..e65bd15b 100644 --- a/src/NeuralNetwork/Sequential/sequentialData.js +++ b/src/NeuralNetwork/Sequential/sequentialData.js @@ -122,7 +122,7 @@ class SequentialData extends NeuralNetworkData { const firstInput = inputArr[0]; if (!Array.isArray(firstInput)) { throw new Error( - "🟪 ml5.js sequence training error: Input data must be sequences (arrays of data points). Each training example should be an array of time steps. For example: [[{temp: 20}, {temp: 21}], [{temp: 22}, {temp: 23}]]" + "🟪 ml5.js sequence training error: Have you run normalizeData() yet? Data must be normalized before training the model." ); } @@ -134,7 +134,9 @@ class SequentialData extends NeuralNetworkData { ); } - const inconsistentSequence = inputArr.find(seq => seq.length !== sequenceLength); + const inconsistentSequence = inputArr.find( + (seq) => seq.length !== sequenceLength + ); if (inconsistentSequence) { throw new Error( `🟪 ml5.js sequence training error: All sequences must have the same length. Found sequences with lengths ${sequenceLength} and ${inconsistentSequence.length}. Please ensure all training sequences have exactly the same number of time steps.` @@ -145,7 +147,7 @@ class SequentialData extends NeuralNetworkData { const firstTimeStep = firstInput[0]; const numFeatures = Object.keys(meta.inputs).length; const timeStepFeatures = Object.keys(firstTimeStep || {}).length; - + if (timeStepFeatures !== numFeatures) { throw new Error( `🟪 ml5.js sequence training error: Feature count mismatch. Expected ${numFeatures} features per time step but found ${timeStepFeatures}. Make sure each time step in your sequences has the same features as defined in your model inputs.` From 7776f28a1d0edf1902471625616f2b0c0f03c3af Mon Sep 17 00:00:00 2001 From: Mathew Ponon Date: Thu, 31 Jul 2025 08:31:54 -0400 Subject: [PATCH 47/47] chore: error for different lengths --- .../Sequential/sequentialUtils.js | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js index 04a3e1cd..ae4feb24 100644 --- a/src/NeuralNetwork/Sequential/sequentialUtils.js +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -224,7 +224,11 @@ class SequentialUtils { // Apply output filtering if we have object outputs and output labels are provided let filteredOutputs = yInputs; if (typeof yInputs === "object" && !Array.isArray(yInputs)) { - filteredOutputs = this.filterOutputsByLabels(yInputs, options, classOptions); + filteredOutputs = this.filterOutputsByLabels( + yInputs, + options, + classOptions + ); } return nnUtils.formatDataAsObject(filteredOutputs, outputLabels); @@ -241,21 +245,29 @@ class SequentialUtils { filterOutputsByLabels(yInputs, options = null, classOptions) { // Get output labels from options or classOptions let outputLabels = null; - - if (options && options.outputLabels && Array.isArray(options.outputLabels)) { + + if ( + options && + options.outputLabels && + Array.isArray(options.outputLabels) + ) { outputLabels = options.outputLabels; - } else if (classOptions && classOptions.outputs && Array.isArray(classOptions.outputs)) { + } else if ( + classOptions && + classOptions.outputs && + Array.isArray(classOptions.outputs) + ) { outputLabels = classOptions.outputs; } - + // If no output labels provided or not an array, return original data if (!outputLabels || !Array.isArray(outputLabels)) { return yInputs; } - + // Filter the object to keep only the specified keys const filteredObj = {}; - outputLabels.forEach(key => { + outputLabels.forEach((key) => { if (yInputs.hasOwnProperty(key)) { filteredObj[key] = yInputs[key]; } @@ -351,8 +363,9 @@ class SequentialUtils { zipArraySequence(arr1, arr2) { if (arr1.length !== arr2.length) { - console.error("arrays do not have the same length"); - return []; + throw new Error( + `🟪 ml5.js sequence training error: Your sequences have different lengths. Try using model.setFixedLength() to make them all the same size!\n\nExample:\nconst fixedData = model.setFixedLength(yourData, 10); // Makes all sequences 10 steps long\nmodel.addData(fixedData.inputs, fixedData.outputs);` + ); } return arr1.map((xs, idx) => {