diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html new file mode 100644 index 00000000..ec78e95b --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/index.html @@ -0,0 +1,28 @@ + + + + + + + + ml5.js neuralNetwork Hand Gesture Loading Pre-trained Model Example + + + + + + +
+

+ How to sign: Hello & Goodbye in ASL. +

+ + diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json new file mode 100644 index 00000000..136bcf90 --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,30,84],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.22.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,84,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[96,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin new file mode 100644 index 00000000..1b38b79e Binary files /dev/null and b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model.weights.bin differ diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json new file mode 100644 index 00000000..528797af --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/model/model_meta.json @@ -0,0 +1,105 @@ +{ + "inputUnits": [84], + "outputUnits": 2, + "inputs": { + "label_0": { "dtype": "number", "min": 0, "max": 479.19925350185474 }, + "label_1": { "dtype": "number", "min": 0, "max": 428.55538772969044 }, + "label_2": { "dtype": "number", "min": 0, "max": 493.1563637363407 }, + "label_3": { "dtype": "number", "min": 0, "max": 427.56692302194773 }, + "label_4": { "dtype": "number", "min": 0, "max": 499.91689098126307 }, + "label_5": { "dtype": "number", "min": 0, "max": 430.02650947613336 }, + "label_6": { "dtype": "number", "min": 0, "max": 496.3208721418956 }, + "label_7": { "dtype": "number", "min": 0, "max": 441.4487064271474 }, + "label_8": { "dtype": "number", "min": 0, "max": 491.1078639783216 }, + "label_9": { "dtype": "number", "min": 0, "max": 449.2829368661615 }, + "label_10": { "dtype": "number", "min": 0, "max": 509.81763121840737 }, + "label_11": { "dtype": "number", "min": 0, "max": 427.9468311472154 }, + "label_12": { "dtype": "number", "min": 0, "max": 519.7167764450397 }, + "label_13": { "dtype": "number", "min": 0, "max": 449.2054737819238 }, + "label_14": { "dtype": "number", "min": 0, "max": 518.5199748098399 }, + "label_15": { "dtype": "number", "min": 0, "max": 463.8811725137893 }, + "label_16": { "dtype": "number", "min": 0, "max": 514.2732060805772 }, + "label_17": { "dtype": "number", "min": 0, "max": 475.8006861523057 }, + "label_18": { "dtype": "number", "min": 0, "max": 499.0736746691703 }, + "label_19": { "dtype": "number", "min": 0, "max": 435.18154671948156 }, + "label_20": { "dtype": "number", "min": 0, "max": 503.95906901514456 }, + "label_21": { "dtype": "number", "min": 0, "max": 453.42442120690896 }, + "label_22": { "dtype": "number", "min": 0, "max": 500.483162019238 }, + "label_23": { "dtype": "number", "min": 0, "max": 468.18940999515536 }, + "label_24": { "dtype": "number", "min": 0, "max": 495.75003969620434 }, + "label_25": { "dtype": "number", "min": 0, "max": 480.72524952103413 }, + "label_26": { "dtype": "number", "min": 0, "max": 483.45096761731435 }, + "label_27": { "dtype": "number", "min": 0, "max": 440.95834852166774 }, + "label_28": { "dtype": "number", "min": 0, "max": 473.19789907159645 }, + "label_29": { "dtype": "number", "min": 0, "max": 458.5087523020914 }, + "label_30": { "dtype": "number", "min": 0, "max": 467.93109224233444 }, + "label_31": { "dtype": "number", "min": 0, "max": 471.2808751940178 }, + "label_32": { "dtype": "number", "min": 0, "max": 465.7011008815248 }, + "label_33": { "dtype": "number", "min": 0, "max": 481.9448468282673 }, + "label_34": { "dtype": "number", "min": 0, "max": 464.5309672255035 }, + "label_35": { "dtype": "number", "min": 0, "max": 444.791971246973 }, + "label_36": { "dtype": "number", "min": 0, "max": 451.3378977170124 }, + "label_37": { "dtype": "number", "min": 0, "max": 459.57942119350326 }, + "label_38": { "dtype": "number", "min": 0, "max": 451.17016603053224 }, + "label_39": { "dtype": "number", "min": 0, "max": 470.1975592491152 }, + "label_40": { + "dtype": "number", + "min": -2.250544608200542, + "max": 455.08975496996834 + }, + "label_41": { "dtype": "number", "min": 0, "max": 478.7628390581024 }, + "label_42": { "dtype": "number", "min": 0, "max": 586.6608405854927 }, + "label_43": { "dtype": "number", "min": 0, "max": 443.59360893451384 }, + "label_44": { "dtype": "number", "min": 0, "max": 575.2940255551233 }, + "label_45": { "dtype": "number", "min": 0, "max": 439.4650023423489 }, + "label_46": { "dtype": "number", "min": 0, "max": 572.6978570905426 }, + "label_47": { "dtype": "number", "min": 0, "max": 445.77737640752645 }, + "label_48": { "dtype": "number", "min": 0, "max": 577.5610386619726 }, + "label_49": { "dtype": "number", "min": 0, "max": 455.3685328800494 }, + "label_50": { "dtype": "number", "min": 0, "max": 587.5799889243797 }, + "label_51": { "dtype": "number", "min": 0, "max": 460.6591159738472 }, + "label_52": { "dtype": "number", "min": 0, "max": 595.3738279672924 }, + "label_53": { "dtype": "number", "min": 0, "max": 435.3837420626638 }, + "label_54": { "dtype": "number", "min": 0, "max": 603.0984481232397 }, + "label_55": { "dtype": "number", "min": 0, "max": 458.47111655979 }, + "label_56": { "dtype": "number", "min": 0, "max": 610.7793182445314 }, + "label_57": { "dtype": "number", "min": 0, "max": 475.95085629845164 }, + "label_58": { "dtype": "number", "min": 0, "max": 618.5882608713695 }, + "label_59": { "dtype": "number", "min": 0, "max": 489.3053477098461 }, + "label_60": { "dtype": "number", "min": 0, "max": 610.2477703388294 }, + "label_61": { "dtype": "number", "min": 0, "max": 439.0911629675826 }, + "label_62": { "dtype": "number", "min": 0, "max": 620.111633978391 }, + "label_63": { "dtype": "number", "min": 0, "max": 465.4726266037314 }, + "label_64": { "dtype": "number", "min": 0, "max": 626.9405786817175 }, + "label_65": { "dtype": "number", "min": 0, "max": 484.92308751687654 }, + "label_66": { "dtype": "number", "min": 0, "max": 633.3462004686493 }, + "label_67": { "dtype": "number", "min": 0, "max": 497.7592120455144 }, + "label_68": { "dtype": "number", "min": 0, "max": 622.5964250264554 }, + "label_69": { "dtype": "number", "min": 0, "max": 443.22402036828703 }, + "label_70": { "dtype": "number", "min": 0, "max": 634.5536612109757 }, + "label_71": { "dtype": "number", "min": 0, "max": 470.2350331722456 }, + "label_72": { "dtype": "number", "min": 0, "max": 642.2073345193608 }, + "label_73": { "dtype": "number", "min": 0, "max": 487.5833155606672 }, + "label_74": { "dtype": "number", "min": 0, "max": 648.6144713693823 }, + "label_75": { "dtype": "number", "min": 0, "max": 498.5309450346654 }, + "label_76": { "dtype": "number", "min": 0, "max": 632.6189459192858 }, + "label_77": { "dtype": "number", "min": 0, "max": 447.8460008763578 }, + "label_78": { "dtype": "number", "min": 0, "max": 644.54236495545 }, + "label_79": { "dtype": "number", "min": 0, "max": 472.5026761828674 }, + "label_80": { "dtype": "number", "min": 0, "max": 651.8202599792932 }, + "label_81": { "dtype": "number", "min": 0, "max": 486.6449571135753 }, + "label_82": { "dtype": "number", "min": 0, "max": 656.9313671078949 }, + "label_83": { "dtype": "number", "min": 0, "max": 497.0423817040591 } + }, + "outputs": { + "label": { + "dtype": "string", + "min": 0, + "max": 1, + "uniqueValues": ["Hello", "Good Bye"], + "legend": { "Hello": [1, 0], "Good Bye": [0, 1] } + } + }, + "isNormalized": true, + "seriesShape": [30, 84] +} diff --git a/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js new file mode 100644 index 00000000..2d0dfd39 --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture-load-model/sketch.js @@ -0,0 +1,158 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates loading a hand gesture classifier through + * ml5.neuralNetwork with the sequenceClassificationWithCNN task. + * This example has been trained with the ASL gestures for Hello and Goodbye. + * + * Reference to sign hello and goodbye in ASL: + * Hello: https://www.signasl.org/sign/hello + * Goodbye: https://www.signasl.org/sign/goodbye + */ + +let video; +let handPose; +let hands = []; +let model; +let isModelLoaded = false; + +let sequence = []; +let sequenceLength = 30; +let curGesture; + +function preload() { + // load the handPose model + handPose = ml5.handPose({ flipHorizontal: true }); +} + +function setup() { + let canvas = createCanvas(640, 480); + canvas.parent("canvasDiv"); + + video = createCapture(VIDEO, { flipped: true }); + video.size(640, 480); + video.hide(); + handPose.detectStart(video, gotHands); + + let options = { + task: "sequenceClassificationWithCNN", + }; + model = ml5.neuralNetwork(options); + + // setup the model files to load + let modelDetails = { + model: "model/model.json", + metadata: "model/model_meta.json", + weights: "model/model.weights.bin", + }; + + // load the model and call modelLoaded once finished + model.load(modelDetails, modelLoaded); +} + +function modelLoaded() { + console.log("Model loaded"); + isModelLoaded = true; +} + +function draw() { + image(video, 0, 0, width, height); + drawHands(); + textSize(16); + stroke(0); + fill(255); + + if (hands.length > 0) { + // hands in frame, add their keypoints to the sequence (input) + let handpoints = getKeypoints(["Left", "Right"]); + sequence.push(handpoints); + text( + "Move your hand(s) out of the frame after finishing the gesture", + 50, + 50 + ); + } else if (sequence.length > 0) { + // hands moved out of the frame, end of sequence + + // Sequence will have varying length at this point, depending on + // how long the hands were in frame - a line simplification algorithm + // (RDP) turns it into the fixed length the NN can work with. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA + let inputs = model.setFixedLength(sequence, sequenceLength); + + // start the classification + if (isModelLoaded) { + model.classify(inputs, gotResults); + } + // reset the sequence + sequence = []; + text("Classifying...", 50, 50); + } else if (curGesture == null) { + // on program start + text("Move your hand(s) into the frame to sign a gesture", 50, 50); + } else { + // after receiving a classification + text('Saw "' + curGesture + '"', 50, 50); + } +} + +// callback function for when the classification fininished +function gotResults(results, error) { + if (error) { + console.error(error); + return; + } + curGesture = results[0].label; +} + +// callback function for when handPose outputs data +function gotHands(results) { + hands = results; +} + +function drawHands() { + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + fill(0, 255, 0); + noStroke(); + circle(keypoint.x, keypoint.y, 5); + } + } +} + +// Return the tracked hand points as flattened array of 84 numbers +// for use as input to the neural network + +function getKeypoints(whichHands = ["Left", "Right"]) { + let keypoints = []; + // look for the left and right hand + for (let whichHand of whichHands) { + let found = false; + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + if (hand.handedness == whichHand) { + // and add the x and y numbers of each tracked keypoint + // to the array + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + keypoints.push(keypoint.x, keypoint.y); + } + found = true; + break; + } + } + if (!found) { + // if we don't find a right or a left hand, add 42 zeros + // to the keypoints array instead + for (let j = 0; j < 42; j++) { + keypoints.push(0); + } + } + } + return keypoints; +} diff --git a/examples/neuralNetwork-sequence-hand-gesture/index.html b/examples/neuralNetwork-sequence-hand-gesture/index.html new file mode 100644 index 00000000..8602322c --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture/index.html @@ -0,0 +1,24 @@ + + + + + + + + ml5.js neuralNetwork Hand Gesture Training and Saving Example + + + + + + +
+ + diff --git a/examples/neuralNetwork-sequence-hand-gesture/sketch.js b/examples/neuralNetwork-sequence-hand-gesture/sketch.js new file mode 100644 index 00000000..c9a162be --- /dev/null +++ b/examples/neuralNetwork-sequence-hand-gesture/sketch.js @@ -0,0 +1,200 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates training a hand gesture classifier through + * ml5.neuralNetwork with the sequenceClassificationWithCNN task. + */ + +let video; +let handPose; +let hands = []; +let model; + +let state = "training"; +let sequence = []; +let sequenceLength = 30; +let gestures = ["Gesture #1", "Gesture #2"]; +let counts = { "Gesture #1": 0, "Gesture #2": 0 }; +let curGesture = gestures[0]; + +let gesture1Button; +let gesture2Button; +let trainButton; + +function preload() { + // load the handPose model + handPose = ml5.handPose({ flipHorizontal: true }); +} + +function setup() { + let canvas = createCanvas(640, 480); + canvas.parent("canvasDiv"); + + video = createCapture(VIDEO, { flipped: true }); + video.size(640, 480); + video.hide(); + handPose.detectStart(video, gotHands); + + let options = { + outputs: ["label"], + task: "sequenceClassificationWithCNN", + debug: true, + learningRate: 0.001, // the default learning rate of 0.01 didn't converge for this use case (0.001 makes smaller steps each epoch) + }; + model = ml5.neuralNetwork(options); + + // setup the UI buttons for training + gesture1Button = createButton("Start recording " + gestures[0]); + gesture1Button.mousePressed(recordGesture1); + gesture2Button = createButton("Start recording " + gestures[1]); + gesture2Button.mousePressed(recordGesture2); + trainButton = createButton("Train and Save Model"); + trainButton.mousePressed(trainModel); +} + +function draw() { + image(video, 0, 0, width, height); + drawHands(); + + if (hands.length > 0) { + // hands in frame, add their keypoints to the sequence + let handpoints = getKeypoints(["Left", "Right"]); + sequence.push(handpoints); + + // This uses the RDP line simplification algorithm to make sure each + // input to the neural network has the same number of points. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA + + let rdp = model.setFixedLength(sequence, sequenceLength); + for (let i = 0; i < rdp.length - 1; i++) { + for (let j = 0; j < rdp[i].length; j += 2) { + stroke(255, 0, 0); + // hide lines to 0,0 + if (rdp[i][j] != 0 && rdp[i + 1][j] != 0) { + line(rdp[i][j], rdp[i][j + 1], rdp[i + 1][j], rdp[i + 1][j + 1]); + } + } + } + } else if (sequence.length > 0) { + // hands moved out of the frame, end of sequence + let inputs = model.setFixedLength(sequence, sequenceLength); + + if (state == "training") { + let outputs = { label: curGesture }; + model.addData(inputs, outputs); + counts[curGesture]++; + } else if (state == "classifying") { + model.classify(inputs, gotResults); + } + // reset the sequence + sequence = []; + } + + // display current state + textSize(16); + stroke(0); + fill(255); + if (state == "training" && sequence.length == 0) { + text("Move your hand(s) into the frame to record " + curGesture, 50, 50); + } else if (state == "training") { + text("Move your hand(s) out of the frame to finish " + curGesture, 50, 50); + } else if (state == "classifying" && curGesture == null) { + text("Try a trained gesture", 50, 50); + } else if (state == "classifying" && curGesture) { + text("Saw " + curGesture, 50, 50); + } + + // show how many times each gesture was recorded + if (state == "training") { + for (let i = 0; i < gestures.length; i++) { + text( + gestures[i] + ": " + counts[gestures[i]], + 50, + height - 50 - (gestures.length - i - 1) * 20 + ); + } + } +} + +function trainModel() { + model.normalizeData(); + let options = { + epochs: 50, + }; + model.train(options, finishedTraining); + + gesture1Button.attribute("disabled", true); + gesture2Button.attribute("disabled", true); + trainButton.attribute("disabled", true); +} + +function finishedTraining() { + state = "classifying"; + model.save(); + curGesture = null; +} + +// callback function for when the classification fininished +function gotResults(results) { + curGesture = results[0].label; +} + +// callback function for when handPose outputs data +function gotHands(results) { + hands = results; +} + +function drawHands() { + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + fill(0, 255, 0); + noStroke(); + circle(keypoint.x, keypoint.y, 5); + } + } +} + +// Return the tracked hand points as flattened array of 84 numbers +// for use as input to the neural network + +function getKeypoints(whichHands = ["Left", "Right"]) { + let keypoints = []; + // look for the left and right hand + for (let whichHand of whichHands) { + let found = false; + for (let i = 0; i < hands.length; i++) { + let hand = hands[i]; + if (hand.handedness == whichHand) { + // and add the x and y numbers of each tracked keypoint + // to the array + for (let j = 0; j < hand.keypoints.length; j++) { + let keypoint = hand.keypoints[j]; + keypoints.push(keypoint.x, keypoint.y); + } + found = true; + break; + } + } + if (!found) { + // if we don't find a right or a left hand, add 42 zeros + // to the keypoints array instead + for (let j = 0; j < 42; j++) { + keypoints.push(0); + } + } + } + return keypoints; +} + +function recordGesture1() { + curGesture = gestures[0]; +} + +function recordGesture2() { + curGesture = gestures[1]; +} diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html new file mode 100644 index 00000000..aa7b2a8c --- /dev/null +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/index.html @@ -0,0 +1,30 @@ + + + + + + + + ml5.js neuralNetwork Gesture Classifier Example with sequenceClassificationWithCNN + + + + + + +
+
+
+ + + +
+ + diff --git a/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js new file mode 100644 index 00000000..069a620c --- /dev/null +++ b/examples/neuralNetwork-sequence-mouse-gesture-rdp/sketch.js @@ -0,0 +1,121 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates how to train your own mouse gesture classifier + * through ml5.neuralNetwork with the sequenceClassificationWithCNN task. + */ + +let model; + +let state = "training"; +let curShape = "circle"; +let sequence = []; +let sequenceLength = 30; + +function setup() { + let canvas = createCanvas(600, 400); + canvas.parent("canvasDiv"); + + let options = { + inputs: ["x", "y"], + outputs: ["label"], + task: "sequenceClassificationWithCNN", + debug: true, + learningRate: 0.005, // smaller learning rate converged better + }; + model = ml5.neuralNetwork(options); + + select("#collCirclesBtn").mouseClicked(collectCircles); + select("#collSquaresBtn").mouseClicked(collectSquares); + select("#trainBtn").mouseClicked(trainModel); +} + +function draw() { + background(220); + + for (let i = 0; i < sequence.length - 1; i++) { + line(sequence[i].x, sequence[i].y, sequence[i + 1].x, sequence[i + 1].y); + } + + // Sequence will have varying length at this point, depending on + // how long the hands were in frame - a line simplification algorithm + // (RDP) turns it into the fixed length the NN can work with. + // For more information about RDP, see: + // https://www.youtube.com/watch?v=ZCXkvwLxBrA + + let rdp = model.setFixedLength(sequence, sequenceLength); + for (let i = 0; i < rdp.length; i++) { + fill(255); + rect(rdp[i].x - 3, rdp[i].y - 3, 6, 6); + } + + // display current state + textSize(16); + fill(0); + if (state == "training") { + text("Now collecting " + curShape + "s", 50, 50); + } else if (state == "classifying" && curShape == null) { + text("Training finished. Draw again to see the classification in action.", 50, 50); + } else if (state == "classifying") { + text("Saw a " + curShape, 50, 50); + } +} + +function mousePressed() { + if (mouseX >= 0 && mouseX < width && mouseY >= 0 && mouseY < height) { + sequence.push({ x: mouseX, y: mouseY }); + } +} + +function mouseDragged() { + if (mouseX >= 0 && mouseX < width && mouseY >= 0 && mouseY < height) { + sequence.push({ x: mouseX, y: mouseY }); + } +} + +function mouseReleased() { + if (sequence.length == 0) return; + + if (state == "training") { + let inputs = model.setFixedLength(sequence, sequenceLength); + let outputs = { label: curShape }; + model.addData(inputs, outputs); + } else if (state == "classifying") { + let inputs = model.setFixedLength(sequence, sequenceLength); + model.classify(inputs, gotResults); + } + // reset the sequence + sequence = []; +} + +function trainModel() { + model.normalizeData(); + + let options = { + epochs: 40, + }; + model.train(options, finishedTraining); + + select("#collCirclesBtn").attribute("disabled", true); + select("#collSquaresBtn").attribute("disabled", true); + select("#trainBtn").attribute("disabled", true); +} + +function finishedTraining() { + state = "classifying"; + curShape = null; +} + +function gotResults(results) { + curShape = results[0].label; +} + +function collectCircles() { + curShape = "circle"; +} + +function collectSquares() { + curShape = "square"; +} diff --git a/examples/neuralNetwork-sequence-weather-prediction/index.html b/examples/neuralNetwork-sequence-weather-prediction/index.html new file mode 100644 index 00000000..402d054e --- /dev/null +++ b/examples/neuralNetwork-sequence-weather-prediction/index.html @@ -0,0 +1,27 @@ + + + + + + + + ml5.js neuralNetwork Weather Prediction Example + + + + + + +
+ +
+ + + diff --git a/examples/neuralNetwork-sequence-weather-prediction/sketch.js b/examples/neuralNetwork-sequence-weather-prediction/sketch.js new file mode 100644 index 00000000..8d7833b7 --- /dev/null +++ b/examples/neuralNetwork-sequence-weather-prediction/sketch.js @@ -0,0 +1,145 @@ +/* + * 👋 Hello! This is an ml5.js example made and shared with ❤️. + * Learn more about the ml5.js project: https://ml5js.org/ + * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md + * + * This example demonstrates loading JSON data and training a Weather Predictor + * through ml5.neuralNetwork with the sequenceRegression task. + * XXX: let us know what this dataset is from/about + */ + +let model; +let data; +let features = [ + "temperature", + "humidity", + "wind_speed", + "pressure", + "precipitation", +]; +let sequenceLength = 10; // NN looks at 10 data points at a time + +let state = "training"; +let predictedRain = 0; +let maxBars = 12; +let graphValues = []; + +function preload() { + data = loadJSON("weather_data.json"); +} + +function setup() { + let canvas = createCanvas(640, 400); + canvas.parent("canvasDiv"); + + let options = { + task: "sequenceRegression", + debug: true, + learningRate: 0.0075, // smaller learning rate helps here + inputs: features, + outputs: features, + }; + model = ml5.neuralNetwork(options); + + // the JSON file has the actual data in a "data" property + data = data.data; + + // step through the data (in a "sliding window" way) and break it into + // sequences of 10 data points, which is what the NN will see at a time + for (let i = 0; i <= data.length - sequenceLength - 1; i++) { + let inputs = data.slice(i, i + sequenceLength); + let outputs = data[i + sequenceLength]; + + console.log("Sequence for training", inputs, outputs); + model.addData(inputs, outputs); + } + model.normalizeData(); + + // train right away + let trainOptions = { + epochs: 70, + }; + model.train(trainOptions, finishedTraining); + + let predictBtn = select("#predictBtn"); + predictBtn.center(); + predictBtn.mouseClicked(predictData); +} + +function draw() { + background(220); + noStroke(); + textSize(20); + textAlign(CENTER); + + if (state == "training") { + text("Training...", width / 2, 160); + } else if (state == "predicting") { + text("Predicted rain: " + nf(predictedRain, 0, 1) + "mm", 320, 160); + push(); + textSize(predictedRain * 5 + 10); + text("🌧️", width / 2, 100); + pop(); + } + drawBarGraph(); +} + +function finishedTraining() { + state = "predicting"; +} + +function predictData() { + // take the last 10 data points to predict the next + let inputs = data.slice(-sequenceLength); + console.log("Sequence for prediction", inputs); + model.predict(inputs, gotResults); +} + +function gotResults(results) { + predictedRain = results[4].value; + + // add result to the bar graph + graphValues.push(results[4].value); + if (graphValues.length > maxBars) { + graphValues.shift(); + } + + // optional: add predicted result to the dataset, so that it will be + // considered in further predictions (going forward in time) + addNewData(results); +} + +function addNewData(newValues) { + data.push({ + date: "One more (synthetic) hour", + temperature: newValues[0].value, + humidity: newValues[1].value, + wind_speed: newValues[2].value, + pressure: newValues[3].value, + precipitation: newValues[4].value, + }); +} + +function drawBarGraph() { + let barWidth = width / maxBars; + let maxDataValue = 78; + + let dryColor = color(235, 242, 255); + let wetColor = color(0, 80, 255); + + for (let i = 0; i < graphValues.length && i < maxBars; i++) { + let barHeight = map(graphValues[i], 0, maxDataValue, 0, height - 180); + let x = i * barWidth; + let y = height - barHeight - 20; + + // interpolate color based on rainfall amount + let lerpAmt = constrain(graphValues[i] / maxDataValue, 0, 1); + let barColor = lerpColor(dryColor, wetColor, lerpAmt); + + push(); + fill(barColor, 100, 255 - barColor); + stroke(100); + rect(x + 5, y, barWidth - 10, barHeight); + pop(); + } +} diff --git a/examples/neuralNetwork-sequence-weather-prediction/weather_data.json b/examples/neuralNetwork-sequence-weather-prediction/weather_data.json new file mode 100644 index 00000000..a45429a3 --- /dev/null +++ b/examples/neuralNetwork-sequence-weather-prediction/weather_data.json @@ -0,0 +1,196 @@ +{ + "data": [ + { + "date": "2024-08-01T00:00:00Z", + "temperature": 28.0, + "humidity": 50, + "wind_speed": 3.0, + "pressure": 1015, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T01:00:00Z", + "temperature": 27.5, + "humidity": 52, + "wind_speed": 4.0, + "pressure": 1014, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T02:00:00Z", + "temperature": 27.0, + "humidity": 55, + "wind_speed": 5.0, + "pressure": 1013, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T03:00:00Z", + "temperature": 26.5, + "humidity": 60, + "wind_speed": 6.0, + "pressure": 1012, + "precipitation": 2.0 + }, + { + "date": "2024-08-01T04:00:00Z", + "temperature": 26.0, + "humidity": 65, + "wind_speed": 8.0, + "pressure": 1010, + "precipitation": 5.0 + }, + { + "date": "2024-08-01T05:00:00Z", + "temperature": 25.5, + "humidity": 70, + "wind_speed": 10.0, + "pressure": 1008, + "precipitation": 10.0 + }, + { + "date": "2024-08-01T06:00:00Z", + "temperature": 25.0, + "humidity": 75, + "wind_speed": 12.0, + "pressure": 1006, + "precipitation": 15.0 + }, + { + "date": "2024-08-01T07:00:00Z", + "temperature": 24.5, + "humidity": 80, + "wind_speed": 14.0, + "pressure": 1004, + "precipitation": 20.0 + }, + { + "date": "2024-08-01T08:00:00Z", + "temperature": 24.0, + "humidity": 85, + "wind_speed": 15.0, + "pressure": 1002, + "precipitation": 25.0 + }, + { + "date": "2024-08-01T09:00:00Z", + "temperature": 23.5, + "humidity": 90, + "wind_speed": 17.0, + "pressure": 1000, + "precipitation": 30.0 + }, + { + "date": "2024-08-01T10:00:00Z", + "temperature": 23.0, + "humidity": 95, + "wind_speed": 20.0, + "pressure": 998, + "precipitation": 35.0 + }, + { + "date": "2024-08-01T11:00:00Z", + "temperature": 24.0, + "humidity": 85, + "wind_speed": 10.0, + "pressure": 1005, + "precipitation": 10.0 + }, + { + "date": "2024-08-01T12:00:00Z", + "temperature": 25.0, + "humidity": 75, + "wind_speed": 7.0, + "pressure": 1010, + "precipitation": 5.0 + }, + { + "date": "2024-08-01T13:00:00Z", + "temperature": 26.0, + "humidity": 65, + "wind_speed": 5.0, + "pressure": 1013, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T14:00:00Z", + "temperature": 27.0, + "humidity": 60, + "wind_speed": 4.0, + "pressure": 1015, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T15:00:00Z", + "temperature": 28.0, + "humidity": 50, + "wind_speed": 3.0, + "pressure": 1018, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T16:00:00Z", + "temperature": 27.0, + "humidity": 55, + "wind_speed": 4.0, + "pressure": 1015, + "precipitation": 0.0 + }, + { + "date": "2024-08-01T17:00:00Z", + "temperature": 26.0, + "humidity": 60, + "wind_speed": 5.0, + "pressure": 1012, + "precipitation": 1.0 + }, + { + "date": "2024-08-01T18:00:00Z", + "temperature": 25.0, + "humidity": 70, + "wind_speed": 7.0, + "pressure": 1009, + "precipitation": 5.0 + }, + { + "date": "2024-08-01T19:00:00Z", + "temperature": 24.0, + "humidity": 80, + "wind_speed": 10.0, + "pressure": 1005, + "precipitation": 10.0 + }, + { + "date": "2024-08-01T20:00:00Z", + "temperature": 23.0, + "humidity": 90, + "wind_speed": 12.0, + "pressure": 1002, + "precipitation": 15.0 + }, + { + "date": "2024-08-01T21:00:00Z", + "temperature": 22.0, + "humidity": 95, + "wind_speed": 15.0, + "pressure": 999, + "precipitation": 20.0 + }, + { + "date": "2024-08-01T22:00:00Z", + "temperature": 21.0, + "humidity": 98, + "wind_speed": 18.0, + "pressure": 995, + "precipitation": 25.0 + }, + { + "date": "2024-08-01T23:00:00Z", + "temperature": 20.0, + "humidity": 100, + "wind_speed": 20.0, + "pressure": 992, + "precipitation": 30.0 + } + ] +} diff --git a/src/NeuralNetwork/Sequential/index.js b/src/NeuralNetwork/Sequential/index.js new file mode 100644 index 00000000..0f57d469 --- /dev/null +++ b/src/NeuralNetwork/Sequential/index.js @@ -0,0 +1,263 @@ +import * as tf from "@tensorflow/tfjs"; +import { DiyNeuralNetwork } from ".."; + +import callCallback from "../../utils/callcallback"; +import setBackend from "../../utils/setBackend"; + +import seqUtils from "./sequentialUtils"; +import SequentialData from "./sequentialData"; +import { createSeqLayers } from "./seqLayers"; + +// call an extension of DIY Neural Network as a new class, override select methods +// which are seen below: +class DIYSequential extends DiyNeuralNetwork { + constructor(options, callback) { + super( + { + ...options, + neuralNetworkData: null, + }, + callback + ); + // call all options set in the this class which is the default, extra option for dataMode + this.options = { ...this.options, ...(options || {}) }; + + this.neuralNetworkData = + this.options.neuralNetworkData || new SequentialData(); + + this.init = this.init.bind(this); + this.ready = callCallback(this.init(), callback); + } + + async init() { + await tf.ready(); + if (this.options.dataUrl) { + await this.loadDataFromUrl(this.options.dataUrl); + } else if (this.options.modelUrl) { + await this.load(this.options.modelUrl); + } + return this; + } + + addData(xInputs, yInputs, options = null) { + // 1. verify format between the three possible types of xinputs + const xs = seqUtils.verifyAndFormatInputs(xInputs, options, this.options); + + // 2. format the yInput - same logic as NN class + const ys = seqUtils.verifyAndFormatOutputs(yInputs, options, this.options); + + // 3. add data to raw + this.neuralNetworkData.addData(xs, ys); + } + + formatInputsForPredictionAll(_input) { + const { meta } = this.neuralNetworkData; + const inputHeaders = Object.keys(meta.inputs); + + const formatted_inputs = seqUtils.verifyAndFormatInputs( + _input, + null, + this.options + ); + + // Validate sequence length against trained model + if ( + this.neuralNetwork && + this.neuralNetwork.model && + formatted_inputs.length > 0 + ) { + const inputSequenceLength = formatted_inputs.length; + const numFeatures = Object.keys(meta.inputs).length; + + try { + // Get the expected input shape from the model + const expectedShape = this.neuralNetwork.model.inputs[0].shape; + const expectedSequenceLength = expectedShape[1]; // [batch, sequence, features] + + if ( + expectedSequenceLength !== null && + inputSequenceLength !== expectedSequenceLength + ) { + throw new Error( + `🟪 ml5.js sequence length mismatch: The model was trained with sequences of length ${expectedSequenceLength}, but you provided a sequence of length ${inputSequenceLength}. Please provide exactly ${expectedSequenceLength} data points for prediction.` + ); + } + } catch (err) { + // If we can't get the model shape, provide a more general error + if (err.message.includes("sequence length mismatch")) { + throw err; + } + } + } + + const normalized_inputs = this.neuralNetworkData.normalizePredictData( + formatted_inputs, + meta.inputs + ); + const output = tf.tensor(normalized_inputs); + + return output; + } + + createMetaData() { + // check if the data is empty + if (this.neuralNetworkData.data.raw.length <= 0) { + throw new Error( + "🟪 ml5.js sequence training error: Must add data before training!" + ); + } + + // this method does not get shape for images but instead for timesteps + const { inputs } = this.options; + + let inputShape; + if (typeof inputs === "number") { + inputShape = inputs; + } else if (Array.isArray(inputs) && inputs.length > 0) { + inputShape = inputs.length; // will be fed into the tensors later + } + + this.neuralNetworkData.createMetadata(inputShape); + } + + addDefaultLayers() { + let layers; + + const seqLayers = createSeqLayers( + this.neuralNetworkData.meta.seriesShape, + this.options.hiddenUnits, + this.numberOfClasses // For output units if needed + ); + + const task = this.options.task; + let taskConditions = task; + switch (taskConditions) { + case "sequenceRegression": + layers = seqLayers.regression; + return this.createNetworkLayers(layers); + + case "sequenceClassification": + layers = seqLayers.classification; + return this.createNetworkLayers(layers); + + case "sequenceClassificationWithCNN": + layers = seqLayers.classificationWithCNN; + return this.createNetworkLayers(layers); + + case "sequenceRegressionWithCNN": + layers = seqLayers.regressionWithCNN; + return this.createNetworkLayers(layers); + + default: + console.error( + "Error: Task is unknown. Check documentation for a list of available sequence tasks. Defaulting to sequenceRegression Tasks" + ); + layers = seqLayers.default; + return this.createNetworkLayers(layers); + } + } + + // included here to fix non convergence issue + compile() { + const LEARNING_RATE = this.options.learningRate; + + let options = {}; + + if ( + this.options.task === "sequenceClassification" || + this.options.task === "sequenceClassificationWithCNN" + ) { + options = { + loss: "categoricalCrossentropy", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; + } else if ( + this.options.task === "sequenceRegression" || + this.options.task === "sequenceRegressionWithCNN" + ) { + options = { + loss: "meanSquaredError", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; + } else { + // if no task given - must be in NN class instead of this + options = { + loss: "meanSquaredError", + optimizer: tf.train.adam, + metrics: ["accuracy"], + }; + } + + options.optimizer = options.optimizer + ? this.neuralNetwork.setOptimizerFunction( + LEARNING_RATE, + options.optimizer + ) + : this.neuralNetwork.setOptimizerFunction(LEARNING_RATE, tf.train.sgd); + + this.neuralNetwork.compile(options); + + // if debug mode is true, then show the model summary + if (this.options.debug) { + this.neuralNetworkVis.modelSummary( + { + name: "Model Summary", + }, + this.neuralNetwork.model + ); + } + } + + // RDP algorithm + setFixedLength(coordinates, targetPointCount) { + const maxEpsilon = int(coordinates.length / 2); + return seqUtils.setFixedLength(coordinates, targetPointCount, maxEpsilon); + } + + getSlidingWindow(data, featureKeys, targetKeys, batchLength = null) { + this.featureKeys = featureKeys; + + if (batchLength == null) { + this.batchLength = int(data.length * 0.2); // set batchlength as a fraction of the total + } else if (batchLength >= data.length) { + throw new Error("batchLength must be smaller than total length of data"); + } else { + this.batchLength = batchLength; + } + + return seqUtils.createSlidingWindowData( + data, + this.batchLength, + this.featureKeys, + targetKeys + ); + } + + getSampleWindow(data) { + if (!this.batchLength || !this.featureKeys) { + throw new Error( + "Your data must be formated through the slidingWindow method first!" + ); + } + return seqUtils.getLatestSequence(data, this.batchLength, this.featureKeys); + } + + /** + * getData + * Returns the raw data that was added through addData + * @returns {Object} - Object with xs, ys, and raw properties + */ + getData() { + const rawData = this.neuralNetworkData.data.raw; + + return { + xs: rawData.map((item) => item.xs), + ys: rawData.map((item) => item.ys), + raw: rawData, + }; + } +} + +export default DIYSequential; //export for taskSelection diff --git a/src/NeuralNetwork/Sequential/seqLayers.js b/src/NeuralNetwork/Sequential/seqLayers.js new file mode 100644 index 00000000..b3b30d54 --- /dev/null +++ b/src/NeuralNetwork/Sequential/seqLayers.js @@ -0,0 +1,152 @@ +export const createSeqLayers = ( + seriesShape, + hiddenUnits, + outputUnits = null +) => { + return { + classificationWithCNN: [ + { + type: "conv1d", + filters: 8, + kernelSize: 3, + activation: "relu", + inputShape: seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "conv1d", + filters: 16, + kernelSize: 3, + activation: "relu", + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "flatten", + }, + { + type: "dense", + units: hiddenUnits, + activation: "relu", + }, + { + type: "dense", + units: outputUnits, + activation: "softmax", + }, + ], + classification: [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: seriesShape, + returnSequences: true, + }, + { + type: "lstm", + units: 8, + activation: "relu", + returnSequences: false, + }, + { + type: "dense", + units: hiddenUnits, + activation: "relu", + }, + { + type: "dense", + units: outputUnits, + activation: "softmax", + }, + ], + regressionWithCNN: [ + { + type: "conv1d", + filters: 8, + kernelSize: 3, + activation: "relu", + inputShape: seriesShape, + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "conv1d", + filters: 16, + kernelSize: 3, + activation: "relu", + }, + { + type: "maxPooling1d", + poolSize: 2, + }, + { + type: "flatten", + }, + { + type: "dense", + units: hiddenUnits, + activation: "relu", + }, + { + type: "dense", + units: outputUnits, + activation: "sigmoid", + }, + ], + regression: [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: seriesShape, + returnSequences: true, + }, + { + type: "lstm", + units: 8, + activation: "relu", + }, + { + type: "dense", + units: hiddenUnits, + activation: "relu", + }, + { + type: "dense", + units: outputUnits, + activation: "sigmoid", + }, + ], + default: [ + { + type: "lstm", + units: 16, + activation: "relu", + inputShape: seriesShape, + }, + { + type: "lstm", + units: 8, + activation: "relu", + }, + { + type: "dense", + units: hiddenUnits, + activation: "relu", + }, + { + type: "dense", + units: outputUnits, + activation: "sigmoid", + }, + ], + }; +}; diff --git a/src/NeuralNetwork/Sequential/sequentialData.js b/src/NeuralNetwork/Sequential/sequentialData.js new file mode 100644 index 00000000..e65bd15b --- /dev/null +++ b/src/NeuralNetwork/Sequential/sequentialData.js @@ -0,0 +1,404 @@ +import * as tf from "@tensorflow/tfjs"; +import axios from "axios"; +import { saveBlob } from "../../utils/io"; +import modelLoader from "../../utils/modelLoader"; +import nnUtils from "../NeuralNetworkUtils"; +import seqUtils from "./sequentialUtils"; + +import NeuralNetworkData from "../NeuralNetworkData"; + +class SequentialData extends NeuralNetworkData { + constructor() { + super(); + } + + /** + * getDTypesFromData + * gets the data types of the data we're using + * important for handling oneHot + * @private + * @void - updates this.meta + */ + getDTypesFromData() { + const meta = { + ...this.meta, + inputs: {}, + outputs: {}, + }; + + const sample = this.data.raw[0]; + + // consistent dTypes have already been checked at add data + const xs = Object.keys(sample.xs[0]); // since time series data is in form of array + const ys = Object.keys(sample.ys); + xs.forEach((prop) => { + meta.inputs[prop] = { + dtype: nnUtils.getDataType(sample.xs[0][prop]), + }; + }); + + ys.forEach((prop) => { + meta.outputs[prop] = { + dtype: nnUtils.getDataType(sample.ys[prop]), + }; + }); + + this.meta = meta; + } + + /** + * get back the min and max of each label + * @private + * @param {Object} inputOrOutputMeta + * @param {"xs" | "ys"} xsOrYs + * @return {Object} + */ + getInputMetaStats(inputOrOutputMeta, xsOrYs) { + const inputMeta = Object.assign({}, inputOrOutputMeta); + + Object.keys(inputMeta).forEach((k) => { + if (inputMeta[k].dtype === "string") { + inputMeta[k].min = 0; + inputMeta[k].max = 1; + } else if (inputMeta[k].dtype === "number") { + let dataAsArray; + if (xsOrYs === "ys") { + dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]); + } else if (xsOrYs === "xs") { + dataAsArray = this.data.raw.flatMap((item) => + item[xsOrYs].map((obj) => obj[k]) + ); + } + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } else if (inputMeta[k].dtype === "array") { + const dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]).flat(); + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } + }); + + return inputMeta; + } + + /** + * convertRawToTensors + * converts array of {xs, ys} to tensors + * @param {*} dataRaw + * + * @return {{ inputs: tf.Tensor, outputs: tf.Tensor }} + */ + convertRawToTensors(dataRaw) { + const meta = Object.assign({}, this.meta); + const dataLength = dataRaw.length; + + return tf.tidy(() => { + const inputArr = []; + const outputArr = []; + + dataRaw.forEach((row) => { + // get xs + const xs = row.xs; + inputArr.push(xs); + + // get ys + const ys = Object.keys(meta.outputs) + .map((k) => { + return row.ys[k]; + }) + .flat(); + + outputArr.push(ys); + }); + + // Validate sequence data structure before creating tensors + if (inputArr.length === 0) { + throw new Error( + "🟪 ml5.js sequence training error: No training data provided. Please add sequence data using model.addData() before training." + ); + } + + // Check if we have proper sequence structure (array of arrays) + const firstInput = inputArr[0]; + if (!Array.isArray(firstInput)) { + throw new Error( + "🟪 ml5.js sequence training error: Have you run normalizeData() yet? Data must be normalized before training the model." + ); + } + + // Validate all sequences have the same length + const sequenceLength = firstInput.length; + if (sequenceLength === 0) { + throw new Error( + "🟪 ml5.js sequence training error: Empty sequences provided. Each sequence must contain at least one data point." + ); + } + + const inconsistentSequence = inputArr.find( + (seq) => seq.length !== sequenceLength + ); + if (inconsistentSequence) { + throw new Error( + `🟪 ml5.js sequence training error: All sequences must have the same length. Found sequences with lengths ${sequenceLength} and ${inconsistentSequence.length}. Please ensure all training sequences have exactly the same number of time steps.` + ); + } + + // Validate we have proper feature structure + const firstTimeStep = firstInput[0]; + const numFeatures = Object.keys(meta.inputs).length; + const timeStepFeatures = Object.keys(firstTimeStep || {}).length; + + if (timeStepFeatures !== numFeatures) { + throw new Error( + `🟪 ml5.js sequence training error: Feature count mismatch. Expected ${numFeatures} features per time step but found ${timeStepFeatures}. Make sure each time step in your sequences has the same features as defined in your model inputs.` + ); + } + + const inputs = tf.tensor(inputArr); + + const outputs = tf.tensor(outputArr.flat(), [ + dataLength, + meta.outputUnits, + ]); + + return { + inputs, + outputs, + }; + }); + } + + /** + * normalize the dataRaw input + * @return {Array} + */ + normalizeDataRaw() { + const normXs = this.normalizeInputData(this.meta.inputs, "xs"); + const normYs = this.normalizeInputData(this.meta.outputs, "ys"); + const normalizedData = seqUtils.zipArraySequence(normXs, normYs); + + return normalizedData; + } + + /** + * @param {Object} inputOrOutputMeta + * @param {"xs" | "ys"} xsOrYs + * @return {Array} + */ + normalizeInputData(inputOrOutputMeta, xsOrYs) { + const dataRaw = this.data.raw; + + // the data length + const dataLength = dataRaw.length; + + // the copy of the inputs.meta[inputOrOutput] + const inputMeta = Object.assign({}, inputOrOutputMeta); + + // normalized output object + const normalized = {}; + Object.keys(inputMeta).forEach((k) => { + // get the min and max values + const options = { + min: inputMeta[k].min, + max: inputMeta[k].max, + }; + + // depending on the input type, normalize accordingly + if (inputMeta[k].dtype === "string") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + options.legend = inputMeta[k].legend; + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "number") { + let dataAsArray; + if (xsOrYs === "ys") { + dataAsArray = this.data.raw.map((item) => item[xsOrYs][k]); + } else if (xsOrYs === "xs") { + dataAsArray = this.data.raw.flatMap((item) => + item[xsOrYs].map((obj) => obj[k]) + ); + } + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "array") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + normalized[k] = dataAsArray.map((item) => + this.normalizeArray(item, options) + ); + } + }); + + let output; + if (xsOrYs == "ys") { + output = [...new Array(dataLength).fill(null)].map((item, idx) => { + const row = { + [xsOrYs]: {}, + }; + + Object.keys(inputMeta).forEach((k) => { + row[xsOrYs][k] = normalized[k][idx]; + }); + + return row; + }); + } else if (xsOrYs == "xs") { + // reshape array - already ready for tensorconversion + const features = Object.keys(inputMeta); + const feature_length = features.length; + + const seriesStep = dataRaw[0]["xs"].length; + + const batch = normalized[features[0]].length / seriesStep; + + this.meta.seriesShape = [seriesStep, feature_length]; + let zipped = []; + + // zip arrays before reshaping + for (let idx = 0; idx < seriesStep * feature_length * batch; idx++) { + features.forEach((k) => { + zipped.push(normalized[k][idx]); + }); + } + + // reshaping + output = seqUtils.reshapeTo3DArray(zipped, [ + batch, + seriesStep, + feature_length, + ]); + } + + return output; + } + + normalizePredictData(dataRaw, inputOrOutputMeta) { + const inputMeta = Object.assign({}, inputOrOutputMeta); + const xsOrYs = "xs"; + const predict_normalized = {}; + Object.keys(inputMeta).forEach((k) => { + // get the min and max values + const options = { + min: inputMeta[k].min, + max: inputMeta[k].max, + }; + if (inputMeta[k].dtype === "string") { + const dataAsArray = dataRaw.map((item) => item[xsOrYs][k]); + options.legend = inputMeta[k].legend; + predict_normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === "number") { + const dataAsArray = Array(dataRaw).flatMap((item) => + item.map((obj) => obj[k]) + ); + predict_normalized[k] = this.normalizeArray(dataAsArray, options); + } + }); + + const features = Object.keys(inputMeta); + const feature_length = features.length; + + const seriesStep = dataRaw.length; + + const batch = 1; + let zipped = []; + + // zip arrays before reshaping + for (let idx = 0; idx < seriesStep * feature_length * batch; idx++) { + features.forEach((k) => { + zipped.push(predict_normalized[k][idx]); + }); + } + // reshaping + const output = seqUtils.reshapeTo3DArray(zipped, [ + batch, + seriesStep, + feature_length, + ]); + return output; + } + + /** + * applyOneHotEncodingsToDataRaw + * does not set this.data.raws + * but rather returns them + */ + applyOneHotEncodingsToDataRaw() { + const meta = Object.assign({}, this.meta); + + const output = this.data.raw.map((row) => { + const xs = { + ...row.xs, + }; + const ys = { + ...row.ys, + }; + + // get xs + Object.keys(meta.inputs).forEach((k) => { + if (meta.inputs[k].legend) { + xs[k] = meta.inputs[k].legend[row.xs[k]]; + } + }); + + Object.keys(meta.outputs).forEach((k) => { + if (meta.outputs[k].legend) { + ys[k] = meta.outputs[k].legend[row.ys[k]]; + } + }); + + return { + xs, + ys, + }; + }); + return output; + } + + /** + * loadJSON + * @param {*} dataUrlOrJson + * @param {*} inputLabels + * @param {*} outputLabels + * @void + */ + async loadJSON(dataUrlOrJson, inputLabels, outputLabels) { + try { + let json; + // handle loading parsedJson + if (dataUrlOrJson instanceof Object) { + json = Object.assign({}, dataUrlOrJson); + } else { + const { data } = await axios.get(dataUrlOrJson); + json = data; + } + + // format the data.raw array + // this.formatRawData(json, inputLabels, outputLabels); + return this.findEntries(json); + } catch (err) { + console.error("error loading json"); + throw new Error(err); + } + } + + /** + * loadCSV + * @param {*} dataUrl + * @param {*} inputLabels + * @param {*} outputLabels + * @void + */ + async loadCSV(dataUrl, inputLabels, outputLabels) { + try { + const myCsv = tf.data.csv(dataUrl); + const loadedData = await myCsv.toArray(); + const json = { + entries: loadedData, + }; + // format the data.raw array + return this.findEntries(json); + } catch (err) { + console.error("error loading csv", err); + throw new Error(err); + } + } +} + +export default SequentialData; diff --git a/src/NeuralNetwork/Sequential/sequentialUtils.js b/src/NeuralNetwork/Sequential/sequentialUtils.js new file mode 100644 index 00000000..ae4feb24 --- /dev/null +++ b/src/NeuralNetwork/Sequential/sequentialUtils.js @@ -0,0 +1,867 @@ +import { data, input } from "@tensorflow/tfjs"; +import nnUtils from "../NeuralNetworkUtils"; + +class SequentialUtils { + constructor(options) { + this.options = options || {}; + } + + /* adding data: can only accept the following formats: + - for xInputs: + 1. Sequence of objects (array of objects) + [{x: , y: },{x: , y: },{x: , y: },{x: , y: }] + 2. Sequence of arrays (array of array, order matters) + [[],[],[],[]] + */ + /** + * verifyAndFormatInputs + * @param {*} xInputs + * @param {*} options + * @param {*} classOptions + */ + verifyAndFormatInputs(xInputs, options = null, classOptions) { + const dataFormat = this.checkInputStructure(xInputs, options); + + // Apply input filtering if we have object sequences and input labels are provided + let filteredInputs = xInputs; + if (dataFormat === "ObjectSequence") { + filteredInputs = this.filterInputsByLabels( + xInputs, + options, + classOptions + ); + } + + return this.formatInputsToObjects( + filteredInputs, + options, + classOptions, + dataFormat + ); + } + + /** + * filterInputsByLabels + * Filters input objects to keep only the keys specified in input labels + * @param {Array} xInputs - Array of input objects + * @param {Object} options - Options object that may contain inputLabels + * @param {Object} classOptions - Class options that may contain inputs + * @returns {Array} - Filtered array of objects + */ + filterInputsByLabels(xInputs, options = null, classOptions) { + // Get input labels from options or classOptions + let inputLabels = null; + + if (options && options.inputLabels && Array.isArray(options.inputLabels)) { + inputLabels = options.inputLabels; + } else if ( + classOptions && + classOptions.inputs && + Array.isArray(classOptions.inputs) + ) { + inputLabels = classOptions.inputs; + } + + // If no input labels provided or not an array, return original data + if (!inputLabels || !Array.isArray(inputLabels)) { + console.log("did nothing"); + return xInputs; + } + + // Filter each object to keep only the specified keys + return xInputs.map((obj) => { + const filteredObj = {}; + inputLabels.forEach((key) => { + if (obj.hasOwnProperty(key)) { + filteredObj[key] = obj[key]; + } + }); + return filteredObj; + }); + } + + checkInputStructure(xInputs, options = null) { + if (!Array.isArray(xInputs)) { + throw new Error("Syntax Error: Data Should be in an Array"); + } + let isObjects = true; + let isArrays = true; + let isValues = true; + + for (let i = 0; i < xInputs.length; i++) { + if (nnUtils.getDataType(xInputs[i]) === "object") { + isArrays = false; + isValues = false; + if (i > 0) { + if ( + Object.keys(xInputs[i - 1]).length !== + Object.keys(xInputs[i]).length || + nnUtils.getDataType(xInputs[i - 1]) === "object" + ) { + throw new Error("Data format is inconsistent"); + } + } + } else if (Array.isArray(xInputs[i])) { + isObjects = false; + isValues = false; + if (i > 0) { + if ( + xInputs[i - 1].length !== xInputs[i].length || + !Array.isArray(xInputs[i - 1]) + ) { + throw new Error("Data format is inconsistent"); + } + } + } else { + if (options.inputLabels) { + isObjects = false; + isArrays = false; + } else { + throw new Error("inputLabels is needed for 1D array inputs"); + } + } + + if (isObjects) { + return "ObjectSequence"; + } else if (isArrays) { + return "ArraySequence"; + } else if (isValues) { + return "ValueSequence"; + } else { + throw new Error("Syntax Error: Input Structure is unknown"); + } + } + } + + formatInputsToObjects(xInputs, options = null, classOptions, dataFormat) { + switch (dataFormat) { + case "ObjectSequence": + return xInputs; + case "ArraySequence": + return this.convertArraySequence(xInputs, options, classOptions); + case "ValueSequence": + return this.convertValueSequence(xInputs, options); + default: + throw new Error("Input Data Structure is unknown"); + } + } + + convertArraySequence(xInputs, options = null, classOptions) { + let label = ""; + + if (options !== null) { + if (options.inputLabels) { + label = options.inputLabels; + } + } else if (classOptions !== null) { + if (classOptions.inputs) { + label = classOptions.inputs; + } + } + + if ( + (typeof label === "string" && label === "") || + (Array.isArray(label) && label.length === 0) + ) { + label = this.getLabelFromNestedArray(xInputs); + } + + return xInputs.map((input) => { + const obj = {}; + input.forEach((value, ind) => { + obj[label[ind]] = value; + }); + return obj; + }); + } + + convertValueSequence(xInputs, options = null) { + const { inputLabels } = options; + if (xInputs.length % inputLabels.length !== 0) { + throw new Error( + "Invalid Input: Number of Labels don't match amount of values" + ); + } + return xInputs.reduce((acc, _, index, array) => { + if (index % inputLabels.length === 0) { + // Create a new object for the current set of values + const obj = {}; + for (let i = 0; i < inputLabels.length; i++) { + obj[inputLabels[i]] = array[index + i]; + } + acc.push(obj); + } + return acc; + }, []); + } + + verifyAndFormatOutputs(yInputs, options = null, classOptions) { + const { outputs } = classOptions; + + let outputLabels; + + if (options !== null) { + if (options.outputLabels) { + outputLabels = options.outputLabels; + } + } + + if (outputs.length > 0) { + if (outputs.every((item) => typeof item === "string")) { + outputLabels = outputs; + } + } else if (typeof yInputs === "object") { + outputLabels = Object.keys(yInputs); + } else { + outputLabels = nnUtils.createLabelsFromArrayValues(yInputs, "output"); + } + + // Make sure that the inputLabels and outputLabels are arrays + if (!(outputLabels instanceof Array)) { + throw new Error("outputLabels must be an array"); + } + + // Apply output filtering if we have object outputs and output labels are provided + let filteredOutputs = yInputs; + if (typeof yInputs === "object" && !Array.isArray(yInputs)) { + filteredOutputs = this.filterOutputsByLabels( + yInputs, + options, + classOptions + ); + } + + return nnUtils.formatDataAsObject(filteredOutputs, outputLabels); + } + + /** + * filterOutputsByLabels + * Filters output objects to keep only the keys specified in output labels + * @param {Object} yInputs - Output object + * @param {Object} options - Options object that may contain outputLabels + * @param {Object} classOptions - Class options that may contain outputs + * @returns {Object} - Filtered output object + */ + filterOutputsByLabels(yInputs, options = null, classOptions) { + // Get output labels from options or classOptions + let outputLabels = null; + + if ( + options && + options.outputLabels && + Array.isArray(options.outputLabels) + ) { + outputLabels = options.outputLabels; + } else if ( + classOptions && + classOptions.outputs && + Array.isArray(classOptions.outputs) + ) { + outputLabels = classOptions.outputs; + } + + // If no output labels provided or not an array, return original data + if (!outputLabels || !Array.isArray(outputLabels)) { + return yInputs; + } + + // Filter the object to keep only the specified keys + const filteredObj = {}; + outputLabels.forEach((key) => { + if (yInputs.hasOwnProperty(key)) { + filteredObj[key] = yInputs[key]; + } + }); + return filteredObj; + } + + prepareLabels(xInputs, yInputs, options = null, classOptions) { + const { inputs, outputs } = this.options; + + let inputLabels; + let outputLabels; + + // options-based values to assign + if (options !== null) { + ({ inputLabels, outputLabels } = options); + } else if (inputs.length > 0 && outputs.length > 0) { + if (inputs.every((item) => typeof item === "string")) { + inputLabels = inputs; + } + if (outputs.every((item) => typeof item === "string")) { + outputLabels = outputs; + } + + // input-based values to assign + } else { + inputLabels = this.getLabelFromNestedArray(xInputs); + if (typeof yInputs === "object") { + outputLabels = Object.keys(yInputs); + } else { + inputLabels = this.getLabelFromNestedArray(yInputs); + } + } + + // Make sure that the inputLabels and outputLabels are arrays + if (!(inputLabels instanceof Array)) { + throw new Error("inputLabels must be an array"); + } + if (!(outputLabels instanceof Array)) { + throw new Error("outputLabels must be an array"); + } + + return inputLabels, outputLabels; + } + + getLabelFromNestedArray(xInputs, prefix = "label") { + // Recursive function to find the deepest level of the array + function traverseArray(array) { + if ( + array.length > 0 && + (typeof array[0] === "string" || typeof array[0] === "number") + ) { + return array.map((_, index) => `${prefix}_${index}`); + } else { + for (const item of array) { + if (Array.isArray(item)) { + const result = traverseArray(item); + if (result) return result; + } + } + } + return null; + } + + if (Array.isArray(xInputs)) { + return traverseArray(xInputs); + } else { + throw new Error("Input data must be an array."); + } + } + + // normalize utilities + reshapeTo3DArray(data, shape) { + const [batch, timeStep, feature] = shape; + let result = []; + let index = 0; + + for (let i = 0; i < batch; i++) { + let batchArray = []; + for (let j = 0; j < timeStep; j++) { + let timeStepArray = []; + for (let k = 0; k < feature; k++) { + timeStepArray.push(data[index]); + index++; + } + batchArray.push(timeStepArray); + } + result.push(batchArray); + } + + return result; + } + + zipArraySequence(arr1, arr2) { + if (arr1.length !== arr2.length) { + throw new Error( + `🟪 ml5.js sequence training error: Your sequences have different lengths. Try using model.setFixedLength() to make them all the same size!\n\nExample:\nconst fixedData = model.setFixedLength(yourData, 10); // Makes all sequences 10 steps long\nmodel.addData(fixedData.inputs, fixedData.outputs);` + ); + } + + return arr1.map((xs, idx) => { + const ys = arr2[idx].ys; // Extract the inner `ys` object + return { + xs: xs, + ys: ys, + }; + }); + } + + // point simplification utilities - Ramer-Douglas-Peucker (RDP) algorithm + setFixedLength(allPoints, targetPointCount, maxEpsilon = 50) { + if (allPoints.length === 0) return []; + + // Check if it's an array of objects with .x and .y properties + if (this.isObjectFormat(allPoints[0])) { + // Original format: array of objects with .x and .y + return this.simplifyObjectCoordinates( + allPoints, + targetPointCount, + maxEpsilon + ); + } + + // Array format + const numColumns = allPoints[0].length; + + if (numColumns === 2) { + // Simple case: [n,2] - just one pair of coordinates + return this.simplifyCoordinatePair( + allPoints, + targetPointCount, + maxEpsilon + ); + } else if (numColumns % 2 === 0) { + // Multiple coordinate pairs: [n,42] -> 21 pairs + const numPairs = numColumns / 2; + const simplifiedPairs = []; + + // Process each coordinate pair separately + for (let pairIndex = 0; pairIndex < numPairs; pairIndex++) { + const coordinatePair = this.extractCoordinatePair(allPoints, pairIndex); + const simplified = this.simplifyCoordinatePair( + coordinatePair, + targetPointCount, + maxEpsilon + ); + simplifiedPairs.push(simplified); + } + + // Recombine all simplified pairs back into [targetPointCount, 42] format + return this.recombineCoordinatePairs(simplifiedPairs, targetPointCount); + } else { + throw new Error( + `Invalid array format: expected even number of columns, got ${numColumns}` + ); + } + } + + // Check if data is in object format (has .x and .y properties) + isObjectFormat(point) { + return ( + point && + typeof point === "object" && + typeof point.x === "number" && + typeof point.y === "number" + ); + } + + // Handle original object format - your exact original algorithm + simplifyObjectCoordinates(allPoints, targetPointCount, maxEpsilon) { + const rdpPoints = []; + const epsilon = this.findEpsilonForPointCountObjects( + allPoints, + targetPointCount, + maxEpsilon + ); + const total = allPoints.length; + const start = allPoints[0]; + const end = allPoints[total - 1]; + + rdpPoints.push(start); + this.rdpObjects(0, total - 1, allPoints, rdpPoints, epsilon); + rdpPoints.push(end); + + if (rdpPoints.length > targetPointCount) { + return rdpPoints.slice(0, targetPointCount); + } else if (rdpPoints.length < targetPointCount) { + const filler = new Array(targetPointCount - rdpPoints.length).fill( + rdpPoints[rdpPoints.length - 1] + ); + rdpPoints.push(...filler); + return rdpPoints; + } + return rdpPoints; + } + + // Original object-based helper functions + findEpsilonForPointCountObjects(points, targetCount, maxEpsilon) { + let low = 0; + let high = maxEpsilon; + let mid; + let simplifiedPointsCount = 0; + + while (high - low > 0.001) { + mid = (low + high) / 2; + simplifiedPointsCount = this.getSimplifiedPointCountObjects(points, mid); + if (simplifiedPointsCount > targetCount) { + low = mid; + } else { + high = mid; + } + } + return mid; + } + + getSimplifiedPointCountObjects(points, epsilon) { + const rdpPoints = []; + const total = points.length; + const start = points[0]; + const end = points[total - 1]; + rdpPoints.push(start); + this.rdpObjects(0, total - 1, points, rdpPoints, epsilon); + rdpPoints.push(end); + return rdpPoints.length; + } + + rdpObjects(startIndex, endIndex, allPoints, rdpPoints, epsilon) { + const nextIndex = this.findFurthestObjects( + allPoints, + startIndex, + endIndex, + epsilon + ); + if (nextIndex > 0) { + if (startIndex !== nextIndex) { + this.rdpObjects(startIndex, nextIndex, allPoints, rdpPoints, epsilon); + } + rdpPoints.push(allPoints[nextIndex]); + if (endIndex !== nextIndex) { + this.rdpObjects(nextIndex, endIndex, allPoints, rdpPoints, epsilon); + } + } + } + + findFurthestObjects(points, a, b, epsilon) { + let recordDistance = -1; + const start = points[a]; + const end = points[b]; + let furthestIndex = -1; + + for (let i = a + 1; i < b; i++) { + const currentPoint = points[i]; + const d = this.lineDistObjects(currentPoint, start, end); + if (d > recordDistance) { + recordDistance = d; + furthestIndex = i; + } + } + + if (recordDistance > epsilon) { + return furthestIndex; + } else { + return -1; + } + } + + lineDistObjects(c, a, b) { + const norm = this.scalarProjectionObjects(c, a, b); + return Math.sqrt(Math.pow(c.x - norm.x, 2) + Math.pow(c.y - norm.y, 2)); + } + + scalarProjectionObjects(p, a, b) { + const ap = { x: p.x - a.x, y: p.y - a.y }; + const ab = { x: b.x - a.x, y: b.y - a.y }; + const abMag = Math.sqrt(ab.x * ab.x + ab.y * ab.y); + + if (abMag === 0) { + return a; + } + + ab.x /= abMag; + ab.y /= abMag; + const dot = ap.x * ab.x + ap.y * ab.y; + return { x: a.x + ab.x * dot, y: a.y + ab.y * dot }; + } + + // Extract one coordinate pair from the data + extractCoordinatePair(allPoints, pairIndex) { + const coordinatePair = []; + const xIndex = pairIndex * 2; + const yIndex = pairIndex * 2 + 1; + + for (let i = 0; i < allPoints.length; i++) { + coordinatePair.push([allPoints[i][xIndex], allPoints[i][yIndex]]); + } + + return coordinatePair; + } + + // Simplify a single coordinate pair using RDP + simplifyCoordinatePair(coordinatePair, targetPointCount, maxEpsilon) { + const rdpPoints = []; + const epsilon = this.findEpsilonForPointCount( + coordinatePair, + targetPointCount, + maxEpsilon + ); + const total = coordinatePair.length; + + if (total <= 2) { + // If we have 2 or fewer points, just pad to target + return this.padToTarget(coordinatePair, targetPointCount); + } + + const start = coordinatePair[0]; + const end = coordinatePair[total - 1]; + + rdpPoints.push(start); + this.rdp(0, total - 1, coordinatePair, rdpPoints, epsilon); + rdpPoints.push(end); + + // Remove duplicates and sort by original order + const uniquePoints = this.removeDuplicatesAndSort( + rdpPoints, + coordinatePair + ); + + return this.padToTarget(uniquePoints, targetPointCount); + } + + // Recombine simplified coordinate pairs back into original format + recombineCoordinatePairs(simplifiedPairs, targetPointCount) { + const result = []; + + for (let i = 0; i < targetPointCount; i++) { + const row = []; + for (let pairIndex = 0; pairIndex < simplifiedPairs.length; pairIndex++) { + const pair = simplifiedPairs[pairIndex][i]; + row.push(pair[0], pair[1]); // x, y + } + result.push(row); + } + + return result; + } + + // Remove duplicates and maintain original order + removeDuplicatesAndSort(rdpPoints, originalPoints) { + const seen = new Set(); + const unique = []; + + // Create a map of original indices + const indexMap = new Map(); + for (let i = 0; i < originalPoints.length; i++) { + const key = `${originalPoints[i][0]},${originalPoints[i][1]}`; + if (!indexMap.has(key)) { + indexMap.set(key, i); + } + } + + // Sort by original index and remove duplicates + rdpPoints.sort((a, b) => { + const keyA = `${a[0]},${a[1]}`; + const keyB = `${b[0]},${b[1]}`; + return indexMap.get(keyA) - indexMap.get(keyB); + }); + + for (const point of rdpPoints) { + const key = `${point[0]},${point[1]}`; + if (!seen.has(key)) { + seen.add(key); + unique.push(point); + } + } + + return unique; + } + + // Pad array to target length + padToTarget(points, targetPointCount) { + if (points.length > targetPointCount) { + return points.slice(0, targetPointCount); + } else if (points.length < targetPointCount) { + const filler = new Array(targetPointCount - points.length).fill( + points[points.length - 1] + ); + return [...points, ...filler]; + } + return points; + } + + // Find epsilon that gives approximately the target point count + findEpsilonForPointCount(points, targetCount, maxEpsilon) { + if (points.length <= targetCount) { + return 0; // No simplification needed + } + + let low = 0; + let high = maxEpsilon; + let mid; + + while (high - low > 0.001) { + mid = (low + high) / 2; + const simplifiedCount = this.getSimplifiedPointCount(points, mid); + + if (simplifiedCount > targetCount) { + low = mid; + } else { + high = mid; + } + } + + return mid; + } + + // Get count of points after simplification with given epsilon + getSimplifiedPointCount(points, epsilon) { + const rdpPoints = []; + const total = points.length; + + if (total <= 2) return total; + + const start = points[0]; + const end = points[total - 1]; + + rdpPoints.push(start); + this.rdp(0, total - 1, points, rdpPoints, epsilon); + rdpPoints.push(end); + + return new Set(rdpPoints.map((p) => `${p[0]},${p[1]}`)).size; + } + + // Core RDP algorithm + rdp(startIndex, endIndex, allPoints, rdpPoints, epsilon) { + const nextIndex = this.findFurthest( + allPoints, + startIndex, + endIndex, + epsilon + ); + + if (nextIndex > 0) { + if (startIndex !== nextIndex) { + this.rdp(startIndex, nextIndex, allPoints, rdpPoints, epsilon); + } + rdpPoints.push(allPoints[nextIndex]); + if (endIndex !== nextIndex) { + this.rdp(nextIndex, endIndex, allPoints, rdpPoints, epsilon); + } + } + } + + // Find furthest point from line segment + findFurthest(points, a, b, epsilon) { + let recordDistance = -1; + const start = points[a]; + const end = points[b]; + let furthestIndex = -1; + + for (let i = a + 1; i < b; i++) { + const currentPoint = points[i]; + const d = this.lineDist(currentPoint, start, end); + + if (d > recordDistance) { + recordDistance = d; + furthestIndex = i; + } + } + + if (recordDistance > epsilon) { + return furthestIndex; + } else { + return -1; + } + } + + // Calculate distance from point to line + lineDist(c, a, b) { + const norm = this.scalarProjection(c, a, b); + return Math.sqrt(Math.pow(c[0] - norm[0], 2) + Math.pow(c[1] - norm[1], 2)); + } + + // Project point onto line segment + scalarProjection(p, a, b) { + const ap = [p[0] - a[0], p[1] - a[1]]; + const ab = [b[0] - a[0], b[1] - a[1]]; + + const abMag = Math.sqrt(ab[0] * ab[0] + ab[1] * ab[1]); + + if (abMag === 0) { + return a; // Start and end points are the same + } + + ab[0] /= abMag; + ab[1] /= abMag; + + const dot = ap[0] * ab[0] + ap[1] * ab[1]; + + return [a[0] + ab[0] * dot, a[1] + ab[1] * dot]; + } + + /** + * Creates training data using a sliding window approach + * @param {Array} data - Array of data objects + * @param {number} targetLength - Length of the sequence window + * @param {Array} featureKeys - Array of keys to use as input features + * @param {Array} targetKeys - Array of keys to predict (can be the same as featureKeys) + * @returns {Object} - Object containing sequences and targets arrays + */ + createSlidingWindowData(data, targetLength, featureKeys, targetKeys) { + // Validate inputs + if (!data || !Array.isArray(data) || data.length <= targetLength) { + throw new Error( + "Data Format is invalid please check usage of this helper function in documentation" + ); + } + + const inputs = []; + const outputs = []; + + // Start from the first possible complete sequence + for ( + let dataIndex = targetLength - 1; + dataIndex < data.length - 1; + dataIndex++ + ) { + let seq = []; + + // Build sequence of previous targetLength steps + for (let x = targetLength - 1; x >= 0; x--) { + let curr = data[dataIndex - x]; + let inputs = {}; + + // Extract only the specified feature keys + featureKeys.forEach((key) => { + inputs[key] = curr[key]; + }); + + seq.push(inputs); + } + + // The target is the next data point after the sequence + let target = data[dataIndex + 1]; + let output = {}; + + // Extract only the specified target keys + targetKeys.forEach((key) => { + output[key] = target[key]; + }); + + inputs.push(seq); + outputs.push(output); + } + + return { inputs, outputs }; + } + + /** + * Creates a sequence from the most recent data points + * @param {Array} data - Array of data objects + * @param {number} sequenceLength - Length of the sequence to create + * @param {Array} featureKeys - Array of keys to include in the sequence + * @returns {Array} - Array of objects containing the selected features + */ + getLatestSequence(data, sequenceLength, featureKeys) { + // Validate inputs + if (!data || !Array.isArray(data) || data.length === 0) { + throw new Error("Invalid data input"); + } + + // Ensure we don't try to get more items than exist in the data + const actualLength = Math.min(sequenceLength, data.length); + + // Get the most recent data points + const latest = data.slice(-actualLength); + + // Create the sequence with selected features + const sequence = latest.map((item) => { + const inputs = {}; + + featureKeys.forEach((key) => { + inputs[key] = item[key]; + }); + + return inputs; + }); + + return sequence; + } +} + +const sequentialUtils = () => { + const instance = new SequentialUtils(); + return instance; +}; + +export default sequentialUtils(); diff --git a/src/NeuralNetwork/index.js b/src/NeuralNetwork/index.js index fa439452..334523c1 100644 --- a/src/NeuralNetwork/index.js +++ b/src/NeuralNetwork/index.js @@ -1249,23 +1249,5 @@ class DiyNeuralNetwork { } } -const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { - let options; - let cb; - - if (inputsOrOptions instanceof Object) { - options = inputsOrOptions; - cb = outputsOrCallback; - } else { - options = { - inputs: inputsOrOptions, - outputs: outputsOrCallback, - }; - cb = callback; - } - - const instance = new DiyNeuralNetwork(options, cb); - return instance; -}; - -export default neuralNetwork; +export { DiyNeuralNetwork }; // Named export for extending for Sequential Neural Network +export default DiyNeuralNetwork; //export for taskSelection.js diff --git a/src/NeuralNetwork/taskSelection.js b/src/NeuralNetwork/taskSelection.js new file mode 100644 index 00000000..8db8b41e --- /dev/null +++ b/src/NeuralNetwork/taskSelection.js @@ -0,0 +1,77 @@ +import DiyNeuralNetwork from "./index.js"; +import DIYSequential from "./Sequential/index.js"; + +// helper function to check if tasks follows specified convention +const isSequenceTask = (task) => { + const sequenceTask = [ + "sequenceClassification", + "sequenceRegression", + "sequenceClassificationWithCNN", + "sequenceRegressionWithCNN", + ]; + return sequenceTask.includes(task); +}; + +// factory function for DIY Neural Network +const createNeuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + let cb; + + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + cb = outputsOrCallback; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + cb = callback; + } + + const instance = new DiyNeuralNetwork(options, cb); + return instance; +}; + +// factory function for DIY Sequential +const createSequential = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + let cb; + + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + cb = outputsOrCallback; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + cb = callback; + } + + const instance = new DIYSequential(options, cb); + return instance; +}; + +// Selection logic for either NeuralNetwork or Sequential +const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { + let options; + + // Parse options first to check the task + if (inputsOrOptions instanceof Object) { + options = inputsOrOptions; + } else { + options = { + inputs: inputsOrOptions, + outputs: outputsOrCallback, + }; + } + + // Choose which factory function to call based on task + if (isSequenceTask(options.task)) { + return createSequential(inputsOrOptions, outputsOrCallback, callback); + } else { + return createNeuralNetwork(inputsOrOptions, outputsOrCallback, callback); + } +}; + +export default neuralNetwork; diff --git a/src/index.js b/src/index.js index 09416c1c..7db37053 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,4 @@ -import neuralNetwork from "./NeuralNetwork"; +import neuralNetwork from "./NeuralNetwork/taskSelection"; import handPose from "./HandPose"; import sentiment from "./Sentiment"; import faceMesh from "./FaceMesh";