[WIP] TimeSeries Feature #192
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| <!-- | ||
| 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| Learn more about the ml5.js project: https://ml5js.org/ | ||
| ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
|
|
||
| This example demonstrates training a Sign Language classifier through ml5.TimeSeries. | ||
| --> | ||
|
|
||
| <html> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js Sign Language Neural Network Train and Save</title> | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
|
|
||
| <body> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,217 @@ | ||
| /* | ||
mop9047 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| * 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| * Learn more about the ml5.js project: https://ml5js.org/ | ||
| * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
| * | ||
| * This example demonstrates training a Sign Language classifier through ml5.TimeSeries. | ||
| */ | ||
|
|
||
| const seqlength = 50; | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| let handPose; | ||
| let video; | ||
|
|
||
| let hands = []; | ||
| let sequence = []; | ||
|
|
||
| let recording_finished = false; | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| let predicted_word = ''; | ||
|
|
||
| // UI variables | ||
| let training_words = {}; | ||
|
|
||
|
|
||
| function preload() { | ||
| // Load the handPose model | ||
| handPose = ml5.handPose(); | ||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
|
|
||
| // setup video capture | ||
| video = createCapture(VIDEO); | ||
| video.size(640, 480); | ||
| video.hide(); | ||
|
|
||
| // place UI elements | ||
| UI(); | ||
|
|
||
| // set backend as either webgl or cpu | ||
| ml5.setBackend('webgl') | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| // use handpose model on video | ||
| handPose.detectStart(video, gotHands); | ||
|
|
||
| // setup the timeseries neural network | ||
| let options = { | ||
| outputs: ['label'], | ||
| task: 'classification', | ||
| dataModality: 'spatial', | ||
| debug: 'true', | ||
| learningRate: 0.001, | ||
| }; | ||
| model = ml5.timeSeries(options); | ||
|
|
||
| } | ||
|
|
||
| function draw() { | ||
| // draw video on frame | ||
| image(video, 0, 0, width, height); | ||
|
|
||
| drawPredictedWord(); | ||
|
|
||
| // if hands are found then start recording | ||
| if(hands.length>0 && recording_finished == false){ | ||
| if (sequence.length <= seqlength){ | ||
| // get coordinates from hands (21 points) | ||
| handpoints = drawPoints(); | ||
| sequence.push(handpoints); | ||
|
|
||
| // once sequence reaches the seqlength, add sequence as just one X value | ||
| } else if (sequence.length>0){ | ||
| // get the training word from the input box | ||
| let train_word = nameField.value() | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| // if there is a word currently in the box then add data with that label | ||
| if (train_word.length > 0){ | ||
| // add data to the model | ||
| let target = {label:train_word} | ||
| model.addData(sequence, target); | ||
| trainingWordsUpdate() | ||
|
|
||
| // if there is no word in the box then classify instead | ||
| } else { | ||
| // classify the data | ||
| model.classify(sequence, gotResults); | ||
| } | ||
|
|
||
| // reset the sequence | ||
| sequence = []; | ||
| recording_finished = true; | ||
| } | ||
|
|
||
| // can only record again when hand is out of frame | ||
| } else { | ||
| if (hands.length == 0){ | ||
| recording_finished = false; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| function drawPoints(){ | ||
| let handpoints = [] | ||
| // iterate through both hands | ||
| for (let i = 0; i < hands.length; i++) { | ||
| let hand = hands[i]; | ||
| for (let j = 0; j < hand.keypoints.length; j++) { | ||
| // access the keypoints in the hand | ||
| let keypoint = hand.keypoints[j]; | ||
| handpoints.push(keypoint.x,keypoint.y) | ||
|
|
||
| fill(0, 255, 0); | ||
| noStroke(); | ||
| circle(keypoint.x, keypoint.y, 5); | ||
| } | ||
| } | ||
| // assign to a different variable before clearing | ||
| const output = handpoints; | ||
| handpoints = []; | ||
|
|
||
| return output; | ||
| } | ||
|
|
||
| // Callback function for when handPose outputs data | ||
| function gotHands(results) { | ||
| // save the output to the hands variable | ||
| hands = results; | ||
| } | ||
|
|
||
| function keyPressed(){ | ||
| if (key == 's'){ | ||
| model.save('hello'); | ||
| } | ||
| if (key == 'z'){ | ||
| model.saveData(); | ||
| } | ||
|
|
||
| if (key == 't'){ | ||
| model.normalizeData(); | ||
| let options = { | ||
| epochs: 100 | ||
| } | ||
| model.train(options,whileTraining,finishedTraining); | ||
| } | ||
| } | ||
|
|
||
| function trainModelAndSave(){ | ||
| model.normalizeData(); | ||
| let options = { | ||
| epochs: 100 | ||
| } | ||
| model.train(options,whileTraining,finishedTraining); | ||
| nameField.value('') | ||
| } | ||
|
|
||
| function whileTraining(epoch) { | ||
| console.log(epoch); | ||
| } | ||
|
|
||
| function finishedTraining() { | ||
| console.log('finished training.'); | ||
| model.save('model'); | ||
| } | ||
|
|
||
| function gotResults(results){ | ||
| predicted_word = results[0].label | ||
| console.log(predicted_word) | ||
| text(predicted_word, 200,200) | ||
| } | ||
|
|
||
| function UI(){ | ||
|
||
| nameField = createInput('') | ||
| nameField.attribute('placeholder', 'Type the word to train') | ||
| nameField.position(110, 500) | ||
| nameField.size(250) | ||
|
|
||
| instructionP = createP( | ||
| 'I want to train: <br><br> 1.) Type any word you want to pair with a gesture, e.g. "HELLO" <br> 2.) Do the gesture associated to the word, make sure to do it until the points disappear. <br> 3.) Move your hand out of the frame and repeat the gesture, do this multiple times <br> 4.) Do the same for other words e.g. "BYE" <br> 5.) Once all data is collected, press Train and Save<br><br> Tip: have at least 5 datasets for each word' | ||
| ); | ||
| instructionP.style("width", "640px"); | ||
| dataCountsP = createP( | ||
| "-> After the gesture a tally will appear here <-" | ||
| ); | ||
|
|
||
| train_but = createButton('Train and Save'); | ||
| train_but.mouseClicked(trainModelAndSave); | ||
| train_but.style("font-family", "Georgia"); | ||
mop9047 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| train_but.style("font-size", "20px"); | ||
| train_but.position(500, 490) | ||
| } | ||
|
|
||
| function drawPredictedWord(){ | ||
| textSize(100) | ||
| fill(255) | ||
| text(predicted_word, 100, height/2) | ||
| } | ||
|
|
||
| function trainingWordsUpdate(){ | ||
| let temp_word = nameField.value(); | ||
| console.log(Object.keys(training_words)); | ||
| if (!(temp_word in training_words)){ | ||
| training_words[temp_word] = 1; | ||
| console.log('here') | ||
| } else { | ||
| console.log(training_words[temp_word]) | ||
| training_words[temp_word]++; | ||
| } | ||
| let counts = '' | ||
| let keys = Object.keys(training_words) | ||
| keys.forEach(element => { | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| counts += element + ' : ' + training_words[element] + "<br>" | ||
| }); | ||
| dataCountsP.html( | ||
| counts | ||
| ); | ||
|
|
||
| } | ||
mop9047 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| <!-- | ||
| 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| Learn more about the ml5.js project: https://ml5js.org/ | ||
| ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
|
|
||
| This example demonstrates loading a Sign Language classifier through ml5.TimeSeries. | ||
| --> | ||
|
|
||
| <html> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js Sign Language Neural Network load model</title> | ||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
|
|
||
| <body> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.8.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./hello.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,42,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[176,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]} |
Binary file not shown.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"inputUnits":[42],"outputUnits":2,"inputs":{"label_0":{"dtype":"number","min":4.151249399907168,"max":586.4725394909854},"label_1":{"dtype":"number","min":186.47223882383636,"max":496.34918695509003},"label_2":{"dtype":"number","min":12.818880217505907,"max":564.7860747522525},"label_3":{"dtype":"number","min":160.9460986889124,"max":478.89482602620234},"label_4":{"dtype":"number","min":20.681431005110262,"max":557.1173870582799},"label_5":{"dtype":"number","min":135.1274696802808,"max":454.0862355189599},"label_6":{"dtype":"number","min":29.375938053231934,"max":562.4826339023859},"label_7":{"dtype":"number","min":113.22511415628927,"max":455.15365538508894},"label_8":{"dtype":"number","min":37.27265551578051,"max":573.3838980891996},"label_9":{"dtype":"number","min":98.00531862273047,"max":473.4382341601794},"label_10":{"dtype":"number","min":2.706973037101564,"max":599.2858408346702},"label_11":{"dtype":"number","min":117.7350326456234,"max":453.76022921684716},"label_12":{"dtype":"number","min":11.635752695869659,"max":612.8243751678727},"label_13":{"dtype":"number","min":91.05094143918305,"max":481.6467136241304},"label_14":{"dtype":"number","min":22.9353041163117,"max":621.0127886598051},"label_15":{"dtype":"number","min":61.619264849841635,"max":499.63536096409143},"label_16":{"dtype":"number","min":33.53953084457643,"max":626.4181148091915},"label_17":{"dtype":"number","min":28.455718477478662,"max":512.7953875856006},"label_18":{"dtype":"number","min":-2.8065139589559984,"max":617.7828981986556},"label_19":{"dtype":"number","min":117.6886729722432,"max":459.5357193516273},"label_20":{"dtype":"number","min":3.7782929928570064,"max":633.7038985044576},"label_21":{"dtype":"number","min":86.77279076496669,"max":486.0751342925063},"label_22":{"dtype":"number","min":16.177018651157255,"max":642.8366376068107},"label_23":{"dtype":"number","min":51.687144639081325,"max":502.64037741142846},"label_24":{"dtype":"number","min":28.1461509145229,"max":650.2419536370577},"label_25":{"dtype":"number","min":15.922382743702723,"max":516.9301399988833},"label_26":{"dtype":"number","min":-6.382516546058305,"max":630.7077663350849},"label_27":{"dtype":"number","min":120.16376158664924,"max":461.0881814514869},"label_28":{"dtype":"number","min":-1.4074379536407533,"max":647.5041251714117},"label_29":{"dtype":"number","min":90.58035685591811,"max":485.04491883378125},"label_30":{"dtype":"number","min":10.174906800459325,"max":658.4893875478738},"label_31":{"dtype":"number","min":71.76407331703523,"max":500.55112323964187},"label_32":{"dtype":"number","min":21.11718120932074,"max":668.566957655395},"label_33":{"dtype":"number","min":39.557348432978586,"max":514.4287318106208},"label_34":{"dtype":"number","min":-7.9534800405596595,"max":641.3232619371444},"label_35":{"dtype":"number","min":126.31599791044414,"max":465.6320514399833},"label_36":{"dtype":"number","min":-3.8369034650104927,"max":658.2044139172733},"label_37":{"dtype":"number","min":103.73604938021917,"max":481.03793223993495},"label_38":{"dtype":"number","min":3.7075645592075435,"max":668.8017566330357},"label_39":{"dtype":"number","min":88.76136006394765,"max":494.63688258092407},"label_40":{"dtype":"number","min":6.9609311353376135,"max":676.9525074586147},"label_41":{"dtype":"number","min":75.97401514052241,"max":506.7948506427954}},"outputs":{"label":{"dtype":"string","min":0,"max":1,"uniqueValues":["hello","bye"],"legend":{"hello":[1,0],"bye":[0,1]}}},"isNormalized":true,"seriesShape":[51,42]} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| /* | ||
| * 👋 Hello! This is an ml5.js example made and shared with ❤️. | ||
| * Learn more about the ml5.js project: https://ml5js.org/ | ||
| * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md | ||
| * | ||
| * This example demonstrates loading a Sign Language classifier through ml5.TimeSeries. | ||
| */ | ||
|
|
||
| // change this to make the recording longer | ||
| const seqlength = 50; | ||
|
|
||
|
|
||
| let handPose; | ||
| let video; | ||
| let hands = []; | ||
| let sequence = []; | ||
| let recording_finished = false; | ||
| let predicted_word = '' | ||
|
|
||
| function preload() { | ||
| // Load the handPose model | ||
| handPose = ml5.handPose(); | ||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
|
|
||
| // create video capture | ||
| video = createCapture(VIDEO); | ||
| video.size(640, 480); | ||
| video.hide(); | ||
|
|
||
| ml5.setBackend('webgl') | ||
|
|
||
| handPose.detectStart(video, gotHands); | ||
|
|
||
| let options = { | ||
| task: 'classification', | ||
| dataModality: 'spatial', | ||
| }; | ||
|
|
||
| model = ml5.timeSeries(options); | ||
|
|
||
| // setup the model files to load | ||
| const modelDetails = { | ||
| model: "model/model.json", | ||
| metadata: "model/model_meta.json", | ||
| weights: "model/model.weights.bin", | ||
| }; | ||
|
|
||
| // load the model and call modelLoaded once finished | ||
| model.load(modelDetails, modelLoaded); | ||
| } | ||
| // call back for load model | ||
| function modelLoaded(){ | ||
| console.log('model loaded!') | ||
| } | ||
|
|
||
| function draw() { | ||
| // draw video on the canvas | ||
| image(video, 0, 0, width, height); | ||
|
|
||
| // put the text on screen after a prediction | ||
| placePredictedText() | ||
|
|
||
| // if hands are found then start recording | ||
| if(hands.length>0 && recording_finished == false){ | ||
| if (sequence.length <= seqlength){ | ||
| // get coordinates from hands (21 points) | ||
| handpoints = drawPoints(); | ||
| sequence.push(handpoints); | ||
|
|
||
| // once sequence reaches the seqlength, add sequence as just one X value | ||
| } else if (sequence.length>0){ | ||
| // classify based on the collected data | ||
| model.classify(sequence, gotResults); | ||
|
|
||
| // reset the sequence | ||
| sequence = []; | ||
| recording_finished = true; | ||
| } | ||
|
|
||
| // can only record again when hand is out of frame | ||
| } else { | ||
| if (hands.length == 0){ | ||
| recording_finished = false; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // draw the points on the hands | ||
| function drawPoints(){ | ||
| let handpoints = [] | ||
| for (let i = 0; i < hands.length; i++) { | ||
| let hand = hands[i]; | ||
| for (let j = 0; j < hand.keypoints.length; j++) { | ||
| let keypoint = hand.keypoints[j]; | ||
| fill(0, 255, 0); | ||
| noStroke(); | ||
| circle(keypoint.x, keypoint.y, 5); | ||
| handpoints.push(keypoint.x,keypoint.y) | ||
| } | ||
| } | ||
| const output = handpoints; | ||
| handpoints = []; return output; | ||
| } | ||
|
|
||
| // Callback function for when handPose outputs data | ||
| function gotHands(results) { | ||
| // save the output to the hands variable | ||
| hands = results; | ||
| } | ||
|
|
||
| // call back for accessing the results | ||
| function gotResults(results){ | ||
| predicted_word = results[0].label | ||
| console.log(predicted_word) | ||
| text(predicted_word, 100,100) | ||
| } | ||
|
|
||
| // for drawing text on screen | ||
| function placePredictedText(){ | ||
| textSize(100) | ||
| fill(255) | ||
| text(predicted_word, 100, height/2) | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.