update generate.scd for noteoff

Author: victor-shepardson

examples/notepredictor/generate.scd

@@ -4,6 +4,19 @@
 // TODO: steerable generation. gui for ranges, temperatures;
 // MIDI controller for pitch set
 
+// TODO: keep track of pending model predictions so there are no races
+// whenever a note is played, feed it to the model
+// count the number of pending predictions
+// if a note is played while there are pending predictions, cancel them
+// ideally, would signal python to interrupt computation
+// less complicated: just ignore the results when they come in
+// there are pending predictions and pending notes:
+// pending predictions need to be ignored when they come in
+// pending notes need to be unscheduled
+
+// TODO: pitch blacklist option
+// use case: forbid model from releasing a note held by the player
+
 (
 ~gui = false;
 MIDIIn.connectAll;
@@ -49,6 +62,9 @@ b.sendMsg("/predictor/predict",
 // duet with the model
 // feeds the model's predictions back to it as well as player input
 (
+~pending_predictions = 0;
+~synths = nil!128;
+~player_held = false!128;
 ~step = 0;
 ~gate = 1;
 t = nil;
@@ -66,23 +82,56 @@ MIDIdef.program(\switch, {
         ~gate = 0;
         SystemClock.clear;
         b.sendMsg("/predictor/reset");
-        y.release;
+        ~synths.do(_.release(1.0));
         SystemClock.clear;
         ~step = 0;
         };
     ~gate.postln;
 });
 
-
 // MIDI from controller
-MIDIdef.noteOn(\input, {
+MIDIdef.noteOff(\input_off, {
     arg val, num, chan, src;
     var t2 = Process.elapsedTime;
     var dt = t2-(t?(t2-~delay)); //time since last note
 
     // cancel any pending predictions
     SystemClock.clear;
+    ~pending_predictions.postln;
+    //get a new prediction in light of current note
+    b.sendMsg("/predictor/predict",
+        \pitch, num, \time, dt, \vel, 0,
+        \allow_start, false, \allow_end, false,
+        \pitch_temp, 0.5, \rhythm_temp, 0.5, \timing_temp, 0.1,
+        \min_time, ~delay, \max_time, 5
+        // \fix_time, ~delay
+    );
+    ~pending_predictions = ~pending_predictions+1;
+
+    // release the previous note
+    ~synths[num]!?(_.release(0.05));
+    ~synths[num] = nil;
+
+    // post the current note
+    [\player, dt, num, 0].postln;
+
+    // mark time of current note
+    t = t2;
+    ~player_t = t;
 
+    ~step = ~step + 1;
+
+    ~player_held[num] = false;
+
+});
+MIDIdef.noteOn(\input_on, {
+    arg val, num, chan, src;
+    var t2 = Process.elapsedTime;
+    var dt = t2-(t?(t2-~delay)); //time since last note
+
+    // cancel any pending predictions
+    SystemClock.clear;
+    ~pending_predictions.postln;
     //get a new prediction in light of current note
     b.sendMsg("/predictor/predict",
         \pitch, num, \time, dt, \vel, val,
@@ -91,12 +140,14 @@ MIDIdef.noteOn(\input, {
         \min_time, ~delay, \max_time, 5
         // \fix_time, ~delay
     );
+    ~pending_predictions = ~pending_predictions+1;
+
 
     // release the previous note
-    y.release(0.05);
+    ~synths[num]!?(_.release(0.05));
 
     // play the current note
-    y = Synth(\pluck, [\freq, num.midicps, \vel, val/127]);//.release(1);
+    ~synths[num] = Synth(\pluck, [\freq, num.midicps, \vel, val/127]);//.release(1);
 
     // post the current note
     [\player, dt, num, val].postln;
@@ -107,6 +158,8 @@ MIDIdef.noteOn(\input, {
 
     ~step = ~step + 1;
     // ~step = 0;
+
+    ~player_held[num] = true;
 });
 
 // OSC return from python
@@ -115,15 +168,21 @@ OSCdef(\return, {
     var num = msg[1]; // MIDI number of predicted note
     var dt = msg[2]; // time to predicted note
     var val = msg[3]; // velocity 0-127
+    var step = msg[4];
 
     // time-to-next note gets 'censored' by the model
     // when over a threshold, in this case 10 seconds,
     // meaning it just predicts 10s rather than any longer time
     var censor = dt>=10.0;
 
+    ~pending_predictions = ~pending_predictions-1;
+    [\step, step].postln;
+
+
     censor.if{
         // if the predicted time is > 10 seconds, don't schedule it, just stop.
-        \censor.postln; y.release(3.0)
+        \censor.postln;
+        // ~synths[num]!?(_.release(3.0));
     }{
         // schedule the predicted note
         SystemClock.sched(dt-~delay, {
@@ -135,7 +194,7 @@ OSCdef(\return, {
                     // in this case don't schedule a note, and reset the model
                     // b.sendMsg("/predictor/reset");
                     //release the last note
-                    y.release(1.0);
+                    ~synths.do(_.release(1.0));
                     // unset time so next note will have dt=0
                     // t = nil;
                     // \reset.postln
@@ -145,28 +204,27 @@ OSCdef(\return, {
                     // (there shouldn't be any, but might
                     // be if there was a lot of fast MIDI input)
                     SystemClock.clear;
+                    ~pending_predictions.postln;
                     // feed model its own prediction as input
                     b.sendMsg("/predictor/predict",
                         \pitch, num, \time, dt_actual, \vel, val,
                         \allow_start, false, \allow_end, true,
                         \pitch_temp, 0.7, \rhythm_temp, 0.7, \timing_temp, 0.1,
                         \min_time, ~delay*2, \max_time, 5,
-                        \min_vel, 10
+                        // \min_vel, 10
                         // \fix_time, ((~step+1)%3==0).if{0.6}{0}  // triads
                         // \fix_time, (~step%8)*0.1 // specific rhythm
 
                     );
-                    // release the previous note
-                    (dt<3e-2).if{
-                        // if the time delay is very small, slow release for chord
-                        y.release(1.0)
+                    ~pending_predictions = ~pending_predictions+1;
+
+                    // play the current note
+                    ~synths[num]!?(_.release(0.05));
+                    (val > 0).if{
+                        ~synths[num] = Synth(\pluck, [\freq, num.midicps, \vel, val/127])
                     }{
-                        // otherwise release fast to play a melody
-                        y.release(0.05)
+                        ~synths[num] = nil
                     };
-                    // play the current note
-                    y = Synth(\pluck, [
-                        \freq, num.midicps, \vel, val/127]);//.release(1);
                     // post the current note
                     [\model, dt, num, val].postln;
                     // mark the actual time of current note
@@ -186,11 +244,10 @@ OSCdef(\return, {
 }, "/prediction", nil);
 )
 
-
 (
 // send a note manually if you don't have a MIDI controller:
 SystemClock.clear;
-y.release(0.2);
+~synths.do(_.release(1.0));
 b.sendMsg("/predictor/reset");
 {MIDIdef.all[\input].func.value(99, 60)}.defer(0.5);
 SystemClock.clear;

examples/notepredictor/server.py

@@ -39,7 +39,7 @@ def _(address, **kw):
                 print('no model loaded')
             else:
                 r = predictor.predict(**kw)
-                return '/prediction', r['pitch'], r['time'], r['velocity']
+                return '/prediction', r['pitch'], r['time'], r['velocity'], r['step']
 
         elif cmd=="reset":
             if predictor is None:

notepredictor/notepredictor/model.py

@@ -201,6 +201,7 @@ def __init__(self,
         # persistent RNN state for inference
         for n,t in zip(self.cell_state_names(), self.initial_state):
             self.register_buffer(n, t.clone())
+        self.step = 0
 
     def cell_state_names(self):
         return tuple(f'cell_state_{i}' for i in range(len(self.initial_state)))
@@ -515,7 +516,10 @@ def predict(self,
                 pred_pitch = pred_pitch.item()
                 pred_time = pred_time.item()
                 pred_vel = pred_vel.item()
+                
+            self.step += 1
             return {
+                'step': self.step,
                 'pitch': pred_pitch, 
                 'time': pred_time,
                 'velocity': pred_vel,
@@ -531,6 +535,7 @@ def reset(self, start=True):
             start: if True, send a start token through the model with dt=0
                    but discard the prediction
         """
+        self.step = 0
         for n,t in zip(self.cell_state_names(), self.initial_state):
             getattr(self, n)[:] = t.detach()
         if start: