multitrack model

Author: victor-shepardson

examples/notepredictor/generate.scd

@@ -51,13 +51,13 @@ OSCdef(\return, {
 }, '/prediction', nil);
 t = Process.elapsedTime;
 b.sendMsg("/predictor/predict",
-    \pitch, 60+12.rand, \time, 0, \vel, 0,
+    \inst, 0, \pitch, 60+12.rand, \time, 0, \vel, 0,
     \pitch_temp, 0.5, \rhythm_temp, 0.5, \timing_temp, 0.1
 );
 )
 
 // set the delay for more precise timing
-~delay = 0.016;
+~delay = 0.02;
 
 // duet with the model
 // feeds the model's predictions back to it as well as player input
@@ -94,13 +94,14 @@ MIDIdef.noteOff(\input_off, {
     arg val, num, chan, src;
     var t2 = Process.elapsedTime;
     var dt = t2-(t?(t2-~delay)); //time since last note
+    var inst = ~player_inst;
 
     // cancel any pending predictions
     SystemClock.clear;
     ~pending_predictions.postln;
     //get a new prediction in light of current note
     b.sendMsg("/predictor/predict",
-        \pitch, num, \time, dt, \vel, 0,
+        \inst, inst, \pitch, num, \time, dt, \vel, 0,
         \allow_start, false, \allow_end, false,
         \pitch_temp, 0.5, \rhythm_temp, 0.5, \timing_temp, 0.1,
         \min_time, ~delay, \max_time, 5
@@ -113,7 +114,7 @@ MIDIdef.noteOff(\input_off, {
     ~synths[num] = nil;
 
     // post the current note
-    [\player, dt, num, 0].postln;
+    [\player, dt, inst, num, 0].postln;
 
     // mark time of current note
     t = t2;
@@ -128,13 +129,14 @@ MIDIdef.noteOn(\input_on, {
     arg val, num, chan, src;
     var t2 = Process.elapsedTime;
     var dt = t2-(t?(t2-~delay)); //time since last note
+    var inst = ~player_inst;
 
     // cancel any pending predictions
     SystemClock.clear;
     ~pending_predictions.postln;
     //get a new prediction in light of current note
     b.sendMsg("/predictor/predict",
-        \pitch, num, \time, dt, \vel, val,
+        \inst, inst, \pitch, num, \time, dt, \vel, val,
         \allow_start, false, \allow_end, false,
         \pitch_temp, 0.5, \rhythm_temp, 0.5, \timing_temp, 0.1,
         \min_time, ~delay, \max_time, 5
@@ -150,7 +152,7 @@ MIDIdef.noteOn(\input_on, {
     ~synths[num] = Synth(\pluck, [\freq, num.midicps, \vel, val/127]);//.release(1);
 
     // post the current note
-    [\player, dt, num, val].postln;
+    [\player, dt, inst, num, val].postln;
 
     // mark time of current note
     t = t2;
@@ -165,31 +167,33 @@ MIDIdef.noteOn(\input_on, {
 // OSC return from python
 OSCdef(\return, {
     arg msg, time, addr, recvPort;
-    var num = msg[1]; // MIDI number of predicted note
-    var dt = msg[2]; // time to predicted note
-    var val = msg[3]; // velocity 0-127
-    var step = msg[4];
+    var inst = msg[1]; // instrument of predicted note
+    var pitch = msg[2]; // MIDI number of predicted note
+    var dt = msg[3]; // time to predicted note
+    var val = msg[4]; // velocity 0-127
+    var end = msg[5];
+    var step = msg[6];
 
     // time-to-next note gets 'censored' by the model
     // when over a threshold, in this case 10 seconds,
     // meaning it just predicts 10s rather than any longer time
     var censor = dt>=10.0;
 
     ~pending_predictions = ~pending_predictions-1;
-    [\step, step].postln;
+    // [\step, step].postln;
 
 
     censor.if{
         // if the predicted time is > 10 seconds, don't schedule it, just stop.
         \censor.postln;
-        // ~synths[num]!?(_.release(3.0));
+        // ~synths[pitch]!?(_.release(3.0));
     }{
         // schedule the predicted note
         SystemClock.sched(dt-~delay, {
             (~gate>0).if{
                 var t2 = Process.elapsedTime;
                 var dt_actual = t2 - t;
-                (num==129).if{
+                (end==1).if{
                     // 129 is the 'stop token',  meaning 'end-of-performance'
                     // in this case don't schedule a note, and reset the model
                     // b.sendMsg("/predictor/reset");
@@ -204,40 +208,36 @@ OSCdef(\return, {
                     // (there shouldn't be any, but might
                     // be if there was a lot of fast MIDI input)
                     SystemClock.clear;
-                    ~pending_predictions.postln;
+                    // ~pending_predictions.postln;
                     // feed model its own prediction as input
                     b.sendMsg("/predictor/predict",
-                        \pitch, num, \time, dt_actual, \vel, val,
+                        \inst, inst, \pitch, pitch, \time, dt_actual, \vel, val,
                         \allow_start, false, \allow_end, true,
                         \pitch_temp, 0.7, \rhythm_temp, 0.7, \timing_temp, 0.1,
-                        \min_time, ~delay*2, \max_time, 5,
-                        // \min_vel, 10
-                        // \fix_time, ((~step+1)%3==0).if{0.6}{0}  // triads
-                        // \fix_time, (~step%8)*0.1 // specific rhythm
-
+                        \min_time, ~delay*0, \max_time, 5,
                     );
                     ~pending_predictions = ~pending_predictions+1;
 
                     // play the current note
-                    ~synths[num]!?(_.release(0.05));
+                    ~synths[pitch]!?(_.release(0.05));
                     (val > 0).if{
-                        ~synths[num] = Synth(\pluck, [\freq, num.midicps, \vel, val/127])
+                        ~synths[pitch] = Synth(\pluck, [\freq, pitch.midicps, \vel, val/127])
                     }{
-                        ~synths[num] = nil
+                        ~synths[pitch] = nil
                     };
                     // post the current note
-                    [\model, dt, num, val].postln;
+                    [\model, step, dt, inst, pitch, val, end].postln;
                     // mark the actual time of current note
                     t = t2;
                     ~machine_t = t;
                     // crudely draw note on piano GUI
                     ~gui.if{
-                        AppClock.sched(0,{k.keyDown(num)});
-                        AppClock.sched(0.2,{k.keyUp(num)});
+                        AppClock.sched(0,{k.keyDown(pitch)});
+                        AppClock.sched(0.2,{k.keyUp(pitch)});
                     }
                 };
                 ~step = ~step+1;
-                [\late, dt_actual-dt].postln;
+                // [\late, dt_actual-dt].postln;
             }
     })};
 
@@ -249,7 +249,8 @@ OSCdef(\return, {
 SystemClock.clear;
 ~synths.do(_.release(1.0));
 b.sendMsg("/predictor/reset");
-{MIDIdef.all[\input].func.value(99, 60)}.defer(0.5);
+~player_inst = 83;
+{MIDIdef.all[\input_on].func.value(99, 60)}.defer(0.5);
 SystemClock.clear;
 )
 // b.sendMsg("/predictor/predict", \pitch, 70, \time, 0, \vel, 64);

examples/notepredictor/server.py

@@ -39,7 +39,7 @@ def _(address, **kw):
                 print('no model loaded')
             else:
                 r = predictor.predict(**kw)
-                return '/prediction', r['pitch'], r['time'], r['velocity'], r['step']
+                return '/prediction', r['instrument'], r['pitch'], r['time'], r['velocity'], r['end'], r['step']
 
         elif cmd=="reset":
             if predictor is None:

notepredictor/notepredictor/data.py

@@ -17,7 +17,8 @@ def __init__(self, data_dir, batch_len, transpose=2, speed=0.1, glob='**/*.pkl')
         self.transpose = transpose
         self.speed = speed
         self.start_token = 128
-        self.end_token = 129
+        self.n_anon = 8
+        self.prog_start_token = 0
         # self.clamp_time = clamp_time
 
     def __len__(self):
@@ -26,11 +27,29 @@ def __len__(self):
     def __getitem__(self, idx):
         f = self.files[idx]
         item = torch.load(f)
+        program = item['program'] # 1-d LongTensor of MIDI programs 0-255
+        # (128-255 are drums)
         pitch = item['pitch'] # 1-d LongTensor of MIDI pitches 0-127
         time = item['time']
         velocity = item['velocity']
+
         assert len(pitch) == len(time)
 
+        unique_melodic = program.masked_select(program<128).unique()
+        unique_drum = program.masked_select(program>=128).unique()
+
+        # # randomly map instruments to 'anonymous melodic' and 'anonymous drum'
+        for pr in unique_melodic:
+            if torch.rand((1,)) < 0.1:
+                r = torch.randint(self.n_anon, size=(1,))
+                program[program==pr] = r+256
+        for pr in unique_drum:
+            if torch.rand((1,)) < 0.1:
+                r = torch.randint(self.n_anon, size=(1,))
+                program[program==pr] = r+256+self.n_anon
+        # shift from 0-index to general MIDI 1-index; use 0 for start token
+        program += 1
+
         # random transpose avoiding out of range notes
         transpose_down = min(self.transpose, pitch.min().item())
         transpose_up = min(self.transpose, 127-pitch.max())
@@ -39,25 +58,35 @@ def __getitem__(self, idx):
 
         # random speed
         # delta t of first note?
+        time = time.float()
         time = time * (1 + random.random()*self.speed*2 - self.speed)
         # dequantize
         # TODO: use actual tactus from MIDI file?
         time = (
             time + (torch.rand_like(time)-0.5)*2e-3
             ).clamp(0., float('inf'))
 
-        # TODO: random velocity curve?
+        # dequantize velocity
+        velocity = velocity.float()
         velocity = (
             velocity + 
             (torch.rand_like(time)-0.5) * ((velocity>0) & (velocity<127)).float()
             ).clamp(0., 127.)
+        # random velocity curve
+        velocity /= 127
+        velocity = velocity ** (2**(torch.randn((1,))/3))
+        velocity *= 127
 
-        # pad with start, end tokens
-        pad = max(1, self.batch_len-len(pitch))
+        # pad with start tokens, zeros
+        pad = max(0, self.batch_len-len(pitch))
+        program = torch.cat((
+            program.new_full((1,), self.prog_start_token),
+            program,
+            program.new_zeros((pad,))))
         pitch = torch.cat((
             pitch.new_full((1,), self.start_token),
             pitch,
-            pitch.new_full((pad,), self.end_token)))
+            pitch.new_zeros((pad,))))
         time = torch.cat((
             time.new_zeros((1,)),
             time,
@@ -66,16 +95,21 @@ def __getitem__(self, idx):
             velocity.new_zeros((1,)),
             velocity,
             velocity.new_zeros((pad,))))
+        # end signal: nonzero for last event
+        end = torch.zeros_like(program)
+        end[-1] = 1
 
         # random slice
         i = random.randint(0, len(pitch)-self.batch_len)
+        program = program[i:i+self.batch_len]
         pitch = pitch[i:i+self.batch_len]
         time = time[i:i+self.batch_len]
         velocity = velocity[i:i+self.batch_len]
-
-        # time = time.clamp(*self.clamp_time)
+        end = end[i:i+self.batch_len]
 
         return {
+            'end':end,
+            'instrument':program,
             'pitch':pitch,
             'time':time,
             'velocity':velocity