Initial work on the ScriptProcessorNode

Author: orottier

examples/script_processor.rs

@@ -0,0 +1,47 @@
+use rand::Rng;
+
+use web_audio_api::context::{
+    AudioContext, AudioContextLatencyCategory, AudioContextOptions, BaseAudioContext,
+};
+use web_audio_api::node::{AudioNode, AudioScheduledSourceNode};
+
+// ScriptProcessorNode example
+//
+// `cargo run --release --example script_processor`
+//
+// If you are on Linux and use ALSA as audio backend backend, you might want to run
+// the example with the `WEB_AUDIO_LATENCY=playback ` env variable which will
+// increase the buffer size to 1024
+//
+// `WEB_AUDIO_LATENCY=playback cargo run --release --example script_processor`
+fn main() {
+    env_logger::init();
+
+    let latency_hint = match std::env::var("WEB_AUDIO_LATENCY").as_deref() {
+        Ok("playback") => AudioContextLatencyCategory::Playback,
+        _ => AudioContextLatencyCategory::default(),
+    };
+
+    let context = AudioContext::new(AudioContextOptions {
+        latency_hint,
+        ..AudioContextOptions::default()
+    });
+
+    let node = context.create_script_processor(128, 1, 1);
+    node.set_onaudioprocess(|e| {
+        let mut rng = rand::thread_rng();
+        e.output_buffer
+            .get_channel_data_mut(0)
+            .iter_mut()
+            .zip(e.input_buffer.get_channel_data(0))
+            .for_each(|(o, i)| *o = *i + rng.gen_range(-0.3..0.3));
+    });
+
+    let mut src = context.create_oscillator();
+    src.frequency().set_value(400.);
+    src.start();
+    src.connect(&node);
+    node.connect(&context.destination());
+
+    std::thread::sleep(std::time::Duration::from_millis(5000));
+}

src/context/base.rs

@@ -214,6 +214,22 @@ pub trait BaseAudioContext {
         PeriodicWave::new(self.base(), options)
     }
 
+    /// Creates an `ScriptProcessorNode` for custom audio processing (deprecated);
+    #[must_use]
+    fn create_script_processor(
+        &self,
+        buffer_size: usize,
+        number_of_input_channels: usize,
+        number_of_output_channels: usize,
+    ) -> node::ScriptProcessorNode {
+        node::ScriptProcessorNode::new(
+            self.base(),
+            buffer_size,
+            number_of_input_channels,
+            number_of_output_channels,
+        )
+    }
+
     /// Creates an `StereoPannerNode` to pan a stereo output
     #[must_use]
     fn create_stereo_panner(&self) -> node::StereoPannerNode {

src/context/mod.rs

@@ -90,6 +90,7 @@ impl From<u8> for AudioContextState {
 /// This object allows for communication with the render thread and dynamic lifetime management.
 //
 // The only way to construct this object is by calling [`BaseAudioContext::register`]
+#[derive(Clone)]
 pub struct AudioContextRegistration {
     /// the audio context in which nodes and connections lives
     context: ConcreteBaseAudioContext,

src/events.rs

@@ -26,6 +26,7 @@ pub(crate) enum EventType {
     Diagnostics,
     Message(AudioNodeId),
     Complete,
+    AudioProcessing(AudioNodeId),
 }
 
 /// The Error Event interface
@@ -40,6 +41,19 @@ pub struct ErrorEvent {
     pub event: Event,
 }
 
+/// The AudioProcessingEvent interface
+#[non_exhaustive]
+#[derive(Debug)]
+pub struct AudioProcessingEvent {
+    /// The input buffer
+    pub input_buffer: AudioBuffer,
+    /// The output buffer
+    pub output_buffer: AudioBuffer,
+    /// The time when the audio will be played in the same time coordinate system as the
+    /// AudioContext's currentTime.
+    pub playback_time: f64,
+}
+
 /// The OfflineAudioCompletionEvent Event interface
 #[non_exhaustive]
 #[derive(Debug)]
@@ -59,6 +73,7 @@ pub(crate) enum EventPayload {
     Message(Box<dyn Any + Send + 'static>),
     AudioContextState(AudioContextState),
     Complete(AudioBuffer),
+    AudioProcessing(AudioProcessingEvent),
 }
 
 #[derive(Debug)]
@@ -123,6 +138,13 @@ impl EventDispatch {
             payload: EventPayload::Complete(buffer),
         }
     }
+
+    pub fn audio_processing(id: AudioNodeId, value: AudioProcessingEvent) -> Self {
+        EventDispatch {
+            type_: EventType::AudioProcessing(id),
+            payload: EventPayload::AudioProcessing(value),
+        }
+    }
 }
 
 pub(crate) enum EventHandler {

src/lib.rs

@@ -31,7 +31,7 @@ pub mod media_streams;
 pub mod node;
 
 mod events;
-pub use events::{ErrorEvent, Event, OfflineAudioCompletionEvent};
+pub use events::*;
 
 mod message_port;
 pub use message_port::MessagePort;

src/node/mod.rs

@@ -47,6 +47,8 @@ mod oscillator;
 pub use oscillator::*;
 mod panner;
 pub use panner::*;
+mod script_processor;
+pub use script_processor::*;
 mod stereo_panner;
 pub use stereo_panner::*;
 mod waveshaper;

src/node/script_processor.rs

@@ -0,0 +1,182 @@
+use crate::context::{AudioContextRegistration, BaseAudioContext};
+use crate::events::{AudioProcessingEvent, EventHandler, EventPayload, EventType};
+use crate::node::{ChannelCountMode, ChannelInterpretation};
+use crate::render::{
+    AudioParamValues, AudioProcessor, AudioRenderQuantum, AudioWorkletGlobalScope,
+};
+use crate::{AudioBuffer, RENDER_QUANTUM_SIZE};
+
+use super::{AudioNode, ChannelConfig, ChannelConfigOptions};
+
+use std::any::Any;
+
+/// An AudioNode which can generate, process, or analyse audio directly using a script (deprecated)
+#[derive(Debug)]
+pub struct ScriptProcessorNode {
+    registration: AudioContextRegistration,
+    channel_config: ChannelConfig,
+    //  bufferSize MUST be one of the following values: 256, 512, 1024, 2048, 4096, 8192, 16384
+    buffer_size: usize,
+}
+
+impl AudioNode for ScriptProcessorNode {
+    fn registration(&self) -> &AudioContextRegistration {
+        &self.registration
+    }
+
+    fn channel_config(&self) -> &ChannelConfig {
+        &self.channel_config
+    }
+
+    fn number_of_inputs(&self) -> usize {
+        1
+    }
+
+    fn number_of_outputs(&self) -> usize {
+        1
+    }
+
+    // TODO channel config constraints
+}
+
+impl ScriptProcessorNode {
+    pub(crate) fn new<C: BaseAudioContext>(
+        context: &C,
+        buffer_size: usize,
+        number_of_input_channels: usize,
+        number_of_output_channels: usize,
+    ) -> Self {
+        // TODO assert valid arguments
+
+        context.base().register(move |registration| {
+            let render = ScriptProcessorRenderer {
+                buffer: None,
+                buffer_size,
+                number_of_output_channels,
+            };
+
+            let channel_config = ChannelConfigOptions {
+                count: number_of_input_channels,
+                count_mode: ChannelCountMode::Explicit,
+                interpretation: ChannelInterpretation::Speakers,
+            };
+
+            let node = ScriptProcessorNode {
+                registration,
+                channel_config: channel_config.into(),
+                buffer_size,
+            };
+
+            (node, Box::new(render))
+        })
+    }
+
+    pub fn buffer_size(&self) -> usize {
+        self.buffer_size
+    }
+
+    /// Register callback to run when the AudioProcessingEvent is dispatched
+    ///
+    /// The event handler processes audio from the input (if any) by accessing the audio data from
+    /// the inputBuffer attribute. The audio data which is the result of the processing (or the
+    /// synthesized data if there are no inputs) is then placed into the outputBuffer.
+    ///
+    /// Only a single event handler is active at any time. Calling this method multiple times will
+    /// override the previous event handler.
+    pub fn set_onaudioprocess<F: FnMut(&mut AudioProcessingEvent) + Send + 'static>(
+        &self,
+        mut callback: F,
+    ) {
+        let registration = self.registration.clone();
+        let callback = move |v| {
+            let mut payload = match v {
+                EventPayload::AudioProcessing(v) => v,
+                _ => unreachable!(),
+            };
+            callback(&mut payload);
+            registration.post_message(payload.output_buffer);
+        };
+
+        self.context().set_event_handler(
+            EventType::AudioProcessing(self.registration().id()),
+            EventHandler::Multiple(Box::new(callback)),
+        );
+    }
+
+    /// Unset the callback to run when the AudioProcessingEvent is dispatched
+    pub fn clear_onaudioprocess(&self) {
+        self.context()
+            .clear_event_handler(EventType::AudioProcessing(self.registration().id()));
+    }
+}
+
+struct ScriptProcessorRenderer {
+    buffer: Option<AudioRenderQuantum>, // TODO buffer_size
+    buffer_size: usize,
+    number_of_output_channels: usize,
+}
+
+// SAFETY:
+// AudioRenderQuantums are not Send but we promise the `buffer` is None before we ship it to the
+// render thread.
+#[allow(clippy::non_send_fields_in_send_ty)]
+unsafe impl Send for ScriptProcessorRenderer {}
+
+impl AudioProcessor for ScriptProcessorRenderer {
+    fn process(
+        &mut self,
+        inputs: &[AudioRenderQuantum],
+        outputs: &mut [AudioRenderQuantum],
+        _params: AudioParamValues<'_>,
+        scope: &AudioWorkletGlobalScope,
+    ) -> bool {
+        // single input/output node
+        let input = &inputs[0];
+        let output = &mut outputs[0];
+
+        let mut silence = input.clone();
+        silence.make_silent();
+        if let Some(buffer) = self.buffer.replace(silence) {
+            *output = buffer;
+        }
+
+        // TODO buffer_size
+        let input_samples = input.channels().iter().map(|c| c.to_vec()).collect();
+        let input_buffer = AudioBuffer::from(input_samples, scope.sample_rate);
+        let output_samples = vec![vec![0.; RENDER_QUANTUM_SIZE]; self.number_of_output_channels];
+        let output_buffer = AudioBuffer::from(output_samples, scope.sample_rate);
+
+        let playback_time =
+            scope.current_time + (RENDER_QUANTUM_SIZE as f32 / scope.sample_rate) as f64; // TODO
+        scope.send_audio_processing_event(input_buffer, output_buffer, playback_time);
+
+        true // TODO - spec says false but that seems weird
+    }
+
+    fn onmessage(&mut self, msg: &mut dyn Any) {
+        if let Some(buffer) = msg.downcast_mut::<AudioBuffer>() {
+            if let Some(render_quantum) = &mut self.buffer {
+                buffer
+                    .channels()
+                    .iter()
+                    .zip(render_quantum.channels_mut())
+                    .for_each(|(i, o)| o.copy_from_slice(i.as_slice())); // TODO bounds check
+            }
+            return;
+        };
+
+        log::warn!("ScriptProcessorRenderer: Dropping incoming message {msg:?}");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::context::OfflineAudioContext;
+    use float_eq::assert_float_eq;
+
+    #[test]
+    fn test() {
+        // TODO how to test?
+    }
+}

src/render/processor.rs

@@ -1,7 +1,7 @@
 //! Audio processing code that runs on the audio rendering thread
 use crate::context::{AudioNodeId, AudioParamId};
-use crate::events::{ErrorEvent, EventDispatch};
-use crate::{Event, RENDER_QUANTUM_SIZE};
+use crate::events::{AudioProcessingEvent, ErrorEvent, EventDispatch};
+use crate::{AudioBuffer, Event, RENDER_QUANTUM_SIZE};
 
 use super::{graph::Node, AudioRenderQuantum, NodeCollection};
 
@@ -57,6 +57,24 @@ impl AudioWorkletGlobalScope {
             .try_send(EventDispatch::ended(self.node_id.get()));
     }
 
+    pub(crate) fn send_audio_processing_event(
+        &self,
+        input_buffer: AudioBuffer,
+        output_buffer: AudioBuffer,
+        playback_time: f64,
+    ) {
+        if let Some(sender) = self.event_sender.as_ref() {
+            // sending could fail if the channel is saturated or the main thread is shutting down
+            let event = AudioProcessingEvent {
+                input_buffer,
+                output_buffer,
+                playback_time,
+            };
+            let dispatch = EventDispatch::audio_processing(self.node_id.get(), event);
+            let _ = sender.try_send(dispatch);
+        }
+    }
+
     pub(crate) fn report_error(&self, error: Box<dyn Any + Send>) {
         pub fn type_name_of_val<T: ?Sized>(_val: &T) -> &'static str {
             std::any::type_name::<T>()