From a6913d129b9ffe7cfd0a4a5e727d1cd3f38f51e0 Mon Sep 17 00:00:00 2001 From: b-ma Date: Thu, 19 Sep 2024 11:13:02 +0200 Subject: [PATCH 1/3] make start time sticky to sample time when inside floating point error zone, cf. buffer stitching wpt test make start time sticky to sample time when inside floating point error zone cf. buffer stitching from wpt --- Cargo.toml | 1 + src/node/audio_buffer_source.rs | 55 ++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6f3a4182..43d407e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ include = [ rust-version = "1.71" [dependencies] +almost = "0.2.0" arc-swap = "1.6" arrayvec = "0.7" cpal = { version = "0.15", optional = true } diff --git a/src/node/audio_buffer_source.rs b/src/node/audio_buffer_source.rs index 7258aae4..d58965bf 100644 --- a/src/node/audio_buffer_source.rs +++ b/src/node/audio_buffer_source.rs @@ -633,12 +633,20 @@ impl AudioProcessor for AudioBufferSourceRenderer { for (i, playback_info) in playback_infos.iter_mut().enumerate() { let current_time = block_time + i as f64 * dt; + // handle floating point errors due to start time computation + // cf. test_subsample_buffer_stitching + if !self.render_state.started { + if almost::equal(current_time, self.start_time) { + self.start_time = current_time; + } + } + // Handle following cases: // - we are before start time // - we are after stop time // - explicit duration (in buffer time reference) has been given and we have reached it // Note that checking against buffer duration is done below to handle looping - if current_time < self.start_time + if (current_time < self.start_time) || current_time >= self.stop_time || self.render_state.buffer_time_elapsed >= self.duration { @@ -647,6 +655,7 @@ impl AudioProcessor for AudioBufferSourceRenderer { // we have now reached start time if !self.render_state.started { + // println!("start, {}, {}, {}", current_time, self.start_time, (current_time - self.start_time).abs()); let delta = current_time - self.start_time; // handle that start time may be between last sample and this one self.offset += delta; @@ -835,6 +844,7 @@ mod tests { use std::sync::{Arc, Mutex}; use crate::context::{BaseAudioContext, OfflineAudioContext}; + use crate::AudioBufferOptions; use crate::RENDER_QUANTUM_SIZE; use super::*; @@ -1817,6 +1827,49 @@ mod tests { assert_float_eq!(channel[..], expected[..], abs_all <= 0.); } + #[test] + // ported from wpt: the-audiobuffersourcenode-interface/sub-sample-buffer-stitching.html + fn test_subsample_buffer_stitching() { + let sample_rate = 44_100.; + let buffer_rate = 44_100.; + let buffer_length = 30; + let frequency = 440.; + + let length = buffer_length * 15; + let mut context = OfflineAudioContext::new(1, length, sample_rate); + + let mut wave_signal = vec![0.; context.length()]; + let omega = 2. * PI / buffer_rate * frequency; + + wave_signal.iter_mut().enumerate().for_each(|(i, s)| { + *s = (omega * i as f32).sin(); + }); + + // Slice the sine wave into many little buffers to be assigned to ABSNs + // that are started at the appropriate times to produce a final sine + // wave. + for k in (0..context.length()).step_by(buffer_length) { + let mut buffer = AudioBuffer::new(AudioBufferOptions { + number_of_channels: 1, + length: buffer_length, + sample_rate, + }); + buffer.copy_to_channel(&wave_signal[k..k + buffer_length], 0); + + let mut src = AudioBufferSourceNode::new(&context, AudioBufferSourceOptions { + buffer: Some(buffer), + ..Default::default() + }); + src.connect(&context.destination()); + src.start_at(k as f64 / buffer_rate as f64); + } + + let result = context.start_rendering_sync(); + let channel = result.get_channel_data(0); + + assert_float_eq!(channel[..], wave_signal[..], abs_all <= 1e-9); + } + #[test] fn test_onended_before_drop() { let sample_rate = 48_000.; From aa4277770be96c1f484e7a342105598a713a3570 Mon Sep 17 00:00:00 2001 From: b-ma Date: Thu, 19 Sep 2024 18:51:03 +0200 Subject: [PATCH 2/3] make start time sticky to sample if in floating point error range + extrapolate sample after end of buffer in certain conditions --- src/node/audio_buffer_source.rs | 189 ++++++++++++++++++-------------- 1 file changed, 109 insertions(+), 80 deletions(-) diff --git a/src/node/audio_buffer_source.rs b/src/node/audio_buffer_source.rs index d58965bf..ae11a337 100644 --- a/src/node/audio_buffer_source.rs +++ b/src/node/audio_buffer_source.rs @@ -633,12 +633,10 @@ impl AudioProcessor for AudioBufferSourceRenderer { for (i, playback_info) in playback_infos.iter_mut().enumerate() { let current_time = block_time + i as f64 * dt; - // handle floating point errors due to start time computation + // Sticky behavior to handle floating point errors due to start time computation // cf. test_subsample_buffer_stitching - if !self.render_state.started { - if almost::equal(current_time, self.start_time) { - self.start_time = current_time; - } + if !self.render_state.started && almost::equal(current_time, self.start_time) { + self.start_time = current_time; } // Handle following cases: @@ -655,7 +653,6 @@ impl AudioProcessor for AudioBufferSourceRenderer { // we have now reached start time if !self.render_state.started { - // println!("start, {}, {}, {}", current_time, self.start_time, (current_time - self.start_time).abs()); let delta = current_time - self.start_time; // handle that start time may be between last sample and this one self.offset += delta; @@ -745,8 +742,9 @@ impl AudioProcessor for AudioBufferSourceRenderer { let next_sample = match buffer_channel.get(prev_frame_index + 1) { Some(val) => *val as f64, + // End of buffer None => { - let sample = if is_looping { + if is_looping { if playback_rate >= 0. { let start_playhead = actual_loop_start * sample_rate; @@ -758,18 +756,31 @@ impl AudioProcessor for AudioBufferSourceRenderer { start_playhead as usize + 1 }; - buffer_channel[start_index] + buffer_channel[start_index] as f64 } else { let end_playhead = actual_loop_end * sample_rate; let end_index = end_playhead as usize; - buffer_channel[end_index] + buffer_channel[end_index] as f64 } } else { - 0. - }; - - sample as f64 + // Handle 2 edge cases: + // 1. We are in a case where buffer time is below buffer + // duration due to floating point errors, but where + // prev_frame_index is last index and k is near 1. We can't + // filter this case before, because it might break + // loops logic. + // 2. Buffer contains only one sample + if almost::equal(*k, 1.) || *prev_frame_index == 0 { + 0. + } else { + // Extrapolate next sample using the last two known samples + // cf. https://github.com/WebAudio/web-audio-api/issues/2032 + let prev_prev_sample = + buffer_channel[*prev_frame_index - 1]; + 2. * prev_sample - prev_prev_sample as f64 + } + } } }; @@ -1130,44 +1141,47 @@ mod tests { #[test] fn test_audio_buffer_resampling() { - [22_500, 38_000, 48_000, 96_000].iter().for_each(|sr| { - let base_sr = 44_100; - let mut context = OfflineAudioContext::new(1, base_sr, base_sr as f32); - - // 1Hz sine at different sample rates - let buf_sr = *sr; - // safe cast for sample rate, see discussion at #113 - let sample_rate = buf_sr as f32; - let mut buffer = context.create_buffer(1, buf_sr, sample_rate); - let mut sine = vec![]; - - for i in 0..buf_sr { - let phase = i as f32 / buf_sr as f32 * 2. * PI; - let sample = phase.sin(); - sine.push(sample); - } + [22_500, 38_000, 43_800, 48_000, 96_000] + .iter() + .for_each(|sr| { + let freq = 1.; + let base_sr = 44_100; + let mut context = OfflineAudioContext::new(1, base_sr, base_sr as f32); + + // 1Hz sine at different sample rates + let buf_sr = *sr; + // safe cast for sample rate, see discussion at #113 + let sample_rate = buf_sr as f32; + let mut buffer = context.create_buffer(1, buf_sr, sample_rate); + let mut sine = vec![]; + + for i in 0..buf_sr { + let phase = freq * i as f32 / buf_sr as f32 * 2. * PI; + let sample = phase.sin(); + sine.push(sample); + } - buffer.copy_to_channel(&sine[..], 0); + buffer.copy_to_channel(&sine[..], 0); - let mut src = context.create_buffer_source(); - src.connect(&context.destination()); - src.set_buffer(buffer); - src.start_at(0. / sample_rate as f64); + let mut src = context.create_buffer_source(); + src.connect(&context.destination()); + src.set_buffer(buffer); + src.start_at(0. / sample_rate as f64); - let result = context.start_rendering_sync(); - let channel = result.get_channel_data(0); + let result = context.start_rendering_sync(); + let channel = result.get_channel_data(0); - // 1Hz sine at audio context sample rate - let mut expected = vec![]; + // 1Hz sine at audio context sample rate + let mut expected = vec![]; - for i in 0..base_sr { - let phase = i as f32 / base_sr as f32 * 2. * PI; - let sample = phase.sin(); - expected.push(sample); - } + for i in 0..base_sr { + let phase = freq * i as f32 / base_sr as f32 * 2. * PI; + let sample = phase.sin(); + expected.push(sample); + } - assert_float_eq!(channel[..], expected[..], abs_all <= 1e-6); - }); + assert_float_eq!(channel[..], expected[..], abs_all <= 1e-6); + }); } #[test] @@ -1273,7 +1287,7 @@ mod tests { } #[test] - fn test_end_of_file_slow_track() { + fn test_end_of_file_slow_track_1() { let sample_rate = 48_000.; let mut context = OfflineAudioContext::new(1, RENDER_QUANTUM_SIZE * 2, sample_rate); @@ -1828,46 +1842,61 @@ mod tests { } #[test] - // ported from wpt: the-audiobuffersourcenode-interface/sub-sample-buffer-stitching.html + // Ported from wpt: the-audiobuffersourcenode-interface/sub-sample-buffer-stitching.html + // Note that in wpt, results are tested against an oscillator node, which fails + // in the (44_100., 43_800., 3.8986e-3) condition for some (yet) unknown reason fn test_subsample_buffer_stitching() { - let sample_rate = 44_100.; - let buffer_rate = 44_100.; - let buffer_length = 30; - let frequency = 440.; + [(44_100., 44_100., 9.0957e-5), (44_100., 43_800., 3.8986e-3)] + .iter() + .for_each(|(sample_rate, buffer_rate, error_threshold)| { + let sample_rate = *sample_rate; + let buffer_rate = *buffer_rate; + let buffer_length = 30; + let frequency = 440.; + + // let length = sample_rate as usize; + let length = buffer_length * 15; + let mut context = OfflineAudioContext::new(2, length, sample_rate); + + let mut wave_signal = vec![0.; context.length()]; + let omega = 2. * PI / buffer_rate * frequency; + wave_signal.iter_mut().enumerate().for_each(|(i, s)| { + *s = (omega * i as f32).sin(); + }); - let length = buffer_length * 15; - let mut context = OfflineAudioContext::new(1, length, sample_rate); + // Slice the sine wave into many little buffers to be assigned to ABSNs + // that are started at the appropriate times to produce a final sine + // wave. + for k in (0..context.length()).step_by(buffer_length) { + let mut buffer = AudioBuffer::new(AudioBufferOptions { + number_of_channels: 1, + length: buffer_length, + sample_rate: buffer_rate, + }); + buffer.copy_to_channel(&wave_signal[k..k + buffer_length], 0); + + let mut src = AudioBufferSourceNode::new( + &context, + AudioBufferSourceOptions { + buffer: Some(buffer), + ..Default::default() + }, + ); + src.connect(&context.destination()); + src.start_at(k as f64 / buffer_rate as f64); + } - let mut wave_signal = vec![0.; context.length()]; - let omega = 2. * PI / buffer_rate * frequency; + let mut expected = vec![0.; context.length()]; + let omega = 2. * PI / sample_rate * frequency; + expected.iter_mut().enumerate().for_each(|(i, s)| { + *s = (omega * i as f32).sin(); + }); - wave_signal.iter_mut().enumerate().for_each(|(i, s)| { - *s = (omega * i as f32).sin(); - }); + let result = context.start_rendering_sync(); + let actual = result.get_channel_data(0); - // Slice the sine wave into many little buffers to be assigned to ABSNs - // that are started at the appropriate times to produce a final sine - // wave. - for k in (0..context.length()).step_by(buffer_length) { - let mut buffer = AudioBuffer::new(AudioBufferOptions { - number_of_channels: 1, - length: buffer_length, - sample_rate, + assert_float_eq!(actual[..], expected[..], abs_all <= error_threshold); }); - buffer.copy_to_channel(&wave_signal[k..k + buffer_length], 0); - - let mut src = AudioBufferSourceNode::new(&context, AudioBufferSourceOptions { - buffer: Some(buffer), - ..Default::default() - }); - src.connect(&context.destination()); - src.start_at(k as f64 / buffer_rate as f64); - } - - let result = context.start_rendering_sync(); - let channel = result.get_channel_data(0); - - assert_float_eq!(channel[..], wave_signal[..], abs_all <= 1e-9); } #[test] From f719fd76047595c038020d00ad53c39e64ba4599 Mon Sep 17 00:00:00 2001 From: b-ma Date: Thu, 19 Sep 2024 19:01:21 +0200 Subject: [PATCH 3/3] cleaning --- src/node/audio_buffer_source.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/audio_buffer_source.rs b/src/node/audio_buffer_source.rs index ae11a337..516338b3 100644 --- a/src/node/audio_buffer_source.rs +++ b/src/node/audio_buffer_source.rs @@ -644,7 +644,7 @@ impl AudioProcessor for AudioBufferSourceRenderer { // - we are after stop time // - explicit duration (in buffer time reference) has been given and we have reached it // Note that checking against buffer duration is done below to handle looping - if (current_time < self.start_time) + if current_time < self.start_time || current_time >= self.stop_time || self.render_state.buffer_time_elapsed >= self.duration {