Merge pull request #314 from uklotzde/decoding

Fix and extend (AAC/M4A/ALAC) decoding

Author: orottier

Cargo.toml

@@ -50,11 +50,17 @@ name = "my_benchmark"
 harness = false
 
 [features]
-default = ["mp3", "ogg", "flac", "wav", "cpal"]
+default = ["mp3", "ogg", "flac", "wav", "m4a", "alac", "cpal"]
 mp3 = ["symphonia/mp3", "creek/decode-mp3"]
 ogg = ["symphonia/ogg", "symphonia/vorbis", "creek/decode-ogg", "creek/decode-vorbis"]
 flac = ["symphonia/flac", "creek/decode-flac"]
 wav = ["symphonia/wav", "symphonia/pcm", "creek/decode-wav", "creek/decode-pcm"]
+# TODO(m4a): Add "creek/decode-aac" after <https://github.com/MeadowlarkDAW/creek/pull/22> has been merged
+aac = ["symphonia/aac"]
+# TODO(m4a): Add "creek/decode-isomp4" after <https://github.com/MeadowlarkDAW/creek/pull/22> has been merged
+m4a = ["aac", "symphonia/isomp4"]
+# TODO(alac): Add "creek/decode-alac" and "creek/decode-isomp4" after <https://github.com/MeadowlarkDAW/creek/pull/22> has been merged
+alac = ["symphonia/alac", "symphonia/isomp4"]
 cpal = ["dep:cpal"]
 cubeb = ["dep:cubeb"]
 cpal-jack = ["cpal", "cpal/jack"]

examples/decoding.rs

@@ -22,14 +22,23 @@ fn main() {
         "samples/sample.flac",
         "samples/sample.ogg",
         "samples/sample.mp3",
+        "samples/sample-aac.m4a",
+        "samples/sample-alac.m4a",
         // cannot decode, format not supported or file corrupted
         "samples/empty_2c.wav",
         "samples/corrupt.wav",
         "samples/sample.aiff",
         "samples/sample.webm", // 48kHz,
     ];
 
-    let latency_hint = match std::env::var("WEB_AUDIO_LATENCY").as_deref() {
+    let latency_hint = match std::env::var("WEB_AUDIO_LATENCY")
+        .as_deref()
+        .map(str::trim)
+        .map(str::to_ascii_lowercase)
+        .as_deref()
+    {
+        Ok("interactive") => AudioContextLatencyCategory::Interactive,
+        Ok("balanced") => AudioContextLatencyCategory::Balanced,
         Ok("playback") => AudioContextLatencyCategory::Playback,
         _ => AudioContextLatencyCategory::default(),
     };

samples/sample-aac.m4a

@@ -5,7 +5,7 @@ use crate::buffer::{AudioBuffer, ChannelData};
 
 use symphonia::core::audio::AudioBufferRef;
 use symphonia::core::audio::Signal;
-use symphonia::core::codecs::{Decoder, DecoderOptions};
+use symphonia::core::codecs::{Decoder, DecoderOptions, FinalizeResult};
 use symphonia::core::conv::FromSample;
 use symphonia::core::errors::Error as SymphoniaError;
 use symphonia::core::formats::{FormatOptions, FormatReader};
@@ -53,6 +53,8 @@ impl<R: Read + Send + Sync> symphonia::core::io::MediaSource for MediaInput<R> {
 pub(crate) struct MediaDecoder {
     format: Box<dyn FormatReader>,
     decoder: Box<dyn Decoder>,
+    track_index: usize,
+    packet_count: usize,
 }
 
 impl MediaDecoder {
@@ -64,24 +66,27 @@ impl MediaDecoder {
     pub fn try_new<R: std::io::Read + Send + Sync + 'static>(
         input: R,
     ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
-        // Symfonia lib needs a Box<dyn MediaSource> - use our own MediaInput
+        // Symphonia lib needs a Box<dyn MediaSource> - use our own MediaInput
         let input = Box::new(MediaInput::new(input));
 
         // Create the media source stream using the boxed media source from above.
-        let mss = symphonia::core::io::MediaSourceStream::new(input, Default::default());
+        let stream = symphonia::core::io::MediaSourceStream::new(input, Default::default());
 
         // Create a hint to help the format registry guess what format reader is appropriate. In this
         // function we'll leave it empty.
         let hint = Hint::new();
 
-        // Use the default options when reading and decoding.
+        // TODO: Allow to customize some options.
         let format_opts: FormatOptions = Default::default();
         let metadata_opts: MetadataOptions = Default::default();
-        let decoder_opts: DecoderOptions = Default::default();
+        let decoder_opts = DecoderOptions {
+            // Opt-in to verify the decoded data against the checksums in the container.
+            verify: true,
+        };
 
         // Probe the media source stream for a format.
         let probed =
-            symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?;
+            symphonia::default::get_probe().format(&hint, stream, &format_opts, &metadata_opts)?;
 
         // Get the format reader yielded by the probe operation.
         let format = probed.format;
@@ -90,117 +95,140 @@ impl MediaDecoder {
         let track = format.default_track().ok_or(SymphoniaError::Unsupported(
             "no default media track available",
         ))?;
-
+        let track_index = format
+            .tracks()
+            .iter()
+            .position(|t| t.id == track.id)
+            .unwrap();
         // Create a (stateful) decoder for the track.
         let decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?;
 
-        Ok(Self { format, decoder })
+        Ok(Self {
+            format,
+            decoder,
+            track_index,
+            packet_count: 0,
+        })
     }
 }
 
 impl Iterator for MediaDecoder {
     type Item = Result<AudioBuffer, Box<dyn Error + Send + Sync>>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        let format = &mut self.format;
-        let decoder = &mut self.decoder;
-
-        // Get the default track.
-        let track = format.default_track().unwrap();
-        let number_of_channels = track.codec_params.channels.unwrap().count();
-        let input_sample_rate = track.codec_params.sample_rate.unwrap() as f32;
-
-        // Store the track identifier, we'll use it to filter packets.
+        let Self {
+            format,
+            decoder,
+            track_index,
+            packet_count,
+        } = self;
+
+        // Get the track.
+        let track = format.tracks().get(*track_index)?;
         let track_id = track.id;
 
         loop {
             // Get the next packet from the format reader.
             let packet = match format.next_packet() {
-                Err(e) => {
-                    log::error!("next packet err {:?}", e);
-                    return None;
+                Err(err) => {
+                    if let SymphoniaError::IoError(err) = &err {
+                        if err.kind() == std::io::ErrorKind::UnexpectedEof {
+                            // End of stream
+                            log::debug!("Decoding finished after {packet_count} packet(s)");
+                            let FinalizeResult { verify_ok } = decoder.finalize();
+                            if verify_ok == Some(false) {
+                                log::warn!("Verification of decoded data failed");
+                            }
+                            return None;
+                        }
+                    }
+                    log::warn!(
+                        "Failed to fetch next packet following packet #{packet_count}: {err}"
+                    );
+                    return Some(Err(Box::new(err)));
+                }
+                Ok(packet) => {
+                    *packet_count += 1;
+                    packet
                 }
-                Ok(p) => p,
             };
 
             // If the packet does not belong to the selected track, skip it.
-            if packet.track_id() != track_id {
+            let packet_track_id = packet.track_id();
+            if packet_track_id != track_id {
+                log::debug!(
+                    "Skipping packet from other track {packet_track_id} while decoding track {track_id}"
+                );
                 continue;
             }
 
-            // Decode the packet into audio samples, ignoring any decode errors.
+            // Decode the packet into audio samples.
             match decoder.decode(&packet) {
-                Ok(audio_buf) => {
-                    let output = convert_buf(audio_buf, number_of_channels, input_sample_rate);
+                Ok(input) => {
+                    let output = convert_buf(input);
                     return Some(Ok(output));
                 }
-                Err(SymphoniaError::DecodeError(e)) => {
-                    // Todo: treat decoding errors as fatal or move to next packet? Context:
-                    // https://github.com/RustAudio/rodio/issues/401#issuecomment-974747404
-                    log::error!("Symphonia DecodeError {:?} - abort stream", e);
-                    return Some(Err(Box::new(SymphoniaError::DecodeError(e))));
+                Err(SymphoniaError::DecodeError(err)) => {
+                    // Recoverable error, continue with the next packet.
+                    log::warn!("Failed to decode packet #{packet_count}: {err}");
                 }
-                Err(SymphoniaError::IoError(e))
-                    if e.kind() == std::io::ErrorKind::UnexpectedEof =>
-                {
-                    // this happens for Wav-files, running into EOF is expected
+                Err(SymphoniaError::IoError(err)) => {
+                    // Recoverable error, continue with the next packet.
+                    log::warn!("I/O error while decoding packet #{packet_count}: {err}");
                 }
-                Err(e) => {
-                    // do not continue processing, return error result
-                    return Some(Err(Box::new(e)));
+                Err(err) => {
+                    // All other errors are considered fatal and decoding must be aborted.
+                    return Some(Err(Box::new(err)));
                 }
             };
         }
     }
 }
 
 /// Convert a Symphonia AudioBufferRef to our own AudioBuffer
-fn convert_buf(
-    input: AudioBufferRef<'_>,
-    number_of_channels: usize,
-    input_sample_rate: f32,
-) -> AudioBuffer {
-    let chans = 0..number_of_channels;
+fn convert_buf(input: AudioBufferRef<'_>) -> AudioBuffer {
+    let channels = 0..input.spec().channels.count();
+    let sample_rate = input.spec().rate as f32;
 
     // This looks a bit awkward but this may be the only way to get the f32 samples
     // out without making double copies.
     use symphonia::core::audio::AudioBufferRef::*;
 
     let data: Vec<Vec<f32>> = match input {
-        U8(buf) => chans
+        U8(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        U16(buf) => chans
+        U16(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        U24(buf) => chans
+        U24(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        U32(buf) => chans
+        U32(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        S8(buf) => chans
+        S8(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        S16(buf) => chans
+        S16(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        S24(buf) => chans
+        S24(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        S32(buf) => chans
+        S32(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        F32(buf) => chans
+        F32(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
-        F64(buf) => chans
+        F64(buf) => channels
             .map(|i| buf.chan(i).iter().copied().map(f32::from_sample).collect())
             .collect(),
     };
 
     let channels = data.into_iter().map(ChannelData::from).collect();
-    AudioBuffer::from_channels(channels, input_sample_rate)
+    AudioBuffer::from_channels(channels, sample_rate)
 }
 
 #[cfg(test)]