fix: decompress concatenated gzip, bzip2, and lz4 streams

Fixes #855

Previously, ouch would only decompress the first frame/stream of
concatenated compressed files. This is a valid format for gzip, bzip2,
and lz4 where multiple compressed streams are concatenated together.

Changes:
- Use MultiGzDecoder instead of GzDecoder for gzip
- Use MultiBzDecoder instead of BzDecoder for bzip2
- Add MultiFrameLz4Decoder wrapper to handle concatenated lz4 frames
- Add integration tests for concatenated stream decompression

Author: njenia

CHANGELOG.md

@@ -44,6 +44,7 @@ Categories Used:
 
 ### Bug Fixes
 
+- Fix decompression of concatenated gzip, bzip2, and lz4 streams [\#855](https://github.com/ouch-org/ouch/issues/855)
 - Fix 7z BadSignature error when compressing and then listing [\#819](https://github.com/ouch-org/ouch/pull/819) ([tommady](https://github.com/tommady))
 - Fix tar extraction count when --quiet [\#824](https://github.com/ouch-org/ouch/pull/824) ([marcospb19](https://github.com/marcospb19))
 - Fix unpacking with merge flag failing without --dir flag [\#826](https://github.com/ouch-org/ouch/pull/826) ([tommady](https://github.com/tommady))

src/commands/decompress.rs

@@ -18,7 +18,7 @@ use crate::{
     info, info_accessible,
     utils::{
         self, io::lock_and_flush_output_stdio, is_path_stdin, nice_directory_display, set_permission_mode,
-        user_wants_to_continue,
+        user_wants_to_continue, MultiFrameLz4Decoder,
     },
     QuestionAction, QuestionPolicy, BUFFER_CAPACITY,
 };
@@ -123,16 +123,16 @@ pub fn decompress_file(options: DecompressOptions) -> crate::Result<()> {
     // Grab previous decoder and wrap it inside of a new one
     let chain_reader_decoder = |format: &CompressionFormat, decoder: Box<dyn Read>| -> crate::Result<Box<dyn Read>> {
         let decoder: Box<dyn Read> = match format {
-            Gzip => Box::new(flate2::read::GzDecoder::new(decoder)),
-            Bzip => Box::new(bzip2::read::BzDecoder::new(decoder)),
+            Gzip => Box::new(flate2::read::MultiGzDecoder::new(decoder)),
+            Bzip => Box::new(bzip2::read::MultiBzDecoder::new(decoder)),
             Bzip3 => {
                 #[cfg(not(feature = "bzip3"))]
                 return Err(archive::bzip3_stub::no_support());
 
                 #[cfg(feature = "bzip3")]
                 Box::new(bzip3::read::Bz3Decoder::new(decoder)?)
             }
-            Lz4 => Box::new(lz4_flex::frame::FrameDecoder::new(decoder)),
+            Lz4 => Box::new(MultiFrameLz4Decoder::new(decoder)?),
             Lzma => Box::new(lzma_rust2::LzmaReader::new_mem_limit(decoder, u32::MAX, None)?),
             Xz => Box::new(lzma_rust2::XzReader::new(decoder, true)),
             Lzip => Box::new(lzma_rust2::LzipReader::new(decoder)?),

src/commands/list.rs

@@ -10,7 +10,7 @@ use crate::{
     commands::warn_user_about_loading_zip_in_memory,
     extension::CompressionFormat::{self, *},
     list::{self, FileInArchive, ListOptions},
-    utils::{io::lock_and_flush_output_stdio, user_wants_to_continue},
+    utils::{io::lock_and_flush_output_stdio, user_wants_to_continue, MultiFrameLz4Decoder},
     QuestionAction, QuestionPolicy, BUFFER_CAPACITY,
 };
 
@@ -47,16 +47,16 @@ pub fn list_archive_contents(
     let chain_reader_decoder =
         |format: CompressionFormat, decoder: Box<dyn Read + Send>| -> crate::Result<Box<dyn Read + Send>> {
             let decoder: Box<dyn Read + Send> = match format {
-                Gzip => Box::new(flate2::read::GzDecoder::new(decoder)),
-                Bzip => Box::new(bzip2::read::BzDecoder::new(decoder)),
+                Gzip => Box::new(flate2::read::MultiGzDecoder::new(decoder)),
+                Bzip => Box::new(bzip2::read::MultiBzDecoder::new(decoder)),
                 Bzip3 => {
                     #[cfg(not(feature = "bzip3"))]
                     return Err(archive::bzip3_stub::no_support());
 
                     #[cfg(feature = "bzip3")]
                     Box::new(bzip3::read::Bz3Decoder::new(decoder).unwrap())
                 }
-                Lz4 => Box::new(lz4_flex::frame::FrameDecoder::new(decoder)),
+                Lz4 => Box::new(MultiFrameLz4Decoder::new(decoder)?),
                 Lzma => Box::new(lzma_rust2::LzmaReader::new_mem_limit(decoder, u32::MAX, None)?),
                 Xz => Box::new(lzma_rust2::XzReader::new(decoder, true)),
                 Lzip => Box::new(lzma_rust2::LzipReader::new(decoder)?),

src/utils/io.rs

@@ -1,4 +1,4 @@
-use std::io::{self, stderr, stdout, StderrLock, StdoutLock, Write};
+use std::io::{self, stderr, stdout, Read, StderrLock, StdoutLock, Write};
 
 use crate::utils::logger;
 
@@ -14,3 +14,77 @@ pub fn lock_and_flush_output_stdio() -> io::Result<StdioOutputLocks> {
 
     Ok((stdout, stderr))
 }
+
+/// A wrapper around lz4_flex::frame::FrameDecoder that handles concatenated lz4 frames.
+/// The standard FrameDecoder only reads a single frame and returns EOF.
+/// This wrapper continues reading subsequent frames until the underlying reader is exhausted.
+pub struct MultiFrameLz4Decoder {
+    buffer: io::Cursor<Vec<u8>>,
+}
+
+impl MultiFrameLz4Decoder {
+    pub fn new(mut reader: impl Read) -> io::Result<Self> {
+        // Decompress all concatenated frames into an in-memory buffer
+        let mut output = Vec::new();
+        let mut input_buffer = Vec::new();
+        reader.read_to_end(&mut input_buffer)?;
+
+        let mut cursor = io::Cursor::new(input_buffer);
+
+        // LZ4 frame magic number (little-endian)
+        const LZ4_MAGIC: [u8; 4] = [0x04, 0x22, 0x4D, 0x18];
+
+        loop {
+            let pos = cursor.position() as usize;
+            let remaining = cursor.get_ref().len() - pos;
+
+            if remaining == 0 {
+                break;
+            }
+
+            if remaining < 4 {
+                return Err(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    "Incomplete LZ4 frame header",
+                ));
+            }
+
+            // Check for magic number
+            let slice = &cursor.get_ref()[pos..pos + 4];
+            if slice != LZ4_MAGIC {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "Invalid LZ4 frame magic number",
+                ));
+            }
+
+            // Create a decoder for this frame starting from current position
+            cursor.set_position(pos as u64);
+            let frame_reader = io::Cursor::new(&cursor.get_ref()[pos..]);
+            let mut decoder = lz4_flex::frame::FrameDecoder::new(frame_reader);
+
+            // Read the frame
+            let start_len = output.len();
+            decoder.read_to_end(&mut output)?;
+
+            // Get how many bytes were consumed from the input
+            let bytes_consumed = decoder.into_inner().position() as usize;
+            cursor.set_position((pos + bytes_consumed) as u64);
+
+            // If no progress was made, break to avoid infinite loop
+            if output.len() == start_len && bytes_consumed == 0 {
+                break;
+            }
+        }
+
+        Ok(Self {
+            buffer: io::Cursor::new(output),
+        })
+    }
+}
+
+impl Read for MultiFrameLz4Decoder {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.buffer.read(buf)
+    }
+}

src/utils/mod.rs

@@ -8,6 +8,7 @@ mod file_visibility;
 mod formatting;
 mod fs;
 pub mod io;
+pub use io::MultiFrameLz4Decoder;
 pub mod logger;
 mod question;
 pub mod threads;

tests/integration.rs

@@ -1209,3 +1209,88 @@ fn tar_hardlink_pack_and_unpack() {
     assert_eq!(out_source_meta.ino(), out_link1_meta.ino());
     assert_eq!(out_link1_meta.ino(), out_link2_meta.ino());
 }
+
+/// Helper function to test decompression of concatenated streams (issue #855).
+/// Takes a file extension and a compression function that compresses a single chunk.
+fn test_concatenated_streams(extension: &str, compress_chunk: impl Fn(&[u8]) -> Vec<u8>) {
+    use std::io::Write;
+
+    let temp_dir = tempdir().unwrap();
+    let root_path = temp_dir.path();
+
+    // Create content for three separate streams
+    let chunks: &[&[u8]] = &[
+        b"First stream content - this is stream 1\n",
+        b"Second stream content - this is stream 2\n",
+        b"Third stream content - this is stream 3\n",
+    ];
+
+    // Create the concatenated file
+    let concatenated_path = root_path.join(format!("concatenated.{extension}"));
+    {
+        let mut file = fs::File::create(&concatenated_path).unwrap();
+        for chunk in chunks {
+            file.write_all(&compress_chunk(chunk)).unwrap();
+        }
+    }
+
+    // Decompress using ouch
+    crate::utils::cargo_bin()
+        .arg("decompress")
+        .arg(&concatenated_path)
+        .arg("-d")
+        .arg(root_path)
+        .arg("--yes")
+        .assert()
+        .success();
+
+    // Verify the output contains all streams
+    let output_path = root_path.join("concatenated");
+    let output_content = fs::read(&output_path).unwrap();
+    let expected_content: Vec<u8> = chunks.iter().flat_map(|c| c.iter().copied()).collect();
+    assert_eq!(
+        output_content, expected_content,
+        "Decompressed content should contain all concatenated {extension} streams"
+    );
+}
+
+/// Test that concatenated gzip streams are fully decompressed (issue #855)
+#[test]
+fn decompress_concatenated_gzip_streams() {
+    use flate2::write::GzEncoder;
+    use flate2::Compression;
+    use std::io::Write;
+
+    test_concatenated_streams("gz", |data| {
+        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(data).unwrap();
+        encoder.finish().unwrap()
+    });
+}
+
+/// Test that concatenated bzip2 streams are fully decompressed (related to issue #855)
+#[test]
+fn decompress_concatenated_bzip2_streams() {
+    use bzip2::write::BzEncoder;
+    use bzip2::Compression;
+    use std::io::Write;
+
+    test_concatenated_streams("bz2", |data| {
+        let mut encoder = BzEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(data).unwrap();
+        encoder.finish().unwrap()
+    });
+}
+
+/// Test that concatenated lz4 frames are fully decompressed (related to issue #855)
+#[test]
+fn decompress_concatenated_lz4_frames() {
+    use lz4_flex::frame::FrameEncoder;
+    use std::io::Write;
+
+    test_concatenated_streams("lz4", |data| {
+        let mut encoder = FrameEncoder::new(Vec::new());
+        encoder.write_all(data).unwrap();
+        encoder.finish().unwrap()
+    });
+}