feat(oma-fetch,oma-refresh): support lzma and lz4 file download and decompress

Author: eatradish

Cargo.lock

@@ -15,7 +15,7 @@ faster-hex = "0.10"
 sha2 = "0.10"
 futures = "0.3"
 # FIXME: issue https://github.com/AOSC-Dev/oma/issues/547, workaround downgrade to 0.4.27
-async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd"] }
+async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd", "lz4"] }
 # Note: Use the `uncheck_liblzma_version` feature here to force dynamic linking
 # even with older liblzma (Ubuntu 22.04 uses 5.2.x). Be mindful, however, that
 # async-compression depends on a different liblzma-rs than what is specified here.

oma-fetch/Cargo.toml

@@ -6,7 +6,9 @@ use std::{
     time::Duration,
 };
 
-use async_compression::futures::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder};
+use async_compression::futures::bufread::{
+    BzDecoder, GzipDecoder, Lz4Decoder, LzmaDecoder, XzDecoder, ZstdDecoder,
+};
 use bon::bon;
 use futures::{AsyncRead, TryStreamExt, io::BufReader};
 use reqwest::{
@@ -477,10 +479,10 @@ impl<'a> SingleDownloader<'a> {
             CompressFile::Xz => &mut XzDecoder::new(BufReader::new(bytes_stream)),
             CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(bytes_stream)),
             CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(bytes_stream)),
-            CompressFile::Nothing => &mut BufReader::new(bytes_stream),
             CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(bytes_stream)),
-            CompressFile::Lzma => unimplemented!(),
-            CompressFile::Lz4 => unimplemented!(),
+            CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(bytes_stream)),
+            CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(bytes_stream)),
+            CompressFile::Nothing => &mut BufReader::new(bytes_stream),
         };
 
         let mut reader = reader.compat();
@@ -627,10 +629,10 @@ impl<'a> SingleDownloader<'a> {
             CompressFile::Xz => &mut XzDecoder::new(BufReader::new(from)),
             CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(from)),
             CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(from)),
-            CompressFile::Nothing => &mut BufReader::new(from),
             CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(from)),
-            CompressFile::Lzma => unimplemented!(),
-            CompressFile::Lz4 => unimplemented!(),
+            CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(from)),
+            CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(from)),
+            CompressFile::Nothing => &mut BufReader::new(from),
         };
 
         let mut reader = reader.compat();

oma-fetch/src/download.rs

@@ -5,10 +5,13 @@ use aho_corasick::AhoCorasick;
 #[cfg(feature = "apt")]
 use oma_apt::config::{Config, ConfigTree};
 use oma_fetch::CompressFile;
+use once_cell::sync::OnceCell;
 use tracing::debug;
 
 use crate::{db::RefreshError, inrelease::ChecksumItem};
 
+static COMPRESSION_ORDER: OnceCell<Vec<CompressFileWrapper>> = OnceCell::new();
+
 #[derive(Debug, Eq, PartialEq)]
 struct CompressFileWrapper {
     compress_file: CompressFile,
@@ -36,7 +39,10 @@ impl From<&str> for CompressFileWrapper {
                 compress_file: CompressFile::Zstd,
             },
             x => {
-                debug!("{x} format is not compress format");
+                if !x.is_ascii() {
+                    debug!("{x} format is not compress format");
+                }
+
                 CompressFileWrapper {
                     compress_file: CompressFile::Nothing,
                 }
@@ -55,11 +61,13 @@ impl PartialOrd for CompressFileWrapper {
 impl Ord for CompressFileWrapper {
     fn cmp(&self, other: &Self) -> Ordering {
         let config = Config::new();
-        let t = config
-            .get_compression_types()
-            .iter()
-            .map(|t| CompressFileWrapper::from(t.as_str()))
-            .collect::<Vec<_>>();
+        let t = COMPRESSION_ORDER.get_or_init(|| {
+            config
+                .get_compression_types()
+                .iter()
+                .map(|t| CompressFileWrapper::from(t.as_str()))
+                .collect::<Vec<_>>()
+        });
 
         let self_pos = t.iter().position(|x| x == self).unwrap();
         let other_pos = t.iter().position(|x| x == other).unwrap();
@@ -71,10 +79,12 @@ impl Ord for CompressFileWrapper {
 #[cfg(not(feature = "apt"))]
 impl Ord for CompressFileWrapper {
     fn cmp(&self, other: &Self) -> Ordering {
-        let t = vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"]
-            .into_iter()
-            .map(CompressFileWrapper::from)
-            .collect::<Vec<_>>();
+        let t = COMPRESSION_ORDER.get_or_init(|| {
+            vec!["zst", "xz", "bz2", "lzma", "gz", "lz4", "uncompressed"]
+                .into_iter()
+                .map(CompressFileWrapper::from)
+                .collect::<Vec<_>>()
+        });
 
         let self_pos = t.iter().position(|x| x == self).unwrap();
         let other_pos = t.iter().position(|x| x == other).unwrap();
@@ -385,15 +395,37 @@ fn compress_file(name: &str) -> CompressFileWrapper {
 #[cfg(feature = "apt")]
 #[test]
 fn test_compression_order() {
-    let mut t = Config::new()
+    let config = Config::new();
+
+    config.set_vector(
+        "Acquire::CompressionTypes::Order",
+        &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"],
+    );
+
+    let mut types = config
         .get_compression_types()
         .iter()
         .map(|t| CompressFileWrapper::from(t.as_str()))
         .collect::<Vec<_>>();
 
-    t.sort();
+    types.sort_unstable();
+    types.reverse();
 
-    dbg!(t);
+    assert_eq!(
+        types,
+        vec![
+            CompressFile::Zstd,
+            CompressFile::Xz,
+            CompressFile::Bz2,
+            CompressFile::Lzma,
+            CompressFile::Gzip,
+            CompressFile::Lz4,
+            CompressFile::Nothing
+        ]
+        .into_iter()
+        .map(|x| x.into())
+        .collect::<Vec<CompressFileWrapper>>()
+    );
 }
 
 #[test]

oma-refresh/src/config.rs

@@ -236,10 +236,18 @@ impl<'a> OmaRefresh<'a> {
             }
 
             #[cfg(not(feature = "apt"))]
-            let list_file = self.source.join("etc/apt/sources.list").to_string_lossy().to_string();
+            let list_file = self
+                .source
+                .join("etc/apt/sources.list")
+                .to_string_lossy()
+                .to_string();
 
             #[cfg(not(feature = "apt"))]
-            let list_dir = self.source.join("etc/apt/sources.list.d").to_string_lossy().to_string();
+            let list_dir = self
+                .source
+                .join("etc/apt/sources.list.d")
+                .to_string_lossy()
+                .to_string();
 
             scan_sources_lists_paths_from_sysroot(list_file, list_dir)
                 .await
@@ -878,6 +886,8 @@ fn collect_download_task(
                     Some("xz") => CompressFile::Xz,
                     Some("bz2") => CompressFile::Bz2,
                     Some("zst") => CompressFile::Zstd,
+                    Some("lzma") => CompressFile::Lzma,
+                    Some("lz4") => CompressFile::Lz4,
                     _ => CompressFile::Nothing,
                 }
             }