Merge pull request #583 from AOSC-Dev/use-apt-config-to-adjust-compression-order

refactor(oma-refresh)!: Use apt config to adjust compression file order

Author: eatradish

Cargo.lock

@@ -15,7 +15,7 @@ faster-hex = "0.10"
 sha2 = "0.10"
 futures = "0.3"
 # FIXME: issue https://github.com/AOSC-Dev/oma/issues/547, workaround downgrade to 0.4.27
-async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd"] }
+async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd", "lz4"] }
 # Note: Use the `uncheck_liblzma_version` feature here to force dynamic linking
 # even with older liblzma (Ubuntu 22.04 uses 5.2.x). Be mindful, however, that
 # async-compression depends on a different liblzma-rs than what is specified here.

oma-fetch/Cargo.toml

@@ -6,7 +6,9 @@ use std::{
     time::Duration,
 };
 
-use async_compression::futures::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder};
+use async_compression::futures::bufread::{
+    BzDecoder, GzipDecoder, Lz4Decoder, LzmaDecoder, XzDecoder, ZstdDecoder,
+};
 use bon::bon;
 use futures::{AsyncRead, TryStreamExt, io::BufReader};
 use reqwest::{
@@ -477,8 +479,10 @@ impl<'a> SingleDownloader<'a> {
             CompressFile::Xz => &mut XzDecoder::new(BufReader::new(bytes_stream)),
             CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(bytes_stream)),
             CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(bytes_stream)),
-            CompressFile::Nothing => &mut BufReader::new(bytes_stream),
             CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(bytes_stream)),
+            CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(bytes_stream)),
+            CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(bytes_stream)),
+            CompressFile::Nothing => &mut BufReader::new(bytes_stream),
         };
 
         let mut reader = reader.compat();
@@ -625,8 +629,10 @@ impl<'a> SingleDownloader<'a> {
             CompressFile::Xz => &mut XzDecoder::new(BufReader::new(from)),
             CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(from)),
             CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(from)),
-            CompressFile::Nothing => &mut BufReader::new(from),
             CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(from)),
+            CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(from)),
+            CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(from)),
+            CompressFile::Nothing => &mut BufReader::new(from),
         };
 
         let mut reader = reader.compat();

oma-fetch/src/download.rs

@@ -46,59 +46,12 @@ pub enum CompressFile {
     Gzip,
     Xz,
     Zstd,
+    Lzma,
+    Lz4,
     #[default]
     Nothing,
 }
 
-// 压缩文件下载顺序：Zstd -> XZ -> Gzip -> Bz2 -> 未压缩
-impl Ord for CompressFile {
-    fn cmp(&self, other: &Self) -> Ordering {
-        match self {
-            CompressFile::Bz2 => match other {
-                CompressFile::Bz2 => Ordering::Equal,
-                CompressFile::Gzip => Ordering::Less,
-                CompressFile::Xz => Ordering::Less,
-                CompressFile::Zstd => Ordering::Less,
-                CompressFile::Nothing => Ordering::Greater,
-            },
-            CompressFile::Gzip => match other {
-                CompressFile::Bz2 => Ordering::Greater,
-                CompressFile::Gzip => Ordering::Less,
-                CompressFile::Xz => Ordering::Less,
-                CompressFile::Zstd => Ordering::Less,
-                CompressFile::Nothing => Ordering::Greater,
-            },
-            CompressFile::Xz => match other {
-                CompressFile::Bz2 => Ordering::Greater,
-                CompressFile::Gzip => Ordering::Greater,
-                CompressFile::Xz => Ordering::Equal,
-                CompressFile::Zstd => Ordering::Less,
-                CompressFile::Nothing => Ordering::Greater,
-            },
-            CompressFile::Zstd => match other {
-                CompressFile::Bz2 => Ordering::Greater,
-                CompressFile::Gzip => Ordering::Greater,
-                CompressFile::Xz => Ordering::Greater,
-                CompressFile::Zstd => Ordering::Equal,
-                CompressFile::Nothing => Ordering::Greater,
-            },
-            CompressFile::Nothing => match other {
-                CompressFile::Bz2 => Ordering::Less,
-                CompressFile::Gzip => Ordering::Less,
-                CompressFile::Xz => Ordering::Less,
-                CompressFile::Zstd => Ordering::Less,
-                CompressFile::Nothing => Ordering::Equal,
-            },
-        }
-    }
-}
-
-impl PartialOrd for CompressFile {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
 impl From<&str> for CompressFile {
     fn from(s: &str) -> Self {
         match s {

oma-fetch/src/lib.rs

@@ -46,3 +46,7 @@ default = ["aosc", "sequoia-nettle-backend", "rustls", "apt"]
 tokio = { version = "1.28", default-features = false, features = ["rt-multi-thread"] }
 flume = "0.11.1"
 oma-utils = { version = "^0.12.0", path = "../oma-utils", features = ["dpkg"] }
+
+[[example]]
+name = "update"
+required-features = ["apt"]

oma-refresh/Cargo.toml

@@ -1,14 +1,106 @@
-use std::{borrow::Cow, collections::HashMap, path::Path};
+use std::{borrow::Cow, cmp::Ordering, collections::HashMap, path::Path};
 
 use ahash::AHashMap;
 use aho_corasick::AhoCorasick;
 #[cfg(feature = "apt")]
 use oma_apt::config::{Config, ConfigTree};
 use oma_fetch::CompressFile;
+use once_cell::sync::OnceCell;
 use tracing::debug;
 
 use crate::{db::RefreshError, inrelease::ChecksumItem};
 
+static COMPRESSION_ORDER: OnceCell<Vec<CompressFileWrapper>> = OnceCell::new();
+
+#[derive(Debug, Eq, PartialEq)]
+struct CompressFileWrapper {
+    compress_file: CompressFile,
+}
+
+impl From<&str> for CompressFileWrapper {
+    fn from(value: &str) -> Self {
+        match value {
+            "xz" => CompressFileWrapper {
+                compress_file: CompressFile::Xz,
+            },
+            "bz2" => CompressFileWrapper {
+                compress_file: CompressFile::Bz2,
+            },
+            "lzma" => CompressFileWrapper {
+                compress_file: CompressFile::Lzma,
+            },
+            "gz" => CompressFileWrapper {
+                compress_file: CompressFile::Gzip,
+            },
+            "lz4" => CompressFileWrapper {
+                compress_file: CompressFile::Lz4,
+            },
+            "zst" => CompressFileWrapper {
+                compress_file: CompressFile::Zstd,
+            },
+            x => {
+                if !x.is_ascii() {
+                    debug!("{x} format is not compress format");
+                }
+
+                CompressFileWrapper {
+                    compress_file: CompressFile::Nothing,
+                }
+            }
+        }
+    }
+}
+
+impl PartialOrd for CompressFileWrapper {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[cfg(feature = "apt")]
+impl Ord for CompressFileWrapper {
+    fn cmp(&self, other: &Self) -> Ordering {
+        let config = Config::new();
+        let t = COMPRESSION_ORDER.get_or_init(|| {
+            config
+                .get_compression_types()
+                .iter()
+                .map(|t| CompressFileWrapper::from(t.as_str()))
+                .collect::<Vec<_>>()
+        });
+
+        let self_pos = t.iter().position(|x| x == self).unwrap();
+        let other_pos = t.iter().position(|x| x == other).unwrap();
+
+        other_pos.cmp(&self_pos)
+    }
+}
+
+#[cfg(not(feature = "apt"))]
+impl Ord for CompressFileWrapper {
+    fn cmp(&self, other: &Self) -> Ordering {
+        let t = COMPRESSION_ORDER.get_or_init(|| {
+            vec!["zst", "xz", "bz2", "lzma", "gz", "lz4", "uncompressed"]
+                .into_iter()
+                .map(CompressFileWrapper::from)
+                .collect::<Vec<_>>()
+        });
+
+        let self_pos = t.iter().position(|x| x == self).unwrap();
+        let other_pos = t.iter().position(|x| x == other).unwrap();
+
+        other_pos.cmp(&self_pos)
+    }
+}
+
+impl From<CompressFile> for CompressFileWrapper {
+    fn from(value: CompressFile) -> Self {
+        Self {
+            compress_file: value,
+        }
+    }
+}
+
 #[cfg(feature = "apt")]
 fn modify_result(
     tree: ConfigTree,
@@ -232,7 +324,7 @@ fn flat_repo_template_match(
 }
 
 fn uncompress_file_name(target: &str) -> Cow<'_, str> {
-    if compress_file(target) == CompressFile::Nothing {
+    if compress_file(target) == CompressFile::Nothing.into() {
         Cow::Borrowed(target)
     } else {
         let compress_target_without_ext = Path::new(target).with_extension("");
@@ -287,15 +379,53 @@ fn get_matches_language(locales: impl IntoIterator<Item = String>) -> Vec<String
     langs
 }
 
-fn compress_file(name: &str) -> CompressFile {
-    CompressFile::from(
-        Path::new(name)
-            .extension()
-            .map(|x| x.to_string_lossy())
-            .unwrap_or_default()
-            .to_string()
-            .as_str(),
-    )
+fn compress_file(name: &str) -> CompressFileWrapper {
+    CompressFileWrapper {
+        compress_file: CompressFile::from(
+            Path::new(name)
+                .extension()
+                .map(|x| x.to_string_lossy())
+                .unwrap_or_default()
+                .to_string()
+                .as_str(),
+        ),
+    }
+}
+
+#[cfg(feature = "apt")]
+#[test]
+fn test_compression_order() {
+    let config = Config::new();
+
+    config.set_vector(
+        "Acquire::CompressionTypes::Order",
+        &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"],
+    );
+
+    let mut types = config
+        .get_compression_types()
+        .iter()
+        .map(|t| CompressFileWrapper::from(t.as_str()))
+        .collect::<Vec<_>>();
+
+    types.sort_unstable();
+    types.reverse();
+
+    assert_eq!(
+        types,
+        vec![
+            CompressFile::Zstd,
+            CompressFile::Xz,
+            CompressFile::Bz2,
+            CompressFile::Lzma,
+            CompressFile::Gzip,
+            CompressFile::Lz4,
+            CompressFile::Nothing
+        ]
+        .into_iter()
+        .map(|x| x.into())
+        .collect::<Vec<CompressFileWrapper>>()
+    );
 }
 
 #[test]

oma-refresh/src/config.rs

@@ -127,6 +127,7 @@ pub struct OmaRefresh<'a> {
     topic_msg: &'a str,
     auth_config: Option<&'a AuthConfig>,
     sources_lists_paths: Option<Vec<PathBuf>>,
+    #[cfg(feature = "apt")]
     #[builder(default)]
     another_apt_options: Vec<String>,
 }
@@ -228,14 +229,25 @@ impl<'a> OmaRefresh<'a> {
                 .apt_config
                 .dir("Dir::Etc::sourceparts", "sources.list.d");
 
-            #[cfg(not(feature = "apt"))]
-            let list_file = self.source.join("etc/apt/sources.list");
+            #[cfg(feature = "apt")]
+            {
+                debug!("sources.list is: {list_file}");
+                debug!("sources.list.d is: {list_dir}");
+            }
 
             #[cfg(not(feature = "apt"))]
-            let list_dir = self.source.join("etc/apt/sources.list.d");
+            let list_file = self
+                .source
+                .join("etc/apt/sources.list")
+                .to_string_lossy()
+                .to_string();
 
-            debug!("sources.list is: {list_file}");
-            debug!("sources.list.d is: {list_dir}");
+            #[cfg(not(feature = "apt"))]
+            let list_dir = self
+                .source
+                .join("etc/apt/sources.list.d")
+                .to_string_lossy()
+                .to_string();
 
             scan_sources_lists_paths_from_sysroot(list_file, list_dir)
                 .await
@@ -320,6 +332,18 @@ impl<'a> OmaRefresh<'a> {
             debug!("Setting apt opt: {k}={v}");
             self.apt_config.set(k, v);
         }
+
+        // default compression order
+        if self
+            .apt_config
+            .find_vector("Acquire::CompressionTypes::Order")
+            .is_empty()
+        {
+            self.apt_config.set_vector(
+                "Acquire::CompressionTypes::Order",
+                &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"],
+            );
+        }
     }
 
     async fn download_release_data(
@@ -862,6 +886,8 @@ fn collect_download_task(
                     Some("xz") => CompressFile::Xz,
                     Some("bz2") => CompressFile::Bz2,
                     Some("zst") => CompressFile::Zstd,
+                    Some("lzma") => CompressFile::Lzma,
+                    Some("lz4") => CompressFile::Lz4,
                     _ => CompressFile::Nothing,
                 }
             }