From 31d67a390aa577c3c1147fa808cc1c2b4dbaf67a Mon Sep 17 00:00:00 2001 From: eatradish Date: Sun, 28 Sep 2025 18:16:20 +0800 Subject: [PATCH 1/2] refactor(oma-refresh)!: Use apt config to adjust compression file order --- oma-fetch/src/download.rs | 4 ++ oma-fetch/src/lib.rs | 51 +--------------- oma-refresh/Cargo.toml | 4 ++ oma-refresh/src/config.rs | 120 ++++++++++++++++++++++++++++++++++---- oma-refresh/src/db.rs | 26 +++++++-- 5 files changed, 140 insertions(+), 65 deletions(-) diff --git a/oma-fetch/src/download.rs b/oma-fetch/src/download.rs index 97d13550a..181db1008 100644 --- a/oma-fetch/src/download.rs +++ b/oma-fetch/src/download.rs @@ -479,6 +479,8 @@ impl<'a> SingleDownloader<'a> { CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(bytes_stream)), CompressFile::Nothing => &mut BufReader::new(bytes_stream), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(bytes_stream)), + CompressFile::Lzma => unimplemented!(), + CompressFile::Lz4 => unimplemented!(), }; let mut reader = reader.compat(); @@ -627,6 +629,8 @@ impl<'a> SingleDownloader<'a> { CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(from)), CompressFile::Nothing => &mut BufReader::new(from), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(from)), + CompressFile::Lzma => unimplemented!(), + CompressFile::Lz4 => unimplemented!(), }; let mut reader = reader.compat(); diff --git a/oma-fetch/src/lib.rs b/oma-fetch/src/lib.rs index abc32b409..cfbec94ef 100644 --- a/oma-fetch/src/lib.rs +++ b/oma-fetch/src/lib.rs @@ -46,59 +46,12 @@ pub enum CompressFile { Gzip, Xz, Zstd, + Lzma, + Lz4, #[default] Nothing, } -// 压缩文件下载顺序：Zstd -> XZ -> Gzip -> Bz2 -> 未压缩 -impl Ord for CompressFile { - fn cmp(&self, other: &Self) -> Ordering { - match self { - CompressFile::Bz2 => match other { - CompressFile::Bz2 => Ordering::Equal, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Gzip => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Xz => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Greater, - CompressFile::Xz => Ordering::Equal, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Zstd => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Greater, - CompressFile::Xz => Ordering::Greater, - CompressFile::Zstd => Ordering::Equal, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Nothing => match other { - CompressFile::Bz2 => Ordering::Less, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Equal, - }, - } - } -} - -impl PartialOrd for CompressFile { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - impl From<&str> for CompressFile { fn from(s: &str) -> Self { match s { diff --git a/oma-refresh/Cargo.toml b/oma-refresh/Cargo.toml index 952d9d710..85e636ec6 100644 --- a/oma-refresh/Cargo.toml +++ b/oma-refresh/Cargo.toml @@ -46,3 +46,7 @@ default = ["aosc", "sequoia-nettle-backend", "rustls", "apt"] tokio = { version = "1.28", default-features = false, features = ["rt-multi-thread"] } flume = "0.11.1" oma-utils = { version = "^0.12.0", path = "../oma-utils", features = ["dpkg"] } + +[[example]] +name = "update" +required-features = ["apt"] diff --git a/oma-refresh/src/config.rs b/oma-refresh/src/config.rs index cc7f79d7e..8634c35e7 100644 --- a/oma-refresh/src/config.rs +++ b/oma-refresh/src/config.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, collections::HashMap, path::Path}; +use std::{borrow::Cow, cmp::Ordering, collections::HashMap, path::Path}; use ahash::AHashMap; use aho_corasick::AhoCorasick; @@ -9,6 +9,88 @@ use tracing::debug; use crate::{db::RefreshError, inrelease::ChecksumItem}; +#[derive(Debug, Eq, PartialEq)] +struct CompressFileWrapper { + compress_file: CompressFile, +} + +impl From<&str> for CompressFileWrapper { + fn from(value: &str) -> Self { + match value { + "xz" => CompressFileWrapper { + compress_file: CompressFile::Xz, + }, + "bz2" => CompressFileWrapper { + compress_file: CompressFile::Bz2, + }, + "lzma" => CompressFileWrapper { + compress_file: CompressFile::Lzma, + }, + "gz" => CompressFileWrapper { + compress_file: CompressFile::Gzip, + }, + "lz4" => CompressFileWrapper { + compress_file: CompressFile::Lz4, + }, + "zst" => CompressFileWrapper { + compress_file: CompressFile::Zstd, + }, + x => { + debug!("{x} format is not compress format"); + CompressFileWrapper { + compress_file: CompressFile::Nothing, + } + } + } + } +} + +impl PartialOrd for CompressFileWrapper { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(feature = "apt")] +impl Ord for CompressFileWrapper { + fn cmp(&self, other: &Self) -> Ordering { + let config = Config::new(); + let t = config + .get_compression_types() + .iter() + .map(|t| CompressFileWrapper::from(t.as_str())) + .collect::>(); + + let self_pos = t.iter().position(|x| x == self).unwrap(); + let other_pos = t.iter().position(|x| x == other).unwrap(); + + other_pos.cmp(&self_pos) + } +} + +#[cfg(not(feature = "apt"))] +impl Ord for CompressFileWrapper { + fn cmp(&self, other: &Self) -> Ordering { + let t = vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"] + .into_iter() + .map(CompressFileWrapper::from) + .collect::>(); + + let self_pos = t.iter().position(|x| x == self).unwrap(); + let other_pos = t.iter().position(|x| x == other).unwrap(); + + other_pos.cmp(&self_pos) + } +} + +impl From for CompressFileWrapper { + fn from(value: CompressFile) -> Self { + Self { + compress_file: value, + } + } +} + #[cfg(feature = "apt")] fn modify_result( tree: ConfigTree, @@ -232,7 +314,7 @@ fn flat_repo_template_match( } fn uncompress_file_name(target: &str) -> Cow<'_, str> { - if compress_file(target) == CompressFile::Nothing { + if compress_file(target) == CompressFile::Nothing.into() { Cow::Borrowed(target) } else { let compress_target_without_ext = Path::new(target).with_extension(""); @@ -287,15 +369,31 @@ fn get_matches_language(locales: impl IntoIterator) -> Vec CompressFile { - CompressFile::from( - Path::new(name) - .extension() - .map(|x| x.to_string_lossy()) - .unwrap_or_default() - .to_string() - .as_str(), - ) +fn compress_file(name: &str) -> CompressFileWrapper { + CompressFileWrapper { + compress_file: CompressFile::from( + Path::new(name) + .extension() + .map(|x| x.to_string_lossy()) + .unwrap_or_default() + .to_string() + .as_str(), + ), + } +} + +#[cfg(feature = "apt")] +#[test] +fn test_compression_order() { + let mut t = Config::new() + .get_compression_types() + .iter() + .map(|t| CompressFileWrapper::from(t.as_str())) + .collect::>(); + + t.sort(); + + dbg!(t); } #[test] diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 5320dddc3..87461f542 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -127,6 +127,7 @@ pub struct OmaRefresh<'a> { topic_msg: &'a str, auth_config: Option<&'a AuthConfig>, sources_lists_paths: Option>, + #[cfg(feature = "apt")] #[builder(default)] another_apt_options: Vec, } @@ -228,14 +229,17 @@ impl<'a> OmaRefresh<'a> { .apt_config .dir("Dir::Etc::sourceparts", "sources.list.d"); - #[cfg(not(feature = "apt"))] - let list_file = self.source.join("etc/apt/sources.list"); + #[cfg(feature = "apt")] + { + debug!("sources.list is: {list_file}"); + debug!("sources.list.d is: {list_dir}"); + } #[cfg(not(feature = "apt"))] - let list_dir = self.source.join("etc/apt/sources.list.d"); + let list_file = self.source.join("etc/apt/sources.list").to_string_lossy().to_string(); - debug!("sources.list is: {list_file}"); - debug!("sources.list.d is: {list_dir}"); + #[cfg(not(feature = "apt"))] + let list_dir = self.source.join("etc/apt/sources.list.d").to_string_lossy().to_string(); scan_sources_lists_paths_from_sysroot(list_file, list_dir) .await @@ -320,6 +324,18 @@ impl<'a> OmaRefresh<'a> { debug!("Setting apt opt: {k}={v}"); self.apt_config.set(k, v); } + + // default compression order + if self + .apt_config + .find_vector("Acquire::CompressionTypes::Order") + .is_empty() + { + self.apt_config.set_vector( + "Acquire::CompressionTypes::Order", + &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"], + ); + } } async fn download_release_data( From 4a8f96297621326afef0d16dfbbe0ca6d01f28ae Mon Sep 17 00:00:00 2001 From: eatradish Date: Tue, 30 Sep 2025 10:24:50 +0800 Subject: [PATCH 2/2] feat(oma-fetch,oma-refresh): support lzma and lz4 file download and decompress --- Cargo.lock | 20 ++++++++++++++ oma-fetch/Cargo.toml | 2 +- oma-fetch/src/download.rs | 16 ++++++----- oma-refresh/src/config.rs | 58 ++++++++++++++++++++++++++++++--------- oma-refresh/src/db.rs | 14 ++++++++-- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e89e0d07a..947f8046a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,7 @@ dependencies = [ "futures-core", "futures-io", "liblzma", + "lz4", "memchr", "pin-project-lite", "zstd", @@ -2605,6 +2606,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lzzzz" version = "2.0.0" diff --git a/oma-fetch/Cargo.toml b/oma-fetch/Cargo.toml index 44d99c0ae..07facf13a 100644 --- a/oma-fetch/Cargo.toml +++ b/oma-fetch/Cargo.toml @@ -15,7 +15,7 @@ faster-hex = "0.10" sha2 = "0.10" futures = "0.3" # FIXME: issue https://github.com/AOSC-Dev/oma/issues/547, workaround downgrade to 0.4.27 -async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd"] } +async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd", "lz4"] } # Note: Use the `uncheck_liblzma_version` feature here to force dynamic linking # even with older liblzma (Ubuntu 22.04 uses 5.2.x). Be mindful, however, that # async-compression depends on a different liblzma-rs than what is specified here. diff --git a/oma-fetch/src/download.rs b/oma-fetch/src/download.rs index 181db1008..cdf920d6a 100644 --- a/oma-fetch/src/download.rs +++ b/oma-fetch/src/download.rs @@ -6,7 +6,9 @@ use std::{ time::Duration, }; -use async_compression::futures::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder}; +use async_compression::futures::bufread::{ + BzDecoder, GzipDecoder, Lz4Decoder, LzmaDecoder, XzDecoder, ZstdDecoder, +}; use bon::bon; use futures::{AsyncRead, TryStreamExt, io::BufReader}; use reqwest::{ @@ -477,10 +479,10 @@ impl<'a> SingleDownloader<'a> { CompressFile::Xz => &mut XzDecoder::new(BufReader::new(bytes_stream)), CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(bytes_stream)), CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(bytes_stream)), - CompressFile::Nothing => &mut BufReader::new(bytes_stream), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(bytes_stream)), - CompressFile::Lzma => unimplemented!(), - CompressFile::Lz4 => unimplemented!(), + CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(bytes_stream)), + CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(bytes_stream)), + CompressFile::Nothing => &mut BufReader::new(bytes_stream), }; let mut reader = reader.compat(); @@ -627,10 +629,10 @@ impl<'a> SingleDownloader<'a> { CompressFile::Xz => &mut XzDecoder::new(BufReader::new(from)), CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(from)), CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(from)), - CompressFile::Nothing => &mut BufReader::new(from), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(from)), - CompressFile::Lzma => unimplemented!(), - CompressFile::Lz4 => unimplemented!(), + CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(from)), + CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(from)), + CompressFile::Nothing => &mut BufReader::new(from), }; let mut reader = reader.compat(); diff --git a/oma-refresh/src/config.rs b/oma-refresh/src/config.rs index 8634c35e7..b4fe349ae 100644 --- a/oma-refresh/src/config.rs +++ b/oma-refresh/src/config.rs @@ -5,10 +5,13 @@ use aho_corasick::AhoCorasick; #[cfg(feature = "apt")] use oma_apt::config::{Config, ConfigTree}; use oma_fetch::CompressFile; +use once_cell::sync::OnceCell; use tracing::debug; use crate::{db::RefreshError, inrelease::ChecksumItem}; +static COMPRESSION_ORDER: OnceCell> = OnceCell::new(); + #[derive(Debug, Eq, PartialEq)] struct CompressFileWrapper { compress_file: CompressFile, @@ -36,7 +39,10 @@ impl From<&str> for CompressFileWrapper { compress_file: CompressFile::Zstd, }, x => { - debug!("{x} format is not compress format"); + if !x.is_ascii() { + debug!("{x} format is not compress format"); + } + CompressFileWrapper { compress_file: CompressFile::Nothing, } @@ -55,11 +61,13 @@ impl PartialOrd for CompressFileWrapper { impl Ord for CompressFileWrapper { fn cmp(&self, other: &Self) -> Ordering { let config = Config::new(); - let t = config - .get_compression_types() - .iter() - .map(|t| CompressFileWrapper::from(t.as_str())) - .collect::>(); + let t = COMPRESSION_ORDER.get_or_init(|| { + config + .get_compression_types() + .iter() + .map(|t| CompressFileWrapper::from(t.as_str())) + .collect::>() + }); let self_pos = t.iter().position(|x| x == self).unwrap(); let other_pos = t.iter().position(|x| x == other).unwrap(); @@ -71,10 +79,12 @@ impl Ord for CompressFileWrapper { #[cfg(not(feature = "apt"))] impl Ord for CompressFileWrapper { fn cmp(&self, other: &Self) -> Ordering { - let t = vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"] - .into_iter() - .map(CompressFileWrapper::from) - .collect::>(); + let t = COMPRESSION_ORDER.get_or_init(|| { + vec!["zst", "xz", "bz2", "lzma", "gz", "lz4", "uncompressed"] + .into_iter() + .map(CompressFileWrapper::from) + .collect::>() + }); let self_pos = t.iter().position(|x| x == self).unwrap(); let other_pos = t.iter().position(|x| x == other).unwrap(); @@ -385,15 +395,37 @@ fn compress_file(name: &str) -> CompressFileWrapper { #[cfg(feature = "apt")] #[test] fn test_compression_order() { - let mut t = Config::new() + let config = Config::new(); + + config.set_vector( + "Acquire::CompressionTypes::Order", + &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"], + ); + + let mut types = config .get_compression_types() .iter() .map(|t| CompressFileWrapper::from(t.as_str())) .collect::>(); - t.sort(); + types.sort_unstable(); + types.reverse(); - dbg!(t); + assert_eq!( + types, + vec![ + CompressFile::Zstd, + CompressFile::Xz, + CompressFile::Bz2, + CompressFile::Lzma, + CompressFile::Gzip, + CompressFile::Lz4, + CompressFile::Nothing + ] + .into_iter() + .map(|x| x.into()) + .collect::>() + ); } #[test] diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 87461f542..91bb7e08b 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -236,10 +236,18 @@ impl<'a> OmaRefresh<'a> { } #[cfg(not(feature = "apt"))] - let list_file = self.source.join("etc/apt/sources.list").to_string_lossy().to_string(); + let list_file = self + .source + .join("etc/apt/sources.list") + .to_string_lossy() + .to_string(); #[cfg(not(feature = "apt"))] - let list_dir = self.source.join("etc/apt/sources.list.d").to_string_lossy().to_string(); + let list_dir = self + .source + .join("etc/apt/sources.list.d") + .to_string_lossy() + .to_string(); scan_sources_lists_paths_from_sysroot(list_file, list_dir) .await @@ -878,6 +886,8 @@ fn collect_download_task( Some("xz") => CompressFile::Xz, Some("bz2") => CompressFile::Bz2, Some("zst") => CompressFile::Zstd, + Some("lzma") => CompressFile::Lzma, + Some("lz4") => CompressFile::Lz4, _ => CompressFile::Nothing, } }