diff --git a/Cargo.lock b/Cargo.lock index befb7188d..73f0626df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,7 @@ dependencies = [ "futures-core", "futures-io", "liblzma", + "lz4", "memchr", "pin-project-lite", "zstd", @@ -642,9 +643,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.38" +version = "1.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" +checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" dependencies = [ "find-msvc-tools", "jobserver", @@ -2465,9 +2466,9 @@ dependencies = [ [[package]] name = "liblzma" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10bf66f4598dc77ff96677c8e763655494f00ff9c1cf79e2eb5bb07bc31f807d" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" dependencies = [ "liblzma-sys", ] @@ -2606,6 +2607,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lzzzz" version = "2.0.0" @@ -4133,9 +4153,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.227" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ece43fc6fbed4eb5392ab50c07334d3e577cbf40997ee896fe7af40bba4245" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -4143,18 +4163,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.227" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a576275b607a2c86ea29e410193df32bc680303c82f31e275bbfcafe8b33be5" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.227" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e694923b8824cf0e9b382adf0f60d4e05f348f357b38833a3fa5ed7c2ede04" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -4469,9 +4489,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753" +checksum = "3bddd368fda2f82ead69c03d46d351987cfa0c2a57abfa37a017f3aa3e9bf69a" dependencies = [ "libc", "memchr", @@ -4754,9 +4774,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", diff --git a/oma-fetch/Cargo.toml b/oma-fetch/Cargo.toml index 44d99c0ae..07facf13a 100644 --- a/oma-fetch/Cargo.toml +++ b/oma-fetch/Cargo.toml @@ -15,7 +15,7 @@ faster-hex = "0.10" sha2 = "0.10" futures = "0.3" # FIXME: issue https://github.com/AOSC-Dev/oma/issues/547, workaround downgrade to 0.4.27 -async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd"] } +async-compression = { version = "=0.4.27", features = ["gzip", "xz", "futures-io", "bzip2", "zstd", "lz4"] } # Note: Use the `uncheck_liblzma_version` feature here to force dynamic linking # even with older liblzma (Ubuntu 22.04 uses 5.2.x). Be mindful, however, that # async-compression depends on a different liblzma-rs than what is specified here. diff --git a/oma-fetch/src/download.rs b/oma-fetch/src/download.rs index df9e5e167..6d32d90f2 100644 --- a/oma-fetch/src/download.rs +++ b/oma-fetch/src/download.rs @@ -6,7 +6,9 @@ use std::{ time::Duration, }; -use async_compression::futures::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder}; +use async_compression::futures::bufread::{ + BzDecoder, GzipDecoder, Lz4Decoder, LzmaDecoder, XzDecoder, ZstdDecoder, +}; use bon::bon; use futures::{AsyncRead, TryStreamExt, io::BufReader}; use reqwest::{ @@ -476,8 +478,10 @@ impl<'a> SingleDownloader<'a> { CompressFile::Xz => &mut XzDecoder::new(BufReader::new(bytes_stream)), CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(bytes_stream)), CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(bytes_stream)), - CompressFile::Nothing => &mut BufReader::new(bytes_stream), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(bytes_stream)), + CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(bytes_stream)), + CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(bytes_stream)), + CompressFile::Nothing => &mut BufReader::new(bytes_stream), }; let mut reader = reader.compat(); @@ -632,8 +636,10 @@ impl<'a> SingleDownloader<'a> { CompressFile::Xz => &mut XzDecoder::new(BufReader::new(from)), CompressFile::Gzip => &mut GzipDecoder::new(BufReader::new(from)), CompressFile::Bz2 => &mut BzDecoder::new(BufReader::new(from)), - CompressFile::Nothing => &mut BufReader::new(from), CompressFile::Zstd => &mut ZstdDecoder::new(BufReader::new(from)), + CompressFile::Lzma => &mut LzmaDecoder::new(BufReader::new(from)), + CompressFile::Lz4 => &mut Lz4Decoder::new(BufReader::new(from)), + CompressFile::Nothing => &mut BufReader::new(from), }; let mut reader = reader.compat(); diff --git a/oma-fetch/src/lib.rs b/oma-fetch/src/lib.rs index df5c1d135..d80bb2cfc 100644 --- a/oma-fetch/src/lib.rs +++ b/oma-fetch/src/lib.rs @@ -46,59 +46,12 @@ pub enum CompressFile { Gzip, Xz, Zstd, + Lzma, + Lz4, #[default] Nothing, } -// 压缩文件下载顺序：Zstd -> XZ -> Gzip -> Bz2 -> 未压缩 -impl Ord for CompressFile { - fn cmp(&self, other: &Self) -> Ordering { - match self { - CompressFile::Bz2 => match other { - CompressFile::Bz2 => Ordering::Equal, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Gzip => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Xz => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Greater, - CompressFile::Xz => Ordering::Equal, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Zstd => match other { - CompressFile::Bz2 => Ordering::Greater, - CompressFile::Gzip => Ordering::Greater, - CompressFile::Xz => Ordering::Greater, - CompressFile::Zstd => Ordering::Equal, - CompressFile::Nothing => Ordering::Greater, - }, - CompressFile::Nothing => match other { - CompressFile::Bz2 => Ordering::Less, - CompressFile::Gzip => Ordering::Less, - CompressFile::Xz => Ordering::Less, - CompressFile::Zstd => Ordering::Less, - CompressFile::Nothing => Ordering::Equal, - }, - } - } -} - -impl PartialOrd for CompressFile { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - impl From<&str> for CompressFile { fn from(s: &str) -> Self { match s { diff --git a/oma-refresh/Cargo.toml b/oma-refresh/Cargo.toml index 952d9d710..85e636ec6 100644 --- a/oma-refresh/Cargo.toml +++ b/oma-refresh/Cargo.toml @@ -46,3 +46,7 @@ default = ["aosc", "sequoia-nettle-backend", "rustls", "apt"] tokio = { version = "1.28", default-features = false, features = ["rt-multi-thread"] } flume = "0.11.1" oma-utils = { version = "^0.12.0", path = "../oma-utils", features = ["dpkg"] } + +[[example]] +name = "update" +required-features = ["apt"] diff --git a/oma-refresh/src/config.rs b/oma-refresh/src/config.rs index cc7f79d7e..b4fe349ae 100644 --- a/oma-refresh/src/config.rs +++ b/oma-refresh/src/config.rs @@ -1,14 +1,106 @@ -use std::{borrow::Cow, collections::HashMap, path::Path}; +use std::{borrow::Cow, cmp::Ordering, collections::HashMap, path::Path}; use ahash::AHashMap; use aho_corasick::AhoCorasick; #[cfg(feature = "apt")] use oma_apt::config::{Config, ConfigTree}; use oma_fetch::CompressFile; +use once_cell::sync::OnceCell; use tracing::debug; use crate::{db::RefreshError, inrelease::ChecksumItem}; +static COMPRESSION_ORDER: OnceCell> = OnceCell::new(); + +#[derive(Debug, Eq, PartialEq)] +struct CompressFileWrapper { + compress_file: CompressFile, +} + +impl From<&str> for CompressFileWrapper { + fn from(value: &str) -> Self { + match value { + "xz" => CompressFileWrapper { + compress_file: CompressFile::Xz, + }, + "bz2" => CompressFileWrapper { + compress_file: CompressFile::Bz2, + }, + "lzma" => CompressFileWrapper { + compress_file: CompressFile::Lzma, + }, + "gz" => CompressFileWrapper { + compress_file: CompressFile::Gzip, + }, + "lz4" => CompressFileWrapper { + compress_file: CompressFile::Lz4, + }, + "zst" => CompressFileWrapper { + compress_file: CompressFile::Zstd, + }, + x => { + if !x.is_ascii() { + debug!("{x} format is not compress format"); + } + + CompressFileWrapper { + compress_file: CompressFile::Nothing, + } + } + } + } +} + +impl PartialOrd for CompressFileWrapper { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(feature = "apt")] +impl Ord for CompressFileWrapper { + fn cmp(&self, other: &Self) -> Ordering { + let config = Config::new(); + let t = COMPRESSION_ORDER.get_or_init(|| { + config + .get_compression_types() + .iter() + .map(|t| CompressFileWrapper::from(t.as_str())) + .collect::>() + }); + + let self_pos = t.iter().position(|x| x == self).unwrap(); + let other_pos = t.iter().position(|x| x == other).unwrap(); + + other_pos.cmp(&self_pos) + } +} + +#[cfg(not(feature = "apt"))] +impl Ord for CompressFileWrapper { + fn cmp(&self, other: &Self) -> Ordering { + let t = COMPRESSION_ORDER.get_or_init(|| { + vec!["zst", "xz", "bz2", "lzma", "gz", "lz4", "uncompressed"] + .into_iter() + .map(CompressFileWrapper::from) + .collect::>() + }); + + let self_pos = t.iter().position(|x| x == self).unwrap(); + let other_pos = t.iter().position(|x| x == other).unwrap(); + + other_pos.cmp(&self_pos) + } +} + +impl From for CompressFileWrapper { + fn from(value: CompressFile) -> Self { + Self { + compress_file: value, + } + } +} + #[cfg(feature = "apt")] fn modify_result( tree: ConfigTree, @@ -232,7 +324,7 @@ fn flat_repo_template_match( } fn uncompress_file_name(target: &str) -> Cow<'_, str> { - if compress_file(target) == CompressFile::Nothing { + if compress_file(target) == CompressFile::Nothing.into() { Cow::Borrowed(target) } else { let compress_target_without_ext = Path::new(target).with_extension(""); @@ -287,15 +379,53 @@ fn get_matches_language(locales: impl IntoIterator) -> Vec CompressFile { - CompressFile::from( - Path::new(name) - .extension() - .map(|x| x.to_string_lossy()) - .unwrap_or_default() - .to_string() - .as_str(), - ) +fn compress_file(name: &str) -> CompressFileWrapper { + CompressFileWrapper { + compress_file: CompressFile::from( + Path::new(name) + .extension() + .map(|x| x.to_string_lossy()) + .unwrap_or_default() + .to_string() + .as_str(), + ), + } +} + +#[cfg(feature = "apt")] +#[test] +fn test_compression_order() { + let config = Config::new(); + + config.set_vector( + "Acquire::CompressionTypes::Order", + &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"], + ); + + let mut types = config + .get_compression_types() + .iter() + .map(|t| CompressFileWrapper::from(t.as_str())) + .collect::>(); + + types.sort_unstable(); + types.reverse(); + + assert_eq!( + types, + vec![ + CompressFile::Zstd, + CompressFile::Xz, + CompressFile::Bz2, + CompressFile::Lzma, + CompressFile::Gzip, + CompressFile::Lz4, + CompressFile::Nothing + ] + .into_iter() + .map(|x| x.into()) + .collect::>() + ); } #[test] diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 5320dddc3..91bb7e08b 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -127,6 +127,7 @@ pub struct OmaRefresh<'a> { topic_msg: &'a str, auth_config: Option<&'a AuthConfig>, sources_lists_paths: Option>, + #[cfg(feature = "apt")] #[builder(default)] another_apt_options: Vec, } @@ -228,14 +229,25 @@ impl<'a> OmaRefresh<'a> { .apt_config .dir("Dir::Etc::sourceparts", "sources.list.d"); - #[cfg(not(feature = "apt"))] - let list_file = self.source.join("etc/apt/sources.list"); + #[cfg(feature = "apt")] + { + debug!("sources.list is: {list_file}"); + debug!("sources.list.d is: {list_dir}"); + } #[cfg(not(feature = "apt"))] - let list_dir = self.source.join("etc/apt/sources.list.d"); + let list_file = self + .source + .join("etc/apt/sources.list") + .to_string_lossy() + .to_string(); - debug!("sources.list is: {list_file}"); - debug!("sources.list.d is: {list_dir}"); + #[cfg(not(feature = "apt"))] + let list_dir = self + .source + .join("etc/apt/sources.list.d") + .to_string_lossy() + .to_string(); scan_sources_lists_paths_from_sysroot(list_file, list_dir) .await @@ -320,6 +332,18 @@ impl<'a> OmaRefresh<'a> { debug!("Setting apt opt: {k}={v}"); self.apt_config.set(k, v); } + + // default compression order + if self + .apt_config + .find_vector("Acquire::CompressionTypes::Order") + .is_empty() + { + self.apt_config.set_vector( + "Acquire::CompressionTypes::Order", + &vec!["zst", "xz", "bz2", "lzma", "gz", "lz4"], + ); + } } async fn download_release_data( @@ -862,6 +886,8 @@ fn collect_download_task( Some("xz") => CompressFile::Xz, Some("bz2") => CompressFile::Bz2, Some("zst") => CompressFile::Zstd, + Some("lzma") => CompressFile::Lzma, + Some("lz4") => CompressFile::Lz4, _ => CompressFile::Nothing, } }