diff --git a/Cargo.lock b/Cargo.lock index 110c4b377..e7033fc93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,6 +221,19 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "av-data" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca67ba5d317924c02180c576157afd54babe48a76ebc66ce6d34bb8ba08308e" +dependencies = [ + "byte-slice-cast", + "bytes", + "num-derive", + "num-rational", + "num-traits", +] + [[package]] name = "av-scenechange" version = "0.14.1" @@ -264,7 +277,7 @@ dependencies = [ "arrayvec", "bitreader", "byteorder", - "fallible_collections", + "fallible_collections 0.5.1", "leb128", "log", ] @@ -480,6 +493,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "byte-slice-cast" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" + [[package]] name = "bytecheck" version = "0.6.12" @@ -574,6 +593,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cfg-expr" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cef5b5a1a6827c7322ae2a636368a573006b27cfa76c7ebd53e834daeaab6a" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -1117,6 +1146,28 @@ dependencies = [ "matches", ] +[[package]] +name = "dav1d" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c3f80814db85397819d464bb553268992c393b4b3b5554b89c1655996d5926" +dependencies = [ + "av-data", + "bitflags 2.10.0", + "dav1d-sys", + "static_assertions", +] + +[[package]] +name = "dav1d-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c91aea6668645415331133ed6f8ddf0e7f40160cd97a12d59e68716a58704b" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "deranged" version = "0.5.5" @@ -1415,6 +1466,15 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible_collections" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88c69768c0a15262df21899142bc6df9b9b823546d4b4b9a7bc2d6c448ec6fd" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "fallible_collections" version = "0.5.1" @@ -1784,6 +1844,15 @@ dependencies = [ "ahash 0.7.8", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.12", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -2085,10 +2154,12 @@ dependencies = [ "bytemuck", "byteorder-lite", "color_quant", + "dav1d", "exr", "gif", "image-webp", "moxcms", + "mp4parse", "num-traits", "png", "qoi", @@ -2100,6 +2171,18 @@ dependencies = [ "zune-jpeg 0.5.12", ] +[[package]] +name = "image-compare" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "176623a137baf75908084aff53578d6336cbd932bffaf12fabe26b343eb13338" +dependencies = [ + "image", + "itertools 0.14.0", + "rayon", + "thiserror 2.0.18", +] + [[package]] name = "image-webp" version = "0.2.4" @@ -2948,6 +3031,20 @@ dependencies = [ "pxfm", ] +[[package]] +name = "mp4parse" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a35203d3c6ce92d5251c77520acb2e57108c88728695aa883f70023624c570" +dependencies = [ + "bitreader", + "byteorder", + "fallible_collections 0.4.9", + "log", + "num-traits", + "static_assertions", +] + [[package]] name = "mutate_once" version = "0.1.2" @@ -3724,6 +3821,18 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perceptual-image" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030ca5c725240423c0128b873e8ffb664e4ce22c5a22273687e6490115aa7a84" +dependencies = [ + "image", + "image-compare", + "phf 0.13.1", + "webp", +] + [[package]] name = "phf" version = "0.11.3" @@ -4112,7 +4221,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -4567,7 +4676,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4856,6 +4965,7 @@ name = "site" version = "0.23.0" dependencies = [ "ahash 0.8.12", + "aho-corasick", "config", "content", "criterion", @@ -4863,6 +4973,7 @@ dependencies = [ "fs-err", "globset", "grass", + "image", "imageproc", "insta", "link_checker", @@ -4871,6 +4982,7 @@ dependencies = [ "memchr", "minify-html", "path-slash", + "perceptual-image", "rayon", "relative-path", "render", @@ -5052,6 +5164,19 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "7.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c8f33736f986f16d69b6cb8b03f55ddcad5c41acc4ccc39dd88e84aa805e7f" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml", + "version-compare", +] + [[package]] name = "tap" version = "1.0.1" @@ -5069,6 +5194,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-lexicon" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" + [[package]] name = "tauri-winres" version = "0.3.5" @@ -5678,6 +5809,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + [[package]] name = "version_check" version = "0.9.5" diff --git a/components/config/src/config/image_compression.rs b/components/config/src/config/image_compression.rs new file mode 100644 index 000000000..a792ba68a --- /dev/null +++ b/components/config/src/config/image_compression.rs @@ -0,0 +1,41 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImageFormat { + Jpeg, + Webp, + Avif, +} + +impl ImageFormat { + pub fn file_extension(&self) -> &str { + match self { + ImageFormat::Jpeg => "jpg", + ImageFormat::Webp => "webp", + ImageFormat::Avif => "avif", + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImageCompression { + /// Glob of files to run the compression on. + pub glob: String, + /// The codec to encode files to. + pub format: ImageFormat, + /// Target SSIM score. + #[serde(default = "default_ssim")] + pub target_ssim: f64, + /// Number of iterations to try and reach target SSIM. + #[serde(default = "default_iterations")] + pub max_iterations: u8, +} + +fn default_ssim() -> f64 { + 0.8295 +} + +fn default_iterations() -> u8 { + 5 +} diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs index 61d70d9ca..50110fcd0 100644 --- a/components/config/src/config/mod.rs +++ b/components/config/src/config/mod.rs @@ -1,3 +1,4 @@ +pub mod image_compression; pub mod languages; pub mod link_checker; pub mod markup; @@ -77,6 +78,7 @@ pub struct Config { pub compile_sass: bool, /// Whether to minify the html output pub minify_html: bool, + pub compress_images: Option>, /// Whether to build the search index for the content pub build_search_index: bool, /// A list of file glob patterns to ignore when processing the content folder. Defaults to none. @@ -466,6 +468,7 @@ impl Default for Config { author: None, compile_sass: false, minify_html: false, + compress_images: None, mode: Mode::Build, build_search_index: false, ignored_content: Vec::new(), diff --git a/components/config/src/lib.rs b/components/config/src/lib.rs index 54e4487b6..c06e2efc1 100644 --- a/components/config/src/lib.rs +++ b/components/config/src/lib.rs @@ -5,6 +5,7 @@ use std::path::Path; pub use crate::config::{ Config, + image_compression::{ImageCompression, ImageFormat}, languages::LanguageOptions, link_checker::LinkChecker, link_checker::LinkCheckerLevel, diff --git a/components/site/Cargo.toml b/components/site/Cargo.toml index 50ef7eb3e..4cef42bc1 100644 --- a/components/site/Cargo.toml +++ b/components/site/Cargo.toml @@ -29,6 +29,9 @@ content = { workspace = true } render = { workspace = true } markdown = { workspace = true } memchr = { workspace = true } +perceptual-image = { version = "0.1.0", features = ["jpeg", "avif", "webp"] } +image.workspace = true +aho-corasick = "1.1.4" [dev-dependencies] tempfile = "3" diff --git a/components/site/src/compress_images.rs b/components/site/src/compress_images.rs new file mode 100644 index 000000000..100b8047a --- /dev/null +++ b/components/site/src/compress_images.rs @@ -0,0 +1,105 @@ +use std::{ + fs::{self, File}, + path::{Path, PathBuf}, +}; + +use aho_corasick::{AhoCorasick, MatchKind}; +use config::ImageCompression; +use errors::Result; +use perceptual_image::{ + PerceptualCompressor, + encoders::{PerceptualAVIFEncoder, PerceptualJpegEncoder, PerceptualWebPEncoder}, +}; +use walkdir::WalkDir; + +pub fn compress_images( + static_path: &Path, + output_path: &Path, + globs: &Vec, +) -> Result<()> { + // Setup paths + let compressed_path = { + let mut compressed_path = PathBuf::from(static_path); + compressed_path.push("compressed_images"); + compressed_path + }; + + // Setup record of compressed files + let mut old_files = Vec::new(); + let mut new_files = Vec::new(); + + for item in globs { + let glob = globset::GlobBuilder::new(&item.glob).build()?.compile_matcher(); + for entry in WalkDir::new(&static_path) + .into_iter() + .filter_entry(|e| !e.path().starts_with(&compressed_path)) + { + let entry = entry?; + let input_path = entry.path(); + + // Search for matches to glob and compress + if glob.is_match(entry.path()) { + let mut output_path = + compressed_path.join(entry.path().strip_prefix(&static_path)?); + output_path.set_extension(item.format.file_extension()); + + fs::create_dir_all(&output_path.parent().unwrap())?; + old_files.push(input_path.strip_prefix(&static_path)?.display().to_string()); + new_files.push(output_path.strip_prefix(&static_path)?.display().to_string()); + + // Only run compression if output file does not already exist + if !Path::exists(&output_path) { + let file = File::create(output_path.clone())?; + let source = image::open(entry.path())?; + let compressor = PerceptualCompressor::new(&source) + .max_iterations(item.max_iterations as usize) + .target_score(item.target_ssim); + + match item.format { + config::ImageFormat::Avif => { + let encoder = PerceptualAVIFEncoder::new(); + compressor.encode(file, encoder)?; + } + config::ImageFormat::Jpeg => { + let encoder = PerceptualJpegEncoder::new(); + compressor.encode(file, encoder)?; + } + config::ImageFormat::Webp => { + let encoder = PerceptualWebPEncoder::new(); + compressor.encode(file, encoder)?; + } + } + } + } + } + } + + // Clean any orphaned files from compressed_images. + for entry in WalkDir::new(compressed_path.clone()) { + let entry = entry?; + let comp_path = entry.path().strip_prefix(&static_path)?.display().to_string(); + + if !entry.path().is_dir() && new_files.iter().find(|x| *x == &comp_path).is_none() { + fs::remove_file(entry.path())?; + } + } + + // Finally, rewrite references to file in output html. + let glob = globset::GlobBuilder::new("**/*.html").build()?.compile_matcher(); + for entry in WalkDir::new(output_path) { + let entry = entry?; + if glob.is_match(entry.path()) { + let contents = fs::read_to_string(entry.path())?; + + // Find any references to replaced files. + let new_contents = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(&old_files)? + .replace_all(&contents, &new_files); + + fs::write(entry.path(), new_contents)?; + } + } + + Ok(()) +} diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index ea5892ebe..1581adbef 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -1,3 +1,4 @@ +mod compress_images; pub mod feeds; pub mod link_checking; mod md_render; @@ -733,6 +734,14 @@ impl Site { // or from templates self.process_images()?; start = log_time(start, "Processed images"); + + if let Some(compress) = &self.config.compress_images + && self.build_mode == BuildMode::Disk + { + compress_images::compress_images(&self.static_path, &self.output_path, compress)?; + start = log_time(start, "Compressed images"); + } + // Processed images will be in static so the last step is to copy it self.copy_static_directories()?; log_time(start, "Copied static dir");