From 59f84625257a54c3bf74e0727a85a00f6fc8f457 Mon Sep 17 00:00:00 2001 From: Crauzer Date: Sat, 22 Feb 2025 22:29:26 +0100 Subject: [PATCH 1/7] feat: start work on builder --- crates/league-toolkit/Cargo.toml | 1 + crates/league-toolkit/src/core/wad/builder.rs | 60 +++++++++++++++++++ crates/league-toolkit/src/core/wad/mod.rs | 3 + 3 files changed, 64 insertions(+) create mode 100644 crates/league-toolkit/src/core/wad/builder.rs diff --git a/crates/league-toolkit/Cargo.toml b/crates/league-toolkit/Cargo.toml index aeac964e..e4eabbf0 100644 --- a/crates/league-toolkit/Cargo.toml +++ b/crates/league-toolkit/Cargo.toml @@ -29,6 +29,7 @@ io-ext = { path = "../io-ext" } league-primitives = { path = "../league-primitives" } zstd = { version = "0.13", default-features = false, optional = true } ruzstd = { version = "0.7", optional = true } +xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } serde = { version = "1.0.204", features = ["derive"], optional = true } paste = "1.0.15" diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs new file mode 100644 index 00000000..58cfd823 --- /dev/null +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -0,0 +1,60 @@ +use std::io::{self, BufReader, BufWriter}; + +use xxhash_rust::xxh64; + +use super::{WadChunk, WadChunkCompression, WadError}; + +pub enum WadBuilderError { + WadError(WadError), +} + +pub struct WadBuilder { + chunk_builders: Vec, +} + +impl WadBuilder { + pub fn new() -> Self { + Self { + chunk_builders: Vec::new(), + } + } + + pub fn with_chunk(mut self, chunk: WadChunkBuilder) -> Self { + self.chunk_builders.push(chunk); + self + } + + pub fn build_to_writer( + self, + writer: W, + provide_chunk_data: impl Fn(u64, &mut BufWriter<&mut [u8]>) -> Result<(), WadBuilderError>, + ) -> Result<(), WadBuilderError> { + todo!() + + // First we need to write a dummy header and TOC, so we can calculate from where to start writing the chunks + } +} + +pub struct WadChunkBuilder { + path: u64, + force_compression: Option, +} + +impl WadChunkBuilder { + pub fn new() -> Self { + Self { + path: 0, + force_compression: None, + } + } + + pub fn with_path(mut self, path: impl AsRef) -> Self { + self.path = xxh64::xxh64(path.as_ref().to_lowercase().as_bytes(), 0); + self + } + + pub fn with_force_compression(mut self, compression: WadChunkCompression) -> Self { + self.force_compression = Some(compression); + self + } +} diff --git a/crates/league-toolkit/src/core/wad/mod.rs b/crates/league-toolkit/src/core/wad/mod.rs index 1e2beb5e..63d20048 100644 --- a/crates/league-toolkit/src/core/wad/mod.rs +++ b/crates/league-toolkit/src/core/wad/mod.rs @@ -1,8 +1,11 @@ //! Wad file handling + +mod builder; mod chunk; mod decoder; mod error; +pub use builder::*; pub use chunk::*; pub use decoder::*; pub use error::*; From e9ade67191e3d986d3e4888013990d2f9d351c12 Mon Sep 17 00:00:00 2001 From: Crauzer Date: Sun, 23 Feb 2025 17:38:22 +0100 Subject: [PATCH 2/7] feat: add builder --- crates/league-toolkit/Cargo.toml | 2 +- crates/league-toolkit/src/core/wad/builder.rs | 170 ++++++++++++++-- crates/league-toolkit/src/core/wad/chunk.rs | 108 +++++++++- crates/league-toolkit/src/core/wad/mod.rs | 2 +- crates/league-toolkit/src/util/league_file.rs | 191 ++++++++++++++++++ crates/league-toolkit/src/util/mod.rs | 1 + 6 files changed, 446 insertions(+), 28 deletions(-) create mode 100644 crates/league-toolkit/src/util/league_file.rs diff --git a/crates/league-toolkit/Cargo.toml b/crates/league-toolkit/Cargo.toml index e4eabbf0..5261601b 100644 --- a/crates/league-toolkit/Cargo.toml +++ b/crates/league-toolkit/Cargo.toml @@ -30,7 +30,7 @@ league-primitives = { path = "../league-primitives" } zstd = { version = "0.13", default-features = false, optional = true } ruzstd = { version = "0.7", optional = true } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } - +itertools = "0.14.0" serde = { version = "1.0.204", features = ["derive"], optional = true } paste = "1.0.15" miette = "7.2.0" diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs index 58cfd823..fff6e336 100644 --- a/crates/league-toolkit/src/core/wad/builder.rs +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -1,53 +1,181 @@ -use std::io::{self, BufReader, BufWriter}; +use std::io::{self, BufReader, BufWriter, Cursor, Read, Seek, SeekFrom, Write}; -use xxhash_rust::xxh64; +use byteorder::{WriteBytesExt, LE}; +use flate2::read::GzEncoder; +use io_ext::measure; +use itertools::Itertools; +use xxhash_rust::{xxh3, xxh64}; use super::{WadChunk, WadChunkCompression, WadError}; +use crate::util::league_file::{get_ideal_compression_for_league_file, identify_league_file}; + +#[derive(Debug, thiserror::Error)] pub enum WadBuilderError { - WadError(WadError), + #[error("wad error")] + WadError(#[from] WadError), + + #[error("io error")] + IoError(#[from] io::Error), + + #[error("unsupported compression type: {0}")] + UnsupportedCompressionType(WadChunkCompression), } +#[derive(Debug, Default)] pub struct WadBuilder { chunk_builders: Vec, } impl WadBuilder { - pub fn new() -> Self { - Self { - chunk_builders: Vec::new(), - } - } - pub fn with_chunk(mut self, chunk: WadChunkBuilder) -> Self { self.chunk_builders.push(chunk); self } - pub fn build_to_writer( + /// Build the WAD file and write it to the given writer. + /// + /// * `writer` - The writer to write the WAD file to. + /// * `provide_chunk_data` - A function that provides the rawdata for each chunk. + pub fn build_to_writer< + TWriter: io::Write + io::Seek, + TChunkDataProvider: Fn(u64, &mut Cursor<&mut Vec>) -> Result<(), WadBuilderError>, + >( self, - writer: W, - provide_chunk_data: impl Fn(u64, &mut BufWriter<&mut [u8]>) -> Result<(), WadBuilderError>, + writer: TWriter, + provide_chunk_data: TChunkDataProvider, ) -> Result<(), WadBuilderError> { - todo!() - // First we need to write a dummy header and TOC, so we can calculate from where to start writing the chunks + let mut writer = BufWriter::new(writer); + + let (header_toc_size, toc_offset) = self.write_dummy_toc::(&mut writer)?; + + let ordered_chunks = self + .chunk_builders + .iter() + .sorted_by_key(|chunk| chunk.path) + .collect::>(); + + let mut final_chunks = Vec::new(); + + let mut current_data_offset = toc_offset + header_toc_size; + for chunk in ordered_chunks { + let mut chunk_data = Vec::new(); + provide_chunk_data(chunk.path, &mut Cursor::new(&mut chunk_data))?; + + let chunk_data_size = chunk_data.len(); + let compressed_data = Self::compress_chunk_data(&chunk_data, chunk.force_compression)?; + let compressed_data_size = compressed_data.len(); + let compressed_checksum = xxh3::xxh3_64(&compressed_data); + + writer.write_all(&compressed_data)?; + + current_data_offset = current_data_offset.wrapping_add(compressed_data_size as u64); + + final_chunks.push(WadChunk { + path_hash: chunk.path, + data_offset: current_data_offset as usize, + compressed_size: compressed_data_size, + uncompressed_size: chunk_data_size, + compression_type: WadChunkCompression::Zstd, + is_duplicated: false, + frame_count: 0, + start_frame: 0, + checksum: compressed_checksum, + }); + } + + writer.seek(SeekFrom::Start(toc_offset))?; + + for chunk in final_chunks { + chunk.write_v3_4(&mut writer)?; + } + + Ok(()) + } + + fn write_dummy_toc( + &self, + writer: &mut BufWriter, + ) -> Result<(u64, u64), WadBuilderError> { + let (header_toc_size, toc_offset) = measure(writer, |writer| { + // Write the header + writer.write_u16::(0x5752)?; + writer.write_u8(3)?; // major + writer.write_u8(4)?; // minor + + // Write dummy ECDSA signature + writer.write_all(&[0; 256])?; + writer.write_u64::(0)?; + + // Write dummy TOC + let toc_offset = writer.stream_position()?; + for _ in self.chunk_builders.iter() { + writer.write_all(&[0; 32])?; + } + + Ok::<_, WadBuilderError>(toc_offset) + })?; + + Ok((header_toc_size, toc_offset)) + } + + fn compress_chunk_data( + data: &[u8], + force_compression: Option, + ) -> Result, WadBuilderError> { + let compressed_data = match force_compression { + Some(compression) => Self::compress_chunk_data_by_compression(data, compression)?, + None => { + let kind = identify_league_file(data); + let compression = get_ideal_compression_for_league_file(kind); + + Self::compress_chunk_data_by_compression(data, compression)? + } + }; + + Ok(compressed_data) + } + + fn compress_chunk_data_by_compression( + data: &[u8], + compression: WadChunkCompression, + ) -> Result, WadBuilderError> { + let mut compressed_data = Vec::new(); + match compression { + WadChunkCompression::None => { + compressed_data = data.to_vec(); + } + WadChunkCompression::GZip => { + let reader = BufReader::new(data); + let mut encoder = GzEncoder::new(reader, flate2::Compression::default()); + + encoder.read_to_end(&mut compressed_data)?; + } + WadChunkCompression::Zstd => { + zstd::Encoder::new(BufWriter::new(&mut compressed_data), 3)?.write_all(data)? + } + WadChunkCompression::Satellite => { + return Err(WadBuilderError::UnsupportedCompressionType(compression)); + } + WadChunkCompression::ZstdMulti => { + return Err(WadBuilderError::UnsupportedCompressionType(compression)); + } + } + + Ok(compressed_data) } } +#[derive(Debug, Clone, Copy, Default)] pub struct WadChunkBuilder { path: u64, + + /// If provided, the chunk will be compressed using the given compression type, otherwise the ideal compression will be used. force_compression: Option, } impl WadChunkBuilder { - pub fn new() -> Self { - Self { - path: 0, - force_compression: None, - } - } - pub fn with_path(mut self, path: impl AsRef) -> Self { self.path = xxh64::xxh64(path.as_ref().to_lowercase().as_bytes(), 0); self diff --git a/crates/league-toolkit/src/core/wad/chunk.rs b/crates/league-toolkit/src/core/wad/chunk.rs index 268d3721..e221e0cb 100644 --- a/crates/league-toolkit/src/core/wad/chunk.rs +++ b/crates/league-toolkit/src/core/wad/chunk.rs @@ -1,6 +1,7 @@ use std::io::{BufReader, Read}; +use std::{fmt, io}; -use byteorder::{ReadBytesExt as _, LE}; +use byteorder::{ReadBytesExt as _, WriteBytesExt as _, LE}; use num_enum::{IntoPrimitive, TryFromPrimitive}; use super::WadError; @@ -16,6 +17,12 @@ pub enum WadChunkCompression { ZstdMulti = 4, } +impl fmt::Display for WadChunkCompression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] /// A single wad chunk @@ -27,12 +34,12 @@ pub struct WadChunk { pub compression_type: WadChunkCompression, pub is_duplicated: bool, pub frame_count: u8, - pub start_frame: u16, + pub start_frame: u32, pub checksum: u64, } impl WadChunk { - pub(crate) fn read(reader: &mut BufReader) -> Result { + pub fn read_v3_1(reader: &mut BufReader) -> Result { let path_hash = reader.read_u64::()?; let data_offset = reader.read_u32::()? as usize; let compressed_size = reader.read_i32::()? as usize; @@ -41,7 +48,9 @@ impl WadChunk { let type_frame_count = reader.read_u8()?; let frame_count = type_frame_count >> 4; let compression_type = WadChunkCompression::try_from_primitive(type_frame_count & 0xF) - .expect("failed to read chunk compression"); + .map_err(|_| WadError::InvalidChunkCompression { + compression: type_frame_count & 0xF, + })?; let is_duplicated = reader.read_u8()? == 1; let start_frame = reader.read_u16::()?; @@ -55,11 +64,57 @@ impl WadChunk { compression_type, is_duplicated, frame_count, - start_frame, + start_frame: start_frame as u32, checksum, }) } + pub fn read_v3_4(reader: &mut BufReader) -> Result { + let path_hash = reader.read_u64::()?; + let data_offset = reader.read_u32::()? as usize; + let compressed_size = reader.read_u32::()? as usize; + let uncompressed_size = reader.read_u32::()? as usize; + + let type_frame_count = reader.read_u8()?; + let frame_count = type_frame_count >> 4; + let compression_type = WadChunkCompression::try_from_primitive(type_frame_count & 0xF) + .map_err(|_| WadError::InvalidChunkCompression { + compression: type_frame_count & 0xF, + })?; + + let start_frame = read_24_bit_subchunk_start_frame(reader)?; + + let checksum = reader.read_u64::()?; + + Ok(WadChunk { + path_hash, + data_offset, + compressed_size, + uncompressed_size, + compression_type, + is_duplicated: false, // v3.4 always has is_duplicated = false + frame_count, + start_frame: start_frame as u32, + checksum, + }) + } + + pub fn write_v3_4(&self, writer: &mut W) -> Result<(), WadError> { + writer.write_u64::(self.path_hash)?; + writer.write_u32::(self.data_offset as u32)?; + writer.write_u32::(self.compressed_size as u32)?; + writer.write_u32::(self.uncompressed_size as u32)?; + + let type_frame_count = (self.frame_count << 4) | (self.compression_type as u8 & 0xF); + writer.write_u8(type_frame_count)?; + + write_24_bit_subchunk_start_frame(writer, self.start_frame)?; + + writer.write_u64::(self.checksum)?; + + Ok(()) + } + pub fn path_hash(&self) -> u64 { self.path_hash } @@ -79,3 +134,46 @@ impl WadChunk { self.checksum } } + +pub(crate) fn read_24_bit_subchunk_start_frame( + reader: &mut BufReader, +) -> Result { + let start_frame_hi = reader.read_u8()? as u32; + let start_frame_lo = reader.read_u8()? as u32; + let start_frame_mi = reader.read_u8()? as u32; + let start_frame = start_frame_hi << 16 | start_frame_mi << 8 | start_frame_lo; + + Ok(start_frame) +} + +pub(crate) fn write_24_bit_subchunk_start_frame( + writer: &mut W, + start_frame: u32, +) -> Result<(), WadError> { + writer.write_u8(((start_frame >> 16) & 0xFF) as u8)?; + writer.write_u8((start_frame & 0xFF) as u8)?; + writer.write_u8(((start_frame >> 8) & 0xFF) as u8)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::io::{BufReader, Cursor}; + + use crate::core::wad::{read_24_bit_subchunk_start_frame, write_24_bit_subchunk_start_frame}; + + #[test] + fn test_read_24_bit_subchunk_start_frame() { + let mut reader = BufReader::new(Cursor::new([0x01, 0x03, 0x02])); + let start_frame = read_24_bit_subchunk_start_frame(&mut reader).unwrap(); + assert_eq!(start_frame, 0x010203); + } + + #[test] + fn test_write_24_bit_subchunk_start_frame() { + let mut writer = Vec::new(); + write_24_bit_subchunk_start_frame(&mut writer, 0x010302).unwrap(); + assert_eq!(writer, [0x01, 0x02, 0x03]); + } +} diff --git a/crates/league-toolkit/src/core/wad/mod.rs b/crates/league-toolkit/src/core/wad/mod.rs index 63d20048..ca0e31a2 100644 --- a/crates/league-toolkit/src/core/wad/mod.rs +++ b/crates/league-toolkit/src/core/wad/mod.rs @@ -66,7 +66,7 @@ impl Wad { let chunk_count = reader.read_i32::()? as usize; let mut chunks = HashMap::::with_capacity(chunk_count); for _ in 0..chunk_count { - let chunk = WadChunk::read(&mut reader)?; + let chunk = WadChunk::read_v3_1(&mut reader)?; chunks .insert(chunk.path_hash(), chunk) .map_or(Ok(()), |chunk| { diff --git a/crates/league-toolkit/src/util/league_file.rs b/crates/league-toolkit/src/util/league_file.rs new file mode 100644 index 00000000..22b3015d --- /dev/null +++ b/crates/league-toolkit/src/util/league_file.rs @@ -0,0 +1,191 @@ +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +use crate::core::wad::WadChunkCompression; + +static LEAGUE_FILE_MAGIC_BYTES: &[LeagueFilePattern] = &[ + LeagueFilePattern::from_bytes(b"r3d2Mesh", LeagueFileKind::StaticMeshBinary), + LeagueFilePattern::from_bytes(b"r3d2sklt", LeagueFileKind::Skeleton), + LeagueFilePattern::from_bytes(b"r3d2ammd", LeagueFileKind::Animation), + LeagueFilePattern::from_bytes(b"r3d2canm", LeagueFileKind::Animation), + LeagueFilePattern::from_fn( + |data| u32::from_le_bytes(data[4..8].try_into().unwrap()) == 1, + 8, + LeagueFileKind::WwisePackage, + ), + LeagueFilePattern::from_fn(|data| &data[1..4] == b"PNG", 4, LeagueFileKind::Png), + LeagueFilePattern::from_bytes(b"DDS ", LeagueFileKind::TextureDds), + LeagueFilePattern::from_bytes(&[0x33, 0x22, 0x11, 0x00], LeagueFileKind::SimpleSkin), + LeagueFilePattern::from_bytes(b"PROP", LeagueFileKind::PropertyBin), + LeagueFilePattern::from_bytes(b"BKHD", LeagueFileKind::WwiseBank), + LeagueFilePattern::from_bytes(b"WGEO", LeagueFileKind::WorldGeometry), + LeagueFilePattern::from_bytes(b"OEGM", LeagueFileKind::MapGeometry), + LeagueFilePattern::from_bytes(b"[Obj", LeagueFileKind::StaticMeshAscii), + LeagueFilePattern::from_fn(|data| &data[1..5] == b"LuaQ", 5, LeagueFileKind::LuaObj), + LeagueFilePattern::from_bytes(b"PreLoad", LeagueFileKind::Preload), + LeagueFilePattern::from_fn( + |data| u32::from_le_bytes(data[..4].try_into().unwrap()) == 3, + 4, + LeagueFileKind::LightGrid, + ), + LeagueFilePattern::from_bytes(b"RST", LeagueFileKind::RiotStringTable), + LeagueFilePattern::from_bytes(b"PTCH", LeagueFileKind::PropertyBinOverride), + LeagueFilePattern::from_fn( + |data| ((u32::from_le_bytes(data[..4].try_into().unwrap()) & 0x00FFFFFF) == 0x00FFD8FF), + 3, + LeagueFileKind::Jpeg, + ), + LeagueFilePattern::from_fn( + |data| u32::from_le_bytes(data[4..8].try_into().unwrap()) == 0x22FD4FC3, + 8, + LeagueFileKind::Skeleton, + ), + LeagueFilePattern::from_bytes(b"TEX\0", LeagueFileKind::Texture), + LeagueFilePattern::from_bytes(b" bool), +} + +struct LeagueFilePattern { + pattern: LeagueFilePatternKind, + min_length: usize, + kind: LeagueFileKind, +} + +impl LeagueFilePattern { + const fn from_bytes(bytes: &'static [u8], kind: LeagueFileKind) -> Self { + Self { + pattern: LeagueFilePatternKind::Bytes(bytes), + min_length: bytes.len(), + kind, + } + } + + const fn from_fn(f: fn(&[u8]) -> bool, min_length: usize, kind: LeagueFileKind) -> Self { + Self { + pattern: LeagueFilePatternKind::Fn(f), + min_length, + kind, + } + } + + fn matches(&self, data: &[u8]) -> bool { + data.len() >= self.min_length + && match self.pattern { + LeagueFilePatternKind::Bytes(bytes) => &data[..bytes.len()] == bytes, + LeagueFilePatternKind::Fn(f) => f(data), + } + } +} + +/// Get the league file kind from an extension. +pub fn get_league_file_kind_from_extension(extension: impl AsRef) -> LeagueFileKind { + if extension.as_ref().len() == 0 { + return LeagueFileKind::Unknown; + } + + match match extension.as_ref().starts_with('.') { + true => &extension.as_ref()[1..], + false => extension.as_ref(), + } { + "anm" => LeagueFileKind::Animation, + "bin" => LeagueFileKind::PropertyBin, + "bnk" => LeagueFileKind::WwiseBank, + "dds" => LeagueFileKind::TextureDds, + "jpg" => LeagueFileKind::Jpeg, + "luaobj" => LeagueFileKind::LuaObj, + "mapgeo" => LeagueFileKind::MapGeometry, + "png" => LeagueFileKind::Png, + "preload" => LeagueFileKind::Preload, + "scb" => LeagueFileKind::StaticMeshBinary, + "sco" => LeagueFileKind::StaticMeshAscii, + "skl" => LeagueFileKind::Skeleton, + "skn" => LeagueFileKind::SimpleSkin, + "stringtable" => LeagueFileKind::RiotStringTable, + "svg" => LeagueFileKind::Svg, + "tex" => LeagueFileKind::Texture, + "wgeo" => LeagueFileKind::WorldGeometry, + "wpk" => LeagueFileKind::WwisePackage, + _ => LeagueFileKind::Unknown, + } +} + +/// Get the extension for a league file kind. +pub fn get_extension_from_league_file_kind(kind: LeagueFileKind) -> &'static str { + match kind { + LeagueFileKind::Animation => "anm", + LeagueFileKind::Jpeg => "jpg", + LeagueFileKind::LightGrid => "lightgrid", + LeagueFileKind::LuaObj => "luaobj", + LeagueFileKind::MapGeometry => "mapgeo", + LeagueFileKind::Png => "png", + LeagueFileKind::Preload => "preload", + LeagueFileKind::PropertyBin => "bin", + LeagueFileKind::PropertyBinOverride => "bin", + LeagueFileKind::RiotStringTable => "stringtable", + LeagueFileKind::SimpleSkin => "skn", + LeagueFileKind::Skeleton => "skl", + LeagueFileKind::StaticMeshAscii => "sco", + LeagueFileKind::StaticMeshBinary => "scb", + LeagueFileKind::Texture => "tex", + LeagueFileKind::TextureDds => "dds", + LeagueFileKind::Unknown => "", + LeagueFileKind::WorldGeometry => "wgeo", + LeagueFileKind::WwiseBank => "bnk", + LeagueFileKind::WwisePackage => "wpk", + LeagueFileKind::Svg => "svg", + } +} + +/// Identify the kind of league file from the provided data. +pub fn identify_league_file(data: &[u8]) -> LeagueFileKind { + for magic_byte in LEAGUE_FILE_MAGIC_BYTES.iter() { + if magic_byte.matches(data) { + return magic_byte.kind; + } + } + + return LeagueFileKind::Unknown; +} + +/// Get the ideal compression for a league file. +pub fn get_ideal_compression_for_league_file(kind: LeagueFileKind) -> WadChunkCompression { + match kind { + LeagueFileKind::WwisePackage | LeagueFileKind::WwiseBank => WadChunkCompression::None, + _ => WadChunkCompression::Zstd, + } +} diff --git a/crates/league-toolkit/src/util/mod.rs b/crates/league-toolkit/src/util/mod.rs index ec5d33c1..9bd1a317 100644 --- a/crates/league-toolkit/src/util/mod.rs +++ b/crates/league-toolkit/src/util/mod.rs @@ -1 +1,2 @@ pub mod hash; +pub mod league_file; From 1a9b8f4903bab7c87103299bfc9120e637416a01 Mon Sep 17 00:00:00 2001 From: Crauzer Date: Tue, 25 Feb 2025 08:58:08 +0100 Subject: [PATCH 3/7] fix: start using new file kind api --- crates/league-toolkit/src/core/wad/builder.rs | 9 +- crates/league-toolkit/src/league_file/kind.rs | 20 ++ .../league-toolkit/src/league_file/pattern.rs | 3 +- crates/league-toolkit/src/util/league_file.rs | 191 ------------------ crates/league-toolkit/src/util/mod.rs | 1 - 5 files changed, 25 insertions(+), 199 deletions(-) delete mode 100644 crates/league-toolkit/src/util/league_file.rs diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs index fff6e336..1679bfab 100644 --- a/crates/league-toolkit/src/core/wad/builder.rs +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -6,9 +6,9 @@ use io_ext::measure; use itertools::Itertools; use xxhash_rust::{xxh3, xxh64}; -use super::{WadChunk, WadChunkCompression, WadError}; +use crate::league_file::LeagueFileKind; -use crate::util::league_file::{get_ideal_compression_for_league_file, identify_league_file}; +use super::{WadChunk, WadChunkCompression, WadError}; #[derive(Debug, thiserror::Error)] pub enum WadBuilderError { @@ -127,10 +127,9 @@ impl WadBuilder { let compressed_data = match force_compression { Some(compression) => Self::compress_chunk_data_by_compression(data, compression)?, None => { - let kind = identify_league_file(data); - let compression = get_ideal_compression_for_league_file(kind); + let kind = LeagueFileKind::identify_from_bytes(data); - Self::compress_chunk_data_by_compression(data, compression)? + Self::compress_chunk_data_by_compression(data, kind.ideal_compression())? } }; diff --git a/crates/league-toolkit/src/league_file/kind.rs b/crates/league-toolkit/src/league_file/kind.rs index 66271c4b..975adbb2 100644 --- a/crates/league-toolkit/src/league_file/kind.rs +++ b/crates/league-toolkit/src/league_file/kind.rs @@ -1,3 +1,5 @@ +use crate::core::wad::WadChunkCompression; + use super::pattern::LEAGUE_FILE_MAGIC_BYTES; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -142,4 +144,22 @@ impl LeagueFileKind { LeagueFileKind::Unknown } + + /// Get the ideal compression for this file type. + /// + /// # Examples + /// ``` + /// # use league_toolkit::league_file::*; + /// # + /// assert_eq!(LeagueFileKind::Animation.ideal_compression(), WadChunkCompression::Zstd); + /// assert_eq!(LeagueFileKind::WwisePackage.ideal_compression(), WadChunkCompression::None); + /// ``` + pub fn ideal_compression(&self) -> WadChunkCompression { + // TODO: Maybe we should move this into the wad module ? + + match self { + LeagueFileKind::WwisePackage | LeagueFileKind::WwiseBank => WadChunkCompression::None, + _ => WadChunkCompression::Zstd, + } + } } diff --git a/crates/league-toolkit/src/league_file/pattern.rs b/crates/league-toolkit/src/league_file/pattern.rs index 211a294b..27381d13 100644 --- a/crates/league-toolkit/src/league_file/pattern.rs +++ b/crates/league-toolkit/src/league_file/pattern.rs @@ -1,5 +1,3 @@ -use lazy_static::lazy_static; - use super::LeagueFileKind; pub static LEAGUE_FILE_MAGIC_BYTES: &[LeagueFilePattern] = &[ @@ -83,6 +81,7 @@ impl LeagueFilePattern { } } +#[cfg(test)] mod tests { use super::*; diff --git a/crates/league-toolkit/src/util/league_file.rs b/crates/league-toolkit/src/util/league_file.rs deleted file mode 100644 index 22b3015d..00000000 --- a/crates/league-toolkit/src/util/league_file.rs +++ /dev/null @@ -1,191 +0,0 @@ -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; - -use crate::core::wad::WadChunkCompression; - -static LEAGUE_FILE_MAGIC_BYTES: &[LeagueFilePattern] = &[ - LeagueFilePattern::from_bytes(b"r3d2Mesh", LeagueFileKind::StaticMeshBinary), - LeagueFilePattern::from_bytes(b"r3d2sklt", LeagueFileKind::Skeleton), - LeagueFilePattern::from_bytes(b"r3d2ammd", LeagueFileKind::Animation), - LeagueFilePattern::from_bytes(b"r3d2canm", LeagueFileKind::Animation), - LeagueFilePattern::from_fn( - |data| u32::from_le_bytes(data[4..8].try_into().unwrap()) == 1, - 8, - LeagueFileKind::WwisePackage, - ), - LeagueFilePattern::from_fn(|data| &data[1..4] == b"PNG", 4, LeagueFileKind::Png), - LeagueFilePattern::from_bytes(b"DDS ", LeagueFileKind::TextureDds), - LeagueFilePattern::from_bytes(&[0x33, 0x22, 0x11, 0x00], LeagueFileKind::SimpleSkin), - LeagueFilePattern::from_bytes(b"PROP", LeagueFileKind::PropertyBin), - LeagueFilePattern::from_bytes(b"BKHD", LeagueFileKind::WwiseBank), - LeagueFilePattern::from_bytes(b"WGEO", LeagueFileKind::WorldGeometry), - LeagueFilePattern::from_bytes(b"OEGM", LeagueFileKind::MapGeometry), - LeagueFilePattern::from_bytes(b"[Obj", LeagueFileKind::StaticMeshAscii), - LeagueFilePattern::from_fn(|data| &data[1..5] == b"LuaQ", 5, LeagueFileKind::LuaObj), - LeagueFilePattern::from_bytes(b"PreLoad", LeagueFileKind::Preload), - LeagueFilePattern::from_fn( - |data| u32::from_le_bytes(data[..4].try_into().unwrap()) == 3, - 4, - LeagueFileKind::LightGrid, - ), - LeagueFilePattern::from_bytes(b"RST", LeagueFileKind::RiotStringTable), - LeagueFilePattern::from_bytes(b"PTCH", LeagueFileKind::PropertyBinOverride), - LeagueFilePattern::from_fn( - |data| ((u32::from_le_bytes(data[..4].try_into().unwrap()) & 0x00FFFFFF) == 0x00FFD8FF), - 3, - LeagueFileKind::Jpeg, - ), - LeagueFilePattern::from_fn( - |data| u32::from_le_bytes(data[4..8].try_into().unwrap()) == 0x22FD4FC3, - 8, - LeagueFileKind::Skeleton, - ), - LeagueFilePattern::from_bytes(b"TEX\0", LeagueFileKind::Texture), - LeagueFilePattern::from_bytes(b" bool), -} - -struct LeagueFilePattern { - pattern: LeagueFilePatternKind, - min_length: usize, - kind: LeagueFileKind, -} - -impl LeagueFilePattern { - const fn from_bytes(bytes: &'static [u8], kind: LeagueFileKind) -> Self { - Self { - pattern: LeagueFilePatternKind::Bytes(bytes), - min_length: bytes.len(), - kind, - } - } - - const fn from_fn(f: fn(&[u8]) -> bool, min_length: usize, kind: LeagueFileKind) -> Self { - Self { - pattern: LeagueFilePatternKind::Fn(f), - min_length, - kind, - } - } - - fn matches(&self, data: &[u8]) -> bool { - data.len() >= self.min_length - && match self.pattern { - LeagueFilePatternKind::Bytes(bytes) => &data[..bytes.len()] == bytes, - LeagueFilePatternKind::Fn(f) => f(data), - } - } -} - -/// Get the league file kind from an extension. -pub fn get_league_file_kind_from_extension(extension: impl AsRef) -> LeagueFileKind { - if extension.as_ref().len() == 0 { - return LeagueFileKind::Unknown; - } - - match match extension.as_ref().starts_with('.') { - true => &extension.as_ref()[1..], - false => extension.as_ref(), - } { - "anm" => LeagueFileKind::Animation, - "bin" => LeagueFileKind::PropertyBin, - "bnk" => LeagueFileKind::WwiseBank, - "dds" => LeagueFileKind::TextureDds, - "jpg" => LeagueFileKind::Jpeg, - "luaobj" => LeagueFileKind::LuaObj, - "mapgeo" => LeagueFileKind::MapGeometry, - "png" => LeagueFileKind::Png, - "preload" => LeagueFileKind::Preload, - "scb" => LeagueFileKind::StaticMeshBinary, - "sco" => LeagueFileKind::StaticMeshAscii, - "skl" => LeagueFileKind::Skeleton, - "skn" => LeagueFileKind::SimpleSkin, - "stringtable" => LeagueFileKind::RiotStringTable, - "svg" => LeagueFileKind::Svg, - "tex" => LeagueFileKind::Texture, - "wgeo" => LeagueFileKind::WorldGeometry, - "wpk" => LeagueFileKind::WwisePackage, - _ => LeagueFileKind::Unknown, - } -} - -/// Get the extension for a league file kind. -pub fn get_extension_from_league_file_kind(kind: LeagueFileKind) -> &'static str { - match kind { - LeagueFileKind::Animation => "anm", - LeagueFileKind::Jpeg => "jpg", - LeagueFileKind::LightGrid => "lightgrid", - LeagueFileKind::LuaObj => "luaobj", - LeagueFileKind::MapGeometry => "mapgeo", - LeagueFileKind::Png => "png", - LeagueFileKind::Preload => "preload", - LeagueFileKind::PropertyBin => "bin", - LeagueFileKind::PropertyBinOverride => "bin", - LeagueFileKind::RiotStringTable => "stringtable", - LeagueFileKind::SimpleSkin => "skn", - LeagueFileKind::Skeleton => "skl", - LeagueFileKind::StaticMeshAscii => "sco", - LeagueFileKind::StaticMeshBinary => "scb", - LeagueFileKind::Texture => "tex", - LeagueFileKind::TextureDds => "dds", - LeagueFileKind::Unknown => "", - LeagueFileKind::WorldGeometry => "wgeo", - LeagueFileKind::WwiseBank => "bnk", - LeagueFileKind::WwisePackage => "wpk", - LeagueFileKind::Svg => "svg", - } -} - -/// Identify the kind of league file from the provided data. -pub fn identify_league_file(data: &[u8]) -> LeagueFileKind { - for magic_byte in LEAGUE_FILE_MAGIC_BYTES.iter() { - if magic_byte.matches(data) { - return magic_byte.kind; - } - } - - return LeagueFileKind::Unknown; -} - -/// Get the ideal compression for a league file. -pub fn get_ideal_compression_for_league_file(kind: LeagueFileKind) -> WadChunkCompression { - match kind { - LeagueFileKind::WwisePackage | LeagueFileKind::WwiseBank => WadChunkCompression::None, - _ => WadChunkCompression::Zstd, - } -} diff --git a/crates/league-toolkit/src/util/mod.rs b/crates/league-toolkit/src/util/mod.rs index 9bd1a317..ec5d33c1 100644 --- a/crates/league-toolkit/src/util/mod.rs +++ b/crates/league-toolkit/src/util/mod.rs @@ -1,2 +1 @@ pub mod hash; -pub mod league_file; From 0950a473170945809d620dacb8d94f9185c97b3d Mon Sep 17 00:00:00 2001 From: Crauzer Date: Tue, 25 Feb 2025 09:13:45 +0100 Subject: [PATCH 4/7] chore: docs --- crates/league-toolkit/src/core/wad/builder.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs index 1679bfab..09123357 100644 --- a/crates/league-toolkit/src/core/wad/builder.rs +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -22,6 +22,16 @@ pub enum WadBuilderError { UnsupportedCompressionType(WadChunkCompression), } +/// Implements a builder interface for creating WAD files. +/// +/// # Examples +/// ``` +/// # use league_toolkit::core::wad::*; +/// # +/// let builder = WadBuilder::default(); +/// builder.with_chunk(WadChunkBuilder::default().with_path("path/to/chunk")); +/// builder.build_to_writer(File::create("output.wad").unwrap()); +/// ``` #[derive(Debug, Default)] pub struct WadBuilder { chunk_builders: Vec, From ec985348d1afc15f6e5e204dffd18a3a5e4f036e Mon Sep 17 00:00:00 2001 From: Crauzer Date: Wed, 26 Feb 2025 19:28:06 +0100 Subject: [PATCH 5/7] feat: working tests --- crates/league-toolkit/src/core/wad/builder.rs | 94 +++++++++++++++---- crates/league-toolkit/src/core/wad/mod.rs | 7 +- 2 files changed, 81 insertions(+), 20 deletions(-) diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs index 09123357..ead99be3 100644 --- a/crates/league-toolkit/src/core/wad/builder.rs +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -49,17 +49,18 @@ impl WadBuilder { /// * `provide_chunk_data` - A function that provides the rawdata for each chunk. pub fn build_to_writer< TWriter: io::Write + io::Seek, - TChunkDataProvider: Fn(u64, &mut Cursor<&mut Vec>) -> Result<(), WadBuilderError>, + TChunkDataProvider: Fn(u64, &mut Cursor>) -> Result<(), WadBuilderError>, >( self, - writer: TWriter, + writer: &mut TWriter, provide_chunk_data: TChunkDataProvider, ) -> Result<(), WadBuilderError> { // First we need to write a dummy header and TOC, so we can calculate from where to start writing the chunks let mut writer = BufWriter::new(writer); - let (header_toc_size, toc_offset) = self.write_dummy_toc::(&mut writer)?; + let (_, toc_offset) = self.write_dummy_toc::(&mut writer)?; + // Sort the chunks by path hash, otherwise League wont load the WAD let ordered_chunks = self .chunk_builders .iter() @@ -68,26 +69,25 @@ impl WadBuilder { let mut final_chunks = Vec::new(); - let mut current_data_offset = toc_offset + header_toc_size; for chunk in ordered_chunks { - let mut chunk_data = Vec::new(); - provide_chunk_data(chunk.path, &mut Cursor::new(&mut chunk_data))?; + let mut cursor = Cursor::new(Vec::new()); + provide_chunk_data(chunk.path, &mut cursor)?; - let chunk_data_size = chunk_data.len(); - let compressed_data = Self::compress_chunk_data(&chunk_data, chunk.force_compression)?; + let chunk_data_size = cursor.get_ref().len(); + let (compressed_data, compression) = + Self::compress_chunk_data(cursor.get_ref(), chunk.force_compression)?; let compressed_data_size = compressed_data.len(); let compressed_checksum = xxh3::xxh3_64(&compressed_data); + let chunk_data_offset = writer.stream_position()?; writer.write_all(&compressed_data)?; - current_data_offset = current_data_offset.wrapping_add(compressed_data_size as u64); - final_chunks.push(WadChunk { path_hash: chunk.path, - data_offset: current_data_offset as usize, + data_offset: chunk_data_offset as usize, compressed_size: compressed_data_size, uncompressed_size: chunk_data_size, - compression_type: WadChunkCompression::Zstd, + compression_type: compression, is_duplicated: false, frame_count: 0, start_frame: 0, @@ -106,7 +106,7 @@ impl WadBuilder { fn write_dummy_toc( &self, - writer: &mut BufWriter, + writer: &mut BufWriter<&mut W>, ) -> Result<(u64, u64), WadBuilderError> { let (header_toc_size, toc_offset) = measure(writer, |writer| { // Write the header @@ -119,6 +119,7 @@ impl WadBuilder { writer.write_u64::(0)?; // Write dummy TOC + writer.write_u32::(self.chunk_builders.len() as u32)?; let toc_offset = writer.stream_position()?; for _ in self.chunk_builders.iter() { writer.write_all(&[0; 32])?; @@ -133,17 +134,22 @@ impl WadBuilder { fn compress_chunk_data( data: &[u8], force_compression: Option, - ) -> Result, WadBuilderError> { - let compressed_data = match force_compression { - Some(compression) => Self::compress_chunk_data_by_compression(data, compression)?, + ) -> Result<(Vec, WadChunkCompression), WadBuilderError> { + let (compressed_data, compression) = match force_compression { + Some(compression) => ( + Self::compress_chunk_data_by_compression(data, compression)?, + compression, + ), None => { let kind = LeagueFileKind::identify_from_bytes(data); + let compression = kind.ideal_compression(); + let compressed_data = Self::compress_chunk_data_by_compression(data, compression)?; - Self::compress_chunk_data_by_compression(data, kind.ideal_compression())? + (compressed_data, compression) } }; - Ok(compressed_data) + Ok((compressed_data, compression)) } fn compress_chunk_data_by_compression( @@ -162,7 +168,9 @@ impl WadBuilder { encoder.read_to_end(&mut compressed_data)?; } WadChunkCompression::Zstd => { - zstd::Encoder::new(BufWriter::new(&mut compressed_data), 3)?.write_all(data)? + let mut encoder = zstd::Encoder::new(BufWriter::new(&mut compressed_data), 3)?; + encoder.write_all(data)?; + encoder.finish()?; } WadChunkCompression::Satellite => { return Err(WadBuilderError::UnsupportedCompressionType(compression)); @@ -176,8 +184,19 @@ impl WadBuilder { } } +/// Implements a builder interface for creating WAD chunks. +/// +/// # Examples +/// ``` +/// # use league_toolkit::core::wad::*; +/// # +/// let builder = WadChunkBuilder::default(); +/// builder.with_path("path/to/chunk"); +/// builder.with_force_compression(WadChunkCompression::Zstd); +/// ``` #[derive(Debug, Clone, Copy, Default)] pub struct WadChunkBuilder { + /// The path hash of the chunk. Hashed using xxhash64. path: u64, /// If provided, the chunk will be compressed using the given compression type, otherwise the ideal compression will be used. @@ -195,3 +214,40 @@ impl WadChunkBuilder { self } } + +#[cfg(test)] +mod tests { + use crate::core::wad::Wad; + + use super::*; + + #[test] + fn test_wad_builder() { + let scratch = Vec::new(); + let mut cursor = Cursor::new(scratch); + + let mut builder = WadBuilder::default(); + builder = builder.with_chunk(WadChunkBuilder::default().with_path("test1")); + builder = builder.with_chunk(WadChunkBuilder::default().with_path("test2")); + builder = builder.with_chunk(WadChunkBuilder::default().with_path("test3")); + + builder + .build_to_writer(&mut cursor, |path, cursor| { + cursor.write_all(&[0xAA; 100])?; + + Ok(()) + }) + .expect("Failed to build WAD"); + + cursor.set_position(0); + + let wad = Wad::mount(cursor).expect("Failed to mount WAD"); + assert_eq!(wad.chunks().len(), 3); + + let chunk = wad.chunks.get(&xxh64::xxh64(b"test1", 0)).unwrap(); + assert_eq!(chunk.path_hash, xxh64::xxh64(b"test1", 0)); + assert_eq!(chunk.compressed_size, 17); + assert_eq!(chunk.uncompressed_size, 100); + assert_eq!(chunk.compression_type, WadChunkCompression::Zstd); + } +} diff --git a/crates/league-toolkit/src/core/wad/mod.rs b/crates/league-toolkit/src/core/wad/mod.rs index ca0e31a2..2f34bdc5 100644 --- a/crates/league-toolkit/src/core/wad/mod.rs +++ b/crates/league-toolkit/src/core/wad/mod.rs @@ -66,7 +66,12 @@ impl Wad { let chunk_count = reader.read_i32::()? as usize; let mut chunks = HashMap::::with_capacity(chunk_count); for _ in 0..chunk_count { - let chunk = WadChunk::read_v3_1(&mut reader)?; + let chunk = match (major, minor) { + (3, 1) => WadChunk::read_v3_1(&mut reader), + (3, 4) => WadChunk::read_v3_4(&mut reader), + _ => Err(WadError::InvalidVersion { major, minor }), + }?; + chunks .insert(chunk.path_hash(), chunk) .map_or(Ok(()), |chunk| { From 4a22324d9093c69e61d98c05555142779b1aab7a Mon Sep 17 00:00:00 2001 From: Crauzer Date: Wed, 26 Feb 2025 19:37:02 +0100 Subject: [PATCH 6/7] fix: compression display --- crates/league-toolkit/src/core/wad/chunk.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/league-toolkit/src/core/wad/chunk.rs b/crates/league-toolkit/src/core/wad/chunk.rs index e221e0cb..a6dcd2da 100644 --- a/crates/league-toolkit/src/core/wad/chunk.rs +++ b/crates/league-toolkit/src/core/wad/chunk.rs @@ -19,7 +19,13 @@ pub enum WadChunkCompression { impl fmt::Display for WadChunkCompression { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + match self { + WadChunkCompression::None => write!(f, "None"), + WadChunkCompression::GZip => write!(f, "GZip"), + WadChunkCompression::Satellite => write!(f, "Satellite"), + WadChunkCompression::Zstd => write!(f, "Zstd"), + WadChunkCompression::ZstdMulti => write!(f, "ZstdMulti"), + } } } From e201a14bb90ef14d31b61f78eeed5ec4020a00d7 Mon Sep 17 00:00:00 2001 From: Crauzer Date: Wed, 26 Feb 2025 19:46:07 +0100 Subject: [PATCH 7/7] fix: tests --- crates/league-toolkit/src/core/wad/builder.rs | 19 ++++++++++++++----- crates/league-toolkit/src/league_file/kind.rs | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/league-toolkit/src/core/wad/builder.rs b/crates/league-toolkit/src/core/wad/builder.rs index ead99be3..59aa7d9e 100644 --- a/crates/league-toolkit/src/core/wad/builder.rs +++ b/crates/league-toolkit/src/core/wad/builder.rs @@ -24,13 +24,22 @@ pub enum WadBuilderError { /// Implements a builder interface for creating WAD files. /// -/// # Examples +/// ## This example builds a WAD file in memory /// ``` /// # use league_toolkit::core::wad::*; -/// # -/// let builder = WadBuilder::default(); -/// builder.with_chunk(WadChunkBuilder::default().with_path("path/to/chunk")); -/// builder.build_to_writer(File::create("output.wad").unwrap()); +/// # use std::io::{Cursor, Write}; +/// +/// let mut builder = WadBuilder::default(); +/// let scratch = Vec::new(); +/// let mut wad_cursor = Cursor::new(scratch); +/// +/// builder = builder.with_chunk(WadChunkBuilder::default().with_path("path/to/chunk")); +/// builder.build_to_writer(&mut wad_cursor, |path, cursor| { +/// cursor.write_all(&[0xAA; 100])?; +/// +/// Ok(()) +/// }) +/// .expect("Failed to build WAD"); /// ``` #[derive(Debug, Default)] pub struct WadBuilder { diff --git a/crates/league-toolkit/src/league_file/kind.rs b/crates/league-toolkit/src/league_file/kind.rs index 975adbb2..05ae937e 100644 --- a/crates/league-toolkit/src/league_file/kind.rs +++ b/crates/league-toolkit/src/league_file/kind.rs @@ -150,6 +150,7 @@ impl LeagueFileKind { /// # Examples /// ``` /// # use league_toolkit::league_file::*; + /// # use league_toolkit::core::wad::WadChunkCompression; /// # /// assert_eq!(LeagueFileKind::Animation.ideal_compression(), WadChunkCompression::Zstd); /// assert_eq!(LeagueFileKind::WwisePackage.ideal_compression(), WadChunkCompression::None);