packfs: support streaming file

Author: dontpanic92

yaobow/packfs/src/lib.rs

@@ -1,5 +1,6 @@
 #![feature(io_error_more)]
 #![feature(cursor_remaining)]
+#![cfg_attr(target_os = "vita", stdarch_arm_neon_intrinsics)]
 
 pub mod cpk;
 pub mod fmb;
@@ -8,6 +9,7 @@ pub mod memory_file;
 pub mod pkg;
 pub mod plain_fs;
 pub mod sfb;
+pub mod streaming_file;
 pub mod ypk;
 pub mod zpk;
 pub mod zpkg;

yaobow/packfs/src/streaming_file.rs

@@ -0,0 +1,73 @@
+use std::{
+    io::{Read, Seek},
+    sync::{Arc, Mutex},
+};
+
+use common::SeekRead;
+use mini_fs::UserFile;
+
+pub struct StreamingFile {
+    reader: Arc<Mutex<dyn SeekRead + Send + Sync>>,
+    position: u64,
+    start_position: u64,
+    end_position: u64,
+}
+
+impl StreamingFile {
+    pub fn new(
+        reader: Arc<Mutex<dyn SeekRead + Send + Sync>>,
+        start_position: u64,
+        end_position: u64,
+    ) -> StreamingFile {
+        Self {
+            reader,
+            position: start_position,
+            start_position,
+            end_position,
+        }
+    }
+}
+
+impl UserFile for StreamingFile {}
+
+impl Read for StreamingFile {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        if self.position >= self.end_position {
+            return Ok(0);
+        }
+
+        let mut reader = self.reader.lock().unwrap();
+        reader.seek(std::io::SeekFrom::Start(self.position))?;
+        let read = reader.read(buf)?;
+        let read = read.min((self.end_position - self.position) as usize);
+        self.position += read as u64;
+        Ok(read)
+    }
+}
+
+impl Seek for StreamingFile {
+    fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+        let new_position = match pos {
+            std::io::SeekFrom::Start(offset) => self.start_position + offset,
+            std::io::SeekFrom::End(offset) => (self.end_position as i64 + offset) as u64,
+            std::io::SeekFrom::Current(offset) => (self.position as i64 + offset) as u64,
+        };
+
+        if new_position < self.start_position {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "Seek before start",
+            ));
+        }
+
+        if new_position > self.end_position {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "Seek after end",
+            ));
+        }
+
+        self.position = new_position;
+        Ok(self.position - self.start_position)
+    }
+}

yaobow/packfs/src/ypk/ypk_archive.rs

@@ -1,27 +1,31 @@
 use std::{
     collections::HashMap,
-    io::{Cursor, Write},
+    io::{Cursor, Read, Write},
+    ops::DerefMut,
+    sync::{Arc, Mutex},
 };
 
 use binrw::{binrw, BinRead, BinWrite};
 use common::{SeekRead, SeekWrite};
+use mini_fs::File;
 
-use crate::memory_file::MemoryFile;
+use crate::{memory_file::MemoryFile, streaming_file::StreamingFile};
 
 pub struct YpkArchive {
-    reader: Box<dyn SeekRead>,
+    reader: Arc<Mutex<dyn SeekRead + Send + Sync>>,
     pub entries: Vec<YpkEntry>,
     entries_hash: HashMap<u64, Vec<usize>>,
 }
 
 impl YpkArchive {
-    pub fn load(mut reader: Box<dyn SeekRead>) -> anyhow::Result<Self> {
-        let header = YpkHeader::read(&mut reader)?;
+    pub fn load(file: Arc<Mutex<dyn SeekRead + Send + Sync>>) -> anyhow::Result<Self> {
+        let mut reader = file.lock().unwrap();
+        let header = YpkHeader::read(&mut reader.deref_mut())?;
 
         reader.seek(std::io::SeekFrom::Start(header.entry_offset))?;
         let mut entries = Vec::with_capacity(header.entry_count as usize);
         for _ in 0..header.entry_count {
-            entries.push(YpkEntry::read(&mut reader)?);
+            entries.push(YpkEntry::read(&mut reader.deref_mut())?);
         }
 
         let mut entries_hash = HashMap::new();
@@ -32,31 +36,44 @@ impl YpkArchive {
                 .push(i);
         }
 
+        drop(reader);
+
         Ok(Self {
-            reader,
+            reader: file,
             entries,
             entries_hash,
         })
     }
 
-    pub fn open(&mut self, name: &str) -> std::io::Result<MemoryFile> {
-        let (offset, actual_size) = {
+    pub fn open(&mut self, name: &str) -> std::io::Result<File> {
+        let (offset, actual_size, is_compressed) = {
             let entry = self.get_entry(name).ok_or_else(|| {
                 std::io::Error::new(
                     std::io::ErrorKind::NotFound,
                     format!("Entry {:?} not found", name),
                 )
             })?;
 
-            (entry.offset, entry.actual_size)
+            (entry.offset, entry.actual_size, entry.is_compressed == 1)
         };
 
-        self.reader.seek(std::io::SeekFrom::Start(offset))?;
+        /*let mut reader = self.reader.lock().unwrap();
+        reader.seek(std::io::SeekFrom::Start(offset))?;
         let mut buf = vec![0; actual_size as usize];
-        self.reader.read_exact(&mut buf)?;
-
-        let buf = zstd::stream::decode_all::<&[u8]>(buf.as_ref());
-        Ok(MemoryFile::new(Cursor::new(buf.unwrap())))
+        reader.read_exact(&mut buf)?;*/
+
+        let mut streaming =
+            StreamingFile::new(self.reader.clone(), offset, offset + actual_size as u64);
+
+        if is_compressed {
+            let buf = zstd::stream::decode_all(&mut streaming)?;
+            return Ok(MemoryFile::new(Cursor::new(buf)).into());
+        } else {
+            /*let mut buf = Vec::new();
+            streaming.read_to_end(&mut buf)?;
+            return Ok(MemoryFile::new(Cursor::new(buf)).into());*/
+            return Ok(streaming.into());
+        }
     }
 
     fn get_entry(&self, name: &str) -> Option<&YpkEntry> {
@@ -102,6 +119,7 @@ pub struct YpkEntry {
     name: Vec<u8>,
 
     offset: u64,
+    is_compressed: u32,
     original_size: u32,
     actual_size: u32,
 }
@@ -129,17 +147,22 @@ impl YpkWriter {
         let offset = self.writer.stream_position()?;
         let original_size = data.len() as u32;
 
+        let (is_compressed, data) = if name.ends_with(".bik") {
+            (false, data.to_vec())
+        } else {
+            (true, zstd::stream::encode_all(data, 0)?)
+        };
+
         let name = normorlize_path(name);
         let name_for_hash = name.to_lowercase();
         let name = name.as_bytes();
 
-        let data = zstd::stream::encode_all(data, 0)?;
-
         self.entries.push(YpkEntry {
             hash: xxhash_rust::xxh3::xxh3_64(name_for_hash.as_bytes()),
             name_len: name.len() as u32,
             name: name.to_vec(),
             offset,
+            is_compressed: is_compressed as u32,
             original_size,
             actual_size: data.len() as u32,
         });

yaobow/packfs/src/ypk/ypk_fs.rs

@@ -1,7 +1,9 @@
-use crate::memory_file::MemoryFile;
-
 use mini_fs::{Entries, Store};
-use std::{cell::RefCell, path::Path};
+use std::{
+    cell::RefCell,
+    path::Path,
+    sync::{Arc, Mutex},
+};
 
 use super::YpkArchive;
 
@@ -12,14 +14,13 @@ pub struct YpkFs {
 impl YpkFs {
     pub fn new<P: AsRef<Path>>(ypk_path: P) -> anyhow::Result<YpkFs> {
         let file = std::fs::File::open(ypk_path.as_ref())?;
-        let archive = RefCell::new(YpkArchive::load(Box::new(file))?);
-
+        let archive = RefCell::new(YpkArchive::load(Arc::new(Mutex::new(file)))?);
         Ok(YpkFs { archive })
     }
 }
 
 impl Store for YpkFs {
-    type File = MemoryFile;
+    type File = mini_fs::File;
 
     fn open_path(&self, path: &Path) -> std::io::Result<Self::File> {
         self.archive.borrow_mut().open(path.to_str().unwrap())