introduce MountedFileSizePolicy

Author: wasm-forge

CHANGELOG.md

@@ -1,9 +1,11 @@
 # Changelog
 
 
-## [v0.10.1]
+## [v0.11.0]
 - Fix mounting on absolute paths
 - Improved mounted memory size reset logic
+- API change: explicit mount file size policy added
+- Update dependencies
 
 ## [v0.10.0]
 - Optimized work with folders
@@ -12,29 +14,29 @@
 - Add fast direntry lookup by name
 
 ## [v0.9.0]
-- switch to ic-stable-structures v0.7
-- refactor project structure
-- integrate ic-test into project for the integration tests
-- update dependencies
+- Switch to ic-stable-structures v0.7
+- Refactor project structure
+- Integrate ic-test into project for the integration tests
+- Update dependencies
 
 
 ## [v0.8.1]
-- switch to ic-cdk v0.18.3
-- switch to pocket-ic v9.0
-- update dependencies
+- Switch to ic-cdk v0.18.3
+- Switch to pocket-ic v9.0
+- Update dependencies
 
 
 ## [v0.8.0]
-- improved Wasi compliance
-- change edition to 2024
-- update dependencies
-- bug fixes
-- faster file look-up
-- file read-write access control
-- fixed writing in append-mode
-- switch to pocket-ic v7.0
-- support for '..' in paths
-- return '.' and '..' during folder file list
+- Improved Wasi compliance
+- Change edition to 2024
+- Update dependencies
+- Bug fixes
+- Faster file look-up
+- File read-write access control
+- Fixed writing in append-mode
+- Switch to pocket-ic v7.0
+- Support for '..' in paths
+- Return '.' and '..' during folder file list
 
 
 ## [v0.7.3]
@@ -98,7 +100,7 @@
 ## [v0.5.0]
 - *API change:* init with memory manager using memory index range rather than first memory index.
 
-[v0.10.1]: https://github.com/wasm-forge/stable-fs/compare/v0.10.0...v0.10.1
+[v0.11.0]: https://github.com/wasm-forge/stable-fs/compare/v0.10.0...v0.11.0
 [v0.10.0]: https://github.com/wasm-forge/stable-fs/compare/v0.9.0...v0.10.0
 [v0.9.0]: https://github.com/wasm-forge/stable-fs/compare/v0.8.1...v0.9.0
 [v0.8.1]: https://github.com/wasm-forge/stable-fs/compare/v0.8.0...v0.8.1

Cargo.toml

@@ -11,7 +11,7 @@ resolver = "2"
 bitflags = "2.9.4"
 ciborium = "0.2.2"
 
-ic-cdk = "0.18.7"
+ic-cdk = "0.18.5"
 ic-stable-structures = "0.7.0"
 
 serde = "1.0.219"

stable-fs/src/fs.rs

@@ -12,7 +12,7 @@ use crate::{
     },
     storage::{
         Storage,
-        types::{DirEntry, DirEntryIndex, FileType, Metadata, Node},
+        types::{DirEntry, DirEntryIndex, FileType, Metadata, MountedFileSizePolicy, Node},
     },
 };
 
@@ -22,7 +22,7 @@ pub use crate::runtime::types::{
 };
 pub use crate::storage::types::FileSize;
 
-// The main class implementing the API to work with the file system.
+/// The main class implementing the API to work with the file system.
 pub struct FileSystem {
     pub(crate) root_fd: Fd,
     pub(crate) fd_table: FdTable,
@@ -185,6 +185,7 @@ impl FileSystem {
         &mut self,
         filename: &str,
         memory: Box<dyn Memory>,
+        policy: MountedFileSizePolicy,
     ) -> Result<(), Error> {
         let filename = filename.strip_prefix('/').unwrap_or(filename);
 
@@ -199,7 +200,7 @@ impl FileSystem {
 
         let result = (|| {
             let file = self.get_file(fd)?;
-            self.storage.mount_node(file.node, memory)
+            self.storage.mount_node(file.node, memory, policy)
         })();
 
         let _ = self.close(fd);

stable-fs/src/fs_tests.rs

@@ -5,7 +5,7 @@ mod tests {
 
     use crate::fs::FileSystem;
     use crate::runtime::types::Fd;
-    use crate::storage::types::Node;
+    use crate::storage::types::{MountedFileSizePolicy, Node};
 
     use ic_stable_structures::memory_manager::{MemoryId, MemoryManager};
     use ic_stable_structures::{Memory, VectorMemory};
@@ -815,8 +815,12 @@ mod tests {
 
         let root_fd = fs.root_fd();
 
-        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
-            .unwrap();
+        fs.mount_memory_file(
+            "test.txt",
+            Box::new(memory.clone()),
+            MountedFileSizePolicy::PreviousOrZero,
+        )
+        .unwrap();
 
         let content = "ABCDEFG123";
 
@@ -836,8 +840,12 @@ mod tests {
 
         let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
         let mut fs = FileSystem::new(Box::new(storage)).unwrap();
-        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
-            .unwrap();
+        fs.mount_memory_file(
+            "test.txt",
+            Box::new(memory.clone()),
+            MountedFileSizePolicy::PreviousOrZero,
+        )
+        .unwrap();
 
         let root_fd = fs.root_fd();
         let content = "ABCDEFG123";
@@ -846,8 +854,12 @@ mod tests {
         // imitate canister upgrade (we keep the memory manager but recreate the file system with the same virtual memories)
         let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
         let mut fs = FileSystem::new(Box::new(storage)).unwrap();
-        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
-            .unwrap();
+        fs.mount_memory_file(
+            "test.txt",
+            Box::new(memory.clone()),
+            MountedFileSizePolicy::PreviousOrZero,
+        )
+        .unwrap();
         let root_fd = fs.root_fd();
 
         let content = read_text_file(&mut fs, root_fd, "test.txt", 0, 100);
@@ -863,8 +875,12 @@ mod tests {
 
         let root_fd = fs.root_fd();
 
-        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
-            .unwrap();
+        fs.mount_memory_file(
+            "test.txt",
+            Box::new(memory.clone()),
+            MountedFileSizePolicy::PreviousOrZero,
+        )
+        .unwrap();
 
         let res = fs.remove_file(root_fd, "test.txt");
 
@@ -887,8 +903,12 @@ mod tests {
 
             let file_name = "test.txt";
 
-            fs.mount_memory_file(file_name, Box::new(memory1.clone()))
-                .unwrap();
+            fs.mount_memory_file(
+                file_name,
+                Box::new(memory1.clone()),
+                MountedFileSizePolicy::PreviousOrZero,
+            )
+            .unwrap();
 
             let content = "ABCDEFG123";
             let len = content.len();
@@ -921,8 +941,12 @@ mod tests {
             fs.unmount_memory_file(file_name).unwrap();
 
             // init new memory into a file
-            fs.mount_memory_file(file_name, Box::new(memory2.clone()))
-                .unwrap();
+            fs.mount_memory_file(
+                file_name,
+                Box::new(memory2.clone()),
+                MountedFileSizePolicy::PreviousOrZero,
+            )
+            .unwrap();
 
             fs.init_memory_file(file_name).unwrap();
 
@@ -947,8 +971,12 @@ mod tests {
             let hello_message = "Hello host".to_string();
             let hello_message2 = "1234Hello from regular file".to_string();
 
-            fs.mount_memory_file(file_name, Box::new(memory.clone()))
-                .unwrap();
+            fs.mount_memory_file(
+                file_name,
+                Box::new(memory.clone()),
+                MountedFileSizePolicy::PreviousOrZero,
+            )
+            .unwrap();
 
             // write something into a host memory file
             write_text_file(&mut fs, dir_fd, file_name, &hello_message, 1).unwrap();
@@ -966,8 +994,12 @@ mod tests {
             assert_eq!(str, "".to_string());
 
             // mount again, the old content should recover
-            fs.mount_memory_file(file_name, Box::new(memory.clone()))
-                .unwrap();
+            fs.mount_memory_file(
+                file_name,
+                Box::new(memory.clone()),
+                MountedFileSizePolicy::PreviousOrZero,
+            )
+            .unwrap();
             let str = read_text_file(&mut fs, dir_fd, file_name, 0, 1000);
             assert_eq!(str, hello_message);
 
@@ -985,8 +1017,12 @@ mod tests {
             assert_eq!(str, hello_message2);
 
             // after mounting, we still have the old content
-            fs.mount_memory_file(file_name, Box::new(memory.clone()))
-                .unwrap();
+            fs.mount_memory_file(
+                file_name,
+                Box::new(memory.clone()),
+                MountedFileSizePolicy::PreviousOrZero,
+            )
+            .unwrap();
             let str = read_text_file(&mut fs, dir_fd, file_name, 0, 1000);
             assert_eq!(str, hello_message);
 

stable-fs/src/storage.rs

@@ -3,7 +3,9 @@ use ic_stable_structures::Memory;
 use crate::{
     error::Error,
     fs::{ChunkSize, ChunkType},
-    storage::types::{DirEntry, DirEntryIndex, FileName, FileSize, Metadata, Node},
+    storage::types::{
+        DirEntry, DirEntryIndex, FileName, FileSize, Metadata, MountedFileSizePolicy, Node,
+    },
 };
 
 mod allocator;
@@ -28,7 +30,13 @@ pub trait Storage {
     fn new_node(&mut self) -> Node;
 
     /// mark node as mounted.
-    fn mount_node(&mut self, node: Node, memory: Box<dyn Memory>) -> Result<(), Error>;
+    fn mount_node(
+        &mut self,
+        node: Node,
+        memory: Box<dyn Memory>,
+        mount_policy: MountedFileSizePolicy,
+    ) -> Result<(), Error>;
+
     /// mark note as not mounted.
     fn unmount_node(&mut self, node: Node) -> Result<Box<dyn Memory>, Error>;
     /// return true if the node is mounted.

stable-fs/src/storage/dummy.rs

@@ -1,7 +1,7 @@
 use crate::{
     error::Error,
     fs::{ChunkSize, ChunkType},
-    storage::types::FileName,
+    storage::types::{FileName, MountedFileSizePolicy},
 };
 
 use super::{
@@ -64,6 +64,7 @@ impl Storage for DummyStorage {
         &mut self,
         _node: Node,
         _memory: Box<dyn ic_stable_structures::Memory>,
+        _policy: MountedFileSizePolicy,
     ) -> Result<(), Error> {
         panic!("Not supported")
     }

stable-fs/src/storage/stable.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, ops::Range};
 
-use crate::storage::types::{FileName, ZEROES};
+use crate::storage::types::{FileName, MountedFileSizePolicy, ZEROES};
 use ic_cdk::stable::WASM_PAGE_SIZE_IN_BYTES;
 use ic_stable_structures::{
     BTreeMap, Cell, Memory,
@@ -980,7 +980,12 @@ impl<M: Memory> Storage for StableStorage<M> {
         Ok(())
     }
 
-    fn mount_node(&mut self, node: Node, memory: Box<dyn Memory>) -> Result<(), Error> {
+    fn mount_node(
+        &mut self,
+        node: Node,
+        memory: Box<dyn Memory>,
+        policy: MountedFileSizePolicy,
+    ) -> Result<(), Error> {
         if self.is_mounted(node) {
             return Err(Error::DeviceOrResourceBusy);
         }
@@ -991,30 +996,23 @@ impl<M: Memory> Storage for StableStorage<M> {
 
         let memory_size = memory.size();
 
-        // activate mount
+        // activate mount, we use mounted metadata after this line
         self.active_mounts.insert(node, memory);
 
-        if let Ok(old_mounted_meta) = self.get_metadata(node) {
-            // the connected memory might have a different size from the old memory, we need to decide which size to use...
-            //
-            // Solution:
-            // if the new memory size in bytes is smaller than the old mounted metadata file size,
-            // we assume this is a new and unknown memory, and reinitialize the metadata size to match the total number of bytes
-            // provided in memory
+        let old_size = if let Ok(old_mounted_meta) = self.get_metadata(node) {
+            let size = old_mounted_meta.size;
+            file_meta = old_mounted_meta;
+            Some(size)
+        } else {
+            None
+        };
 
-            if memory_size * WASM_PAGE_SIZE_IN_BYTES < old_mounted_meta.size {
-                let mut meta = old_mounted_meta.clone();
+        let new_size = policy.get_mounted_file_size(old_size, memory_size);
 
-                meta.size = memory_size * WASM_PAGE_SIZE_IN_BYTES;
-                self.put_metadata(node, &meta)?;
-            }
-        } else {
-            // take a copy of the file meta, set the size to the size of the memory
-            file_meta.size = memory_size * WASM_PAGE_SIZE_IN_BYTES;
+        file_meta.size = new_size;
 
-            // update mounted metadata
-            self.put_metadata(node, &file_meta)?;
-        };
+        // update mounted metadata
+        self.put_metadata(node, &file_meta)?;
 
         Ok(())
     }
@@ -1062,7 +1060,7 @@ impl<M: Memory> Storage for StableStorage<M> {
             remainder -= to_read;
         }
 
-        self.mount_node(node, memory)?;
+        self.mount_node(node, memory, MountedFileSizePolicy::PreviousOrZero)?;
 
         self.put_metadata(node, &meta)?;
 
@@ -1102,7 +1100,7 @@ impl<M: Memory> Storage for StableStorage<M> {
 
         self.put_metadata(node, &meta)?;
 
-        self.mount_node(node, memory)?;
+        self.mount_node(node, memory, MountedFileSizePolicy::PreviousOrZero)?;
 
         Ok(())
     }