Merge remote-tracking branch 'tweedegolf/master' into feature/remove-all-items

Author: ryan-summers

CHANGELOG.md

@@ -4,6 +4,18 @@
 
 ## Unreleased
 
+- Added a new `map::remove_all_items()` API to remove all stored items in flash.
+
+# 2.0.2 07-05-24
+
+- Added check for too big items that won't ever fit in flash so it returns a good clear error.
+
+# 2.0.1 06-05-24
+
+- Implemented the `get_len` function for all built-in key types
+
+# 2.0.0 06-05-24
+
 - *Breaking:* Made the cache API a bit more strict. Caches now always have to be passed as a mutable reference.
   The API before would lead to a lot of extra unncesessary binary size.
 - *Breaking:* Removed the `StorageItem` trait in favor of two separate `Key` and `Value` traits. This helps cut
@@ -13,7 +25,6 @@
 - Added `erase_all` function as a helper to erase the flash in a region.
 - *Breaking:* Changed the way that queue iteration works. Now there's an `iter` function instead of two separate `peek_many` and `pop_many` functions. The new iter returns an entry from which you can get the data that was just peeked. If you want to pop it, then call the pop function on the entry.
 - Added `arrayvec` feature that when activated impls the `Key` trait for `ArrayVec` and `ArrayString`.
-- Added a new `map::remove_all_items()` API to remove all stored items in flash.
 
 ## 1.0.0 01-03-24
 
@@ -116,4 +127,4 @@
 
 ## 0.1.0 - 12-01-23
 
-- Initial release
+- Initial release

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sequential-storage"
-version = "2.0.0"
+version = "2.0.2"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A crate for storing data in flash with minimal erase cycles."

README.md

@@ -39,6 +39,16 @@ That way you've covered the worst-case execution time for that part of your appl
 
 A cache performance regression might be a bug though. Open an issue to discus your situation if you find a regression.
 
+## MSRV
+
+This crate has no further guarantees other than being able to run on the latest stable compiler.  
+Increasing the MSRV is not seen as a breaking change semver-wise.  
+If you find yourself in trouble with this, feel free to open an issue.
+
+## Example
+
+See the `map` and `queue` module level documentation for examples.
+
 ## Features
 
 - Key value datastore (Map)

build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo::rustc-check-cfg=cfg(fuzzing_repro)");
+}

example/Cargo.lock

@@ -178,19 +178,19 @@ impl ItemHeader {
     }
 
     /// Get the address of the start of the data for this item
-    pub fn data_address<S: NorFlash>(address: u32) -> u32 {
+    pub const fn data_address<S: NorFlash>(address: u32) -> u32 {
         address + round_up_to_alignment::<S>(Self::LENGTH as u32)
     }
 
     /// Get the location of the next item in flash
-    pub fn next_item_address<S: NorFlash>(&self, address: u32) -> u32 {
+    pub const fn next_item_address<S: NorFlash>(&self, address: u32) -> u32 {
         let data_address = ItemHeader::data_address::<S>(address);
         data_address + round_up_to_alignment::<S>(self.length as u32)
     }
 
     /// Calculates the amount of bytes available for data.
     /// Essentially, it's the given amount minus the header and minus some alignment padding.
-    pub fn available_data_bytes<S: NorFlash>(total_available: u32) -> Option<u32> {
+    pub const fn available_data_bytes<S: NorFlash>(total_available: u32) -> Option<u32> {
         let data_start = Self::data_address::<S>(0);
         let data_end = round_down_to_alignment::<S>(total_available);
 

src/item.rs

@@ -148,6 +148,11 @@ const fn calculate_page_index<S: NorFlash>(flash_range: Range<u32>, address: u32
     (address - flash_range.start) as usize / S::ERASE_SIZE
 }
 
+const fn calculate_page_size<S: NorFlash>() -> usize {
+    // Page minus the two page status words
+    S::ERASE_SIZE - S::WORD_SIZE * 2
+}
+
 /// The marker being used for page states
 const MARKER: u8 = 0;
 
@@ -366,7 +371,6 @@ pub enum Error<S> {
         backtrace: std::backtrace::Backtrace,
     },
     /// The item cannot be stored anymore because the storage is full.
-    /// If you get this error some data may be lost.
     FullStorage,
     /// It's been detected that the memory is likely corrupted.
     /// You may want to erase the memory to recover.
@@ -381,6 +385,11 @@ pub enum Error<S> {
     BufferTooSmall(usize),
     /// A serialization error (from the key or value)
     SerializationError(SerializationError),
+    /// The item does not fit in flash, ever.
+    /// This is different from [Error::FullStorage] because this item is too big to fit even in empty flash.
+    ///
+    /// See the readme for more info about the constraints on item sizes.
+    ItemTooBig,
 }
 
 impl<S> From<SerializationError> for Error<S> {
@@ -416,6 +425,7 @@ where
                 "A provided buffer was to small to be used. Needed was {needed}"
             ),
             Error::SerializationError(value) => write!(f, "Map value error: {value}"),
+            Error::ItemTooBig => write!(f, "The item is too big to fit in the flash"),
         }
     }
 }

src/lib.rs

@@ -425,6 +425,15 @@ async fn store_item_inner<'d, K: Key, S: NorFlash>(
                     .serialize_into(&mut data_buffer[key_len..])
                     .map_err(Error::SerializationError)?;
 
+            if item_data_length > u16::MAX as usize
+                || item_data_length
+                    > calculate_page_size::<S>()
+                        .saturating_sub(ItemHeader::data_address::<S>(0) as usize)
+            {
+                cache.unmark_dirty();
+                return Err(Error::ItemTooBig);
+            }
+
             let free_spot_address = find_next_free_item_spot(
                 flash,
                 flash_range.clone(),
@@ -710,6 +719,10 @@ macro_rules! impl_key_num {
                     core::mem::size_of::<Self>(),
                 ))
             }
+
+            fn get_len(_buffer: &[u8]) -> Result<usize, SerializationError> {
+                Ok(core::mem::size_of::<Self>())
+            }
         }
     };
 }
@@ -741,6 +754,10 @@ impl<const N: usize> Key for [u8; N] {
 
         Ok((buffer[..N].try_into().unwrap(), N))
     }
+
+    fn get_len(_buffer: &[u8]) -> Result<usize, SerializationError> {
+        Ok(N)
+    }
 }
 
 /// The trait that defines how map values are serialized and deserialized.
@@ -1438,4 +1455,34 @@ mod tests {
             .is_none());
         }
     }
+
+    #[test]
+    async fn store_too_big_item() {
+        let mut flash = MockFlashBig::new(mock_flash::WriteCountCheck::Twice, None, true);
+        const FLASH_RANGE: Range<u32> = 0x000..0x1000;
+
+        store_item(
+            &mut flash,
+            FLASH_RANGE,
+            &mut cache::NoCache::new(),
+            &mut [0; 1024],
+            0u8,
+            &[0; 1024 - 4 * 2 - 8 - 1],
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(
+            store_item(
+                &mut flash,
+                FLASH_RANGE,
+                &mut cache::NoCache::new(),
+                &mut [0; 1024],
+                0u8,
+                &[0; 1024 - 4 * 2 - 8 - 1 + 1],
+            )
+            .await,
+            Err(Error::ItemTooBig)
+        );
+    }
 }

src/map.rs

@@ -108,12 +108,12 @@ async fn push_inner<S: NorFlash>(
     }
 
     // Data must fit in a single page
-    if data.len()
-        > ItemHeader::available_data_bytes::<S>((S::ERASE_SIZE - S::WORD_SIZE * 2) as u32).unwrap()
-            as usize
+    if data.len() > u16::MAX as usize
+        || data.len()
+            > calculate_page_size::<S>().saturating_sub(ItemHeader::data_address::<S>(0) as usize)
     {
         cache.unmark_dirty();
-        return Err(Error::BufferTooBig);
+        return Err(Error::ItemTooBig);
     }
 
     let current_page = find_youngest_page(flash, flash_range.clone(), cache).await?;
@@ -1300,4 +1300,32 @@ mod tests {
             0
         );
     }
+
+    #[test]
+    async fn store_too_big_item() {
+        let mut flash = MockFlashBig::new(WriteCountCheck::Twice, None, true);
+        const FLASH_RANGE: Range<u32> = 0x000..0x1000;
+
+        push(
+            &mut flash,
+            FLASH_RANGE,
+            &mut cache::NoCache::new(),
+            &[0; 1024 - 4 * 2 - 8],
+            false,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(
+            push(
+                &mut flash,
+                FLASH_RANGE,
+                &mut cache::NoCache::new(),
+                &[0; 1024 - 4 * 2 - 8 + 1],
+                false,
+            )
+            .await,
+            Err(Error::ItemTooBig)
+        );
+    }
 }