Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 4 additions & 62 deletions firewood/src/iter.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
// Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE.md for licensing terms.

mod filtered_key_range;
pub(crate) mod returnable;

use crate::merkle::{Key, Value};
use crate::v2::api::{KeyType, KeyValuePair};

pub use self::filtered_key_range::{FilteredKeyRangeExt, FilteredKeyRangeIter};
use firewood_storage::{
BranchNode, Child, FileIoError, NibblesIterator, Node, PathBuf, PathComponent, PathIterItem,
SharedNode, TriePathFromUnpackedBytes, TrieReader,
Expand Down Expand Up @@ -298,12 +301,6 @@ impl<'a, T: TrieReader> MerkleKeyValueIter<'a, T> {
iter: MerkleNodeIter::new(merkle, key.as_ref().into()),
}
}

/// Returns a new iterator that will emit key-value pairs up to and
/// including `last_key`.
pub fn stop_after_key<K: KeyType>(self, last_key: Option<K>) -> FilteredKeyRangeIter<Self, K> {
FilteredKeyRangeIter::new(self, last_key)
}
}

impl<T: TrieReader> Iterator for MerkleKeyValueIter<'_, T> {
Expand Down Expand Up @@ -568,61 +565,6 @@ fn key_from_nibble_iter<Iter: Iterator<Item = u8>>(mut nibbles: Iter) -> Key {
data.into_boxed_slice()
}

/// An iterator over key-value pairs that stops after a specified final key.
#[derive(Debug)]
#[must_use = "iterators are lazy and do nothing unless consumed"]
pub enum FilteredKeyRangeIter<I, K> {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: naming

FilteredKeyRangeIter::Unfiltered seems odd. Perhaps MaybeFilteredKeyRangeIter is better?

Unfiltered { iter: I },
Filtered { iter: I, last_key: K },
Exhausted,
}

impl<I: Iterator<Item = T>, T: KeyValuePair, K: KeyType> FilteredKeyRangeIter<I, K> {
/// Creates a new [`FilteredKeyRangeIter`] that will iterate over `iter`
/// stopping early if `last_key` is `Some` and a key greater than it is
/// encountered.
pub fn new(iter: I, last_key: Option<K>) -> Self {
match last_key {
Some(k) => FilteredKeyRangeIter::Filtered { iter, last_key: k },
None => FilteredKeyRangeIter::Unfiltered { iter },
}
}
}

impl<I: Iterator<Item = T>, T: KeyValuePair, K: KeyType> Iterator for FilteredKeyRangeIter<I, K> {
type Item = Result<(T::Key, T::Value), T::Error>;

fn next(&mut self) -> Option<Self::Item> {
match self {
FilteredKeyRangeIter::Unfiltered { iter } => iter.next().map(T::try_into_tuple),
FilteredKeyRangeIter::Filtered { iter, last_key } => {
match iter.next().map(T::try_into_tuple) {
Some(Ok((key, value))) if key.as_ref() <= last_key.as_ref() => {
Some(Ok((key, value)))
}
Some(Err(e)) => Some(Err(e)),
_ => {
*self = FilteredKeyRangeIter::Exhausted;
None
}
}
}
FilteredKeyRangeIter::Exhausted => None,
}
}

fn size_hint(&self) -> (usize, Option<usize>) {
match self {
FilteredKeyRangeIter::Unfiltered { iter } => iter.size_hint(),
FilteredKeyRangeIter::Filtered { iter, .. } => {
let (_, upper) = iter.size_hint();
(0, upper)
}
FilteredKeyRangeIter::Exhausted => (0, Some(0)),
}
}
}

#[cfg(test)]
#[expect(clippy::indexing_slicing, clippy::unwrap_used)]
mod tests {
Expand Down
69 changes: 69 additions & 0 deletions firewood/src/iter/filtered_key_range.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright (C) 2025, Ava Labs, Inc. All rights reserved.
// See the file LICENSE.md for licensing terms.

use crate::v2::api::{KeyType, KeyValuePair};

pub trait FilteredKeyRangeExt: Iterator<Item: KeyValuePair> + Sized {
/// Returns a new iterator that will emit key-value pairs up to and
/// including `last_key`.
fn stop_after_key<K: KeyType>(self, last_key: Option<K>) -> FilteredKeyRangeIter<Self, K> {
FilteredKeyRangeIter::new(self, last_key)
}
}

impl<I: Iterator<Item: KeyValuePair>> FilteredKeyRangeExt for I {}

/// An iterator over key-value pairs that stops after a specified final key.
#[derive(Debug)]
#[must_use = "iterators are lazy and do nothing unless consumed"]
pub enum FilteredKeyRangeIter<I, K> {
Unfiltered { iter: I },
Filtered { iter: I, last_key: K },
Exhausted,
}

impl<I: Iterator<Item = T>, T: KeyValuePair, K: KeyType> FilteredKeyRangeIter<I, K> {
/// Creates a new [`FilteredKeyRangeIter`] that will iterate over `iter`
/// stopping early if `last_key` is `Some` and a key greater than it is
/// encountered.
pub fn new(iter: I, last_key: Option<K>) -> Self {
match last_key {
Some(k) => FilteredKeyRangeIter::Filtered { iter, last_key: k },
None => FilteredKeyRangeIter::Unfiltered { iter },
}
}
}

impl<I: Iterator<Item = T>, T: KeyValuePair, K: KeyType> Iterator for FilteredKeyRangeIter<I, K> {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also impl FusedIterator since this code will also fuse the underlying iterator.

Probably should document that it will fuse it as well.

type Item = Result<(T::Key, T::Value), T::Error>;

fn next(&mut self) -> Option<Self::Item> {
match self {
FilteredKeyRangeIter::Unfiltered { iter } => iter.next().map(T::try_into_tuple),
FilteredKeyRangeIter::Filtered { iter, last_key } => {
match iter.next().map(T::try_into_tuple) {
Some(Ok((key, value))) if key.as_ref() <= last_key.as_ref() => {
Some(Ok((key, value)))
}
Some(Err(e)) => Some(Err(e)),
_ => {
*self = FilteredKeyRangeIter::Exhausted;
None
}
}
}
FilteredKeyRangeIter::Exhausted => None,
}
}

fn size_hint(&self) -> (usize, Option<usize>) {
match self {
FilteredKeyRangeIter::Unfiltered { iter } => iter.size_hint(),
FilteredKeyRangeIter::Filtered { iter, .. } => {
let (_, upper) = iter.size_hint();
(0, upper)
}
FilteredKeyRangeIter::Exhausted => (0, Some(0)),
}
}
}
68 changes: 68 additions & 0 deletions firewood/src/iter/returnable.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Copyright (C) 2025, Ava Labs, Inc. All rights reserved.
// See the file LICENSE.md for licensing terms.

pub(crate) trait ReturnableIteratorExt: Iterator + Sized {
/// Wraps this iterator in a [`ReturnableIterator`].
fn returnable(self) -> ReturnableIterator<Self> {
ReturnableIterator::new(self)
}
}

impl<I: Iterator> ReturnableIteratorExt for I {}

/// Similar to a peekable iterator. In addition to being able to peek at the
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I'm following what this does that Peekable's blanket implementation does not do., except it's a little less flexible. Peekable keeps a peeked: Option<Option<I::Item>> inside it, but also supports stuff like peek_mut which can be used to implement return_item.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also supports stuff like peek_mut which can be used to implement return_item.

peek_mut does not do what I need. If it returned &mut Option<I::Item>, that would be sufficient. However, it returns Option<&mut I::Item> which is not the same thing.

Peekable provides no way for us to push the item back to the front of the iterator.

This is similar to using VecDeque to push_front but with a slot for only one element.

/// next item without consuming it, it also allows "returning" an item back to
/// the iterator to be yielded on the next call to [`next()`].
///
/// [`next()`]: Iterator::next
pub(crate) struct ReturnableIterator<I: Iterator> {
iter: I,
next: Option<I::Item>,
}

impl<I: Iterator> ReturnableIterator<I> {
pub(crate) const fn new(iter: I) -> Self {
Self { iter, next: None }
}

/// Peeks at the next item without consuming it. The next call to
/// [`next()`] will still return this item.
///
/// [`next()`]: Iterator::next
pub(crate) fn peek(&mut self) -> Option<&mut I::Item> {
if self.next.is_none() {
self.next = self.iter.next();
}

self.next.as_mut()
}

/// Puts an item back to be returned on the next call to [`next()`]. This
/// makes it easy to "un-read" a single item from the iterator without
/// needing to implement complex buffering logic.
///
/// NOTE: This will replace and return any item that was already in the
/// return slot.
///
/// [`next()`]: Iterator::next
pub(crate) const fn return_item(&mut self, head: I::Item) -> Option<I::Item> {
self.next.replace(head)
}
}

impl<I: Iterator> Iterator for ReturnableIterator<I> {
type Item = I::Item;

fn next(&mut self) -> Option<Self::Item> {
self.next.take().or_else(|| self.iter.next())
}

fn size_hint(&self) -> (usize, Option<usize>) {
let (lower, upper) = self.iter.size_hint();
let head_count = usize::from(self.next.is_some());
(
lower.saturating_add(head_count),
upper.and_then(|u| u.checked_add(head_count)),
)
}
}
57 changes: 13 additions & 44 deletions firewood/src/merkle/merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ use firewood_storage::{FileIoError, TrieReader};

use crate::{
db::BatchOp,
iter::{FilteredKeyRangeIter, MerkleKeyValueIter},
iter::{
FilteredKeyRangeExt, FilteredKeyRangeIter, MerkleKeyValueIter,
returnable::{ReturnableIterator, ReturnableIteratorExt},
},
merkle::Key,
v2::api::{BatchIter, KeyType, KeyValuePair},
};
Expand Down Expand Up @@ -39,12 +42,12 @@ where
last_key: Option<K>,
kvp_iter: impl IntoIterator<IntoIter = I>,
) -> Self {
let base_iter = merkle
.key_value_iter_from_key(first_key.as_ref().map(AsRef::as_ref).unwrap_or_default())
.stop_after_key(last_key);
Self {
trie: ReturnableIterator::new(base_iter),
kvp: ReturnableIterator::new(FilteredKeyRangeIter::new(kvp_iter.into_iter(), None)),
trie: merkle
.key_value_iter_from_key(first_key.as_ref().map(AsRef::as_ref).unwrap_or_default())
.stop_after_key(last_key)
.returnable(),
kvp: kvp_iter.into_iter().stop_after_key(None).returnable(),
}
}
}
Expand All @@ -68,14 +71,14 @@ where

(Some(Err(err)), kvp) => {
if let Some(kvp) = kvp {
self.kvp.set_next(kvp);
self.kvp.return_item(kvp);
}

Some(Err(err))
}
(trie, Some(Err(err))) => {
if let Some(trie) = trie {
self.trie.set_next(trie);
self.trie.return_item(trie);
}

Some(Err(err.into()))
Expand All @@ -96,7 +99,7 @@ where
match <[u8] as Ord>::cmp(&base_key, kvp_key.as_ref()) {
std::cmp::Ordering::Less => {
// retain the kvp iterator's current item.
self.kvp.set_next(Ok((kvp_key, kvp_value)));
self.kvp.return_item(Ok((kvp_key, kvp_value)));

// trie key is less than next kvp key, so it must be deleted.
Some(Ok(BatchOp::Delete {
Expand All @@ -119,7 +122,7 @@ where
}
std::cmp::Ordering::Greater => {
// retain the trie iterator's current item.
self.trie.set_next(Ok((base_key, node_value)));
self.trie.return_item(Ok((base_key, node_value)));
// trie key is greater than next kvp key, so we need to insert it.
Some(Ok(BatchOp::Put {
key: EitherKey::Right(kvp_key),
Expand All @@ -133,40 +136,6 @@ where
}
}

/// Similar to a peekable iterator. Instead of peeking at the next item, it allows
/// you to put it back to be returned on the next call to `next()`.
struct ReturnableIterator<I: Iterator> {
iter: I,
next: Option<I::Item>,
}

impl<I: Iterator> ReturnableIterator<I> {
const fn new(iter: I) -> Self {
Self { iter, next: None }
}

const fn set_next(&mut self, head: I::Item) -> Option<I::Item> {
self.next.replace(head)
}
}

impl<I: Iterator> Iterator for ReturnableIterator<I> {
type Item = I::Item;

fn next(&mut self) -> Option<Self::Item> {
self.next.take().or_else(|| self.iter.next())
}

fn size_hint(&self) -> (usize, Option<usize>) {
let (lower, upper) = self.iter.size_hint();
let head_count = usize::from(self.next.is_some());
(
lower.saturating_add(head_count),
upper.and_then(|u| u.checked_add(head_count)),
)
}
}

#[derive(Debug)]
pub(super) enum EitherKey<L, R> {
Left(L),
Expand Down
8 changes: 5 additions & 3 deletions firewood/src/merkle/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ mod merge;
/// Parallel merkle
pub mod parallel;

use crate::iter::{MerkleKeyValueIter, PathIterator};
use crate::iter::FilteredKeyRangeExt;
use crate::iter::{MerkleKeyValueIter, PathIterator, returnable::ReturnableIteratorExt};
use crate::v2::api::{
self, BatchIter, FrozenProof, FrozenRangeProof, KeyType, KeyValuePair, ValueType,
};
Expand Down Expand Up @@ -406,7 +407,8 @@ impl<T: TrieReader> Merkle<T> {

let mut iter = self
.key_value_iter_from_key(start_key.unwrap_or_default())
.stop_after_key(end_key);
.stop_after_key(end_key)
.returnable();

// don't consume the iterator so we can determine if we hit the
// limit or exhausted the iterator later
Expand All @@ -422,7 +424,7 @@ impl<T: TrieReader> Merkle<T> {

let end_proof = if let Some(limit) = limit
&& limit.get() <= key_values.len()
&& iter.next().is_some()
&& iter.peek().is_some()
{
// limit was provided, we hit it, and there is at least one more key
// end proof is for the last key provided
Expand Down