feat: Change proof serialization/deserialization (#1638)

## Why this should be merged

Second PR for change proof FFI (builds on
#1637). This PR includes
`fwd_change_proof_to_bytes` and `fwd_change_proof_from_bytes` for
serialization/deserization of a change proof.

## How this works
Mostly follows the serialization/deserialization implementation in range
proof. Main change is to support serializing/deserializing `BatchOp`s
which are used for storing the difference between two revisions.

## How this was tested

Basic serialization and deserialization tests in `proofs_test.go`,
including a round trip test where a change proof is serialized,
deserialized, and serialized again, and verifying the two serialized
proofs match.

Author: bernard-avalabs

ffi/firewood.h

@@ -92,6 +92,25 @@ func newSerializedRangeProof(
 	return proofBytes
 }
 
+func newSerializedChangeProof(
+	t *testing.T,
+	db *Database,
+	startRoot, endRoot Hash,
+	startKey, endKey maybe,
+	proofLen uint32,
+) []byte {
+	r := require.New(t)
+
+	// TODO: Replace with newVerifiedChangeProof after adding verify.
+	proof, err := db.ChangeProof(startRoot, endRoot, startKey, endKey, proofLen)
+	r.NoError(err)
+
+	proofBytes, err := proof.MarshalBinary()
+	r.NoError(err)
+
+	return proofBytes
+}
+
 func TestRangeProofEmptyDB(t *testing.T) {
 	r := require.New(t)
 	db := newTestDatabase(t)
@@ -412,3 +431,59 @@ func TestChangeProofCreation(t *testing.T) {
 	_, err = db.ChangeProof(root1, root2, nothing(), nothing(), changeProofLenUnbounded)
 	r.NoError(err)
 }
+
+func TestChangeProofDiffersAfterUpdate(t *testing.T) {
+	r := require.New(t)
+	db := newTestDatabase(t)
+
+	// Insert first half of data in the first batch
+	_, _, batch := kvForTest(10000)
+	root1, err := db.Update(batch[:2500])
+	r.NoError(err)
+
+	// Insert the rest in the second batch
+	root2, err := db.Update(batch[2500:5000])
+	r.NoError(err)
+	r.NotEqual(root1, root2)
+
+	// get a proof
+	proof1 := newSerializedChangeProof(t, db, root1, root2, nothing(), nothing(), changeProofLenUnbounded)
+	r.NoError(err)
+
+	// insert more data
+	root3, err := db.Update(batch[5000:])
+	r.NoError(err)
+	r.NotEqual(root2, root3)
+
+	// get a proof again
+	proof2 := newSerializedChangeProof(t, db, root2, root3, nothing(), nothing(), changeProofLenUnbounded)
+	// ensure the proofs are different
+	r.NotEqual(proof1, proof2)
+}
+
+func TestRoundTripChangeProofSerialization(t *testing.T) {
+	r := require.New(t)
+	db := newTestDatabase(t)
+
+	// Insert some data.
+	_, _, batch := kvForTest(10)
+	root1, err := db.Update(batch[:5])
+	r.NoError(err)
+
+	root2, err := db.Update(batch[5:])
+	r.NoError(err)
+
+	// get a proof
+	proofBytes := newSerializedChangeProof(t, db, root1, root2, nothing(), nothing(), changeProofLenUnbounded)
+
+	// Deserialize the proof.
+	proof := new(ChangeProof)
+	err = proof.UnmarshalBinary(proofBytes)
+	r.NoError(err)
+	t.Cleanup(func() { r.NoError(proof.Free()) })
+
+	// serialize the proof again
+	serialized, err := proof.MarshalBinary()
+	r.NoError(err)
+	r.Equal(proofBytes, serialized)
+}

ffi/proofs_test.go

@@ -3,14 +3,15 @@
 
 use std::num::NonZeroUsize;
 
-#[cfg(feature = "ethhash")]
-use firewood::ProofError;
 #[cfg(feature = "ethhash")]
 use firewood_storage::TrieHash;
 #[cfg(feature = "ethhash")]
 use rlp::Rlp;
 
-use firewood::v2::api::{self, FrozenChangeProof};
+use firewood::{
+    ProofError,
+    v2::api::{self, FrozenChangeProof},
+};
 
 use crate::{
     BorrowedBytes, CResult, ChangeProofResult, DatabaseHandle, HashKey, HashResult, Maybe,
@@ -366,8 +367,12 @@ pub extern "C" fn fwd_change_proof_find_next_key(
 ///
 /// The other [`ValueResult`] variants are not used.
 #[unsafe(no_mangle)]
-pub extern "C" fn fwd_change_proof_to_bytes(_proof: Option<&ChangeProofContext>) -> ValueResult {
-    CResult::from_err("not yet implemented")
+pub extern "C" fn fwd_change_proof_to_bytes(proof: Option<&ChangeProofContext>) -> ValueResult {
+    crate::invoke_with_handle(proof, |ctx| {
+        let mut vec = Vec::new();
+        ctx.proof.write_to_vec(&mut vec);
+        vec
+    })
 }
 
 /// Deserialize a `ChangeProof` from bytes.
@@ -384,8 +389,11 @@ pub extern "C" fn fwd_change_proof_to_bytes(_proof: Option<&ChangeProofContext>)
 ///   well-formed. The verify method must be called to ensure the proof is cryptographically valid.
 /// - [`ChangeProofResult::Err`] containing an error message if the proof could not be parsed.
 #[unsafe(no_mangle)]
-pub extern "C" fn fwd_change_proof_from_bytes(_bytes: BorrowedBytes) -> ChangeProofResult {
-    CResult::from_err("not yet implemented")
+pub extern "C" fn fwd_change_proof_from_bytes(bytes: BorrowedBytes) -> ChangeProofResult {
+    crate::invoke(move || {
+        FrozenChangeProof::from_slice(&bytes)
+            .map_err(|err| api::Error::ProofError(ProofError::Deserialization(err)))
+    })
 }
 
 /// Frees the memory associated with a `ChangeProofContext`.

ffi/src/proofs/change.rs

@@ -18,7 +18,12 @@ use super::{
     reader::{ProofReader, ReadError, ReadItem, V0Reader, Version0},
     types::{Proof, ProofNode, ProofType},
 };
-use crate::v2::api::FrozenRangeProof;
+use crate::{
+    db::BatchOp,
+    merkle::{Key, Value},
+    proofs::magic::{BATCH_DELETE, BATCH_DELETE_RANGE, BATCH_PUT},
+    v2::api::{FrozenChangeProof, FrozenRangeProof},
+};
 
 impl FrozenRangeProof {
     /// Parses a `FrozenRangeProof` from the given byte slice.
@@ -72,6 +77,46 @@ impl<T: Version0> Version0 for Box<[T]> {
     }
 }
 
+impl FrozenChangeProof {
+    /// Parses a `FrozenChangeProof` from the given byte slice.
+    ///
+    /// Currently only V0 proofs are supported. See [`FrozenChangeProof::write_to_vec`]
+    /// for the serialization format.
+    ///
+    /// # Errors
+    ///
+    /// Returns a [`ReadError`] if the data is invalid. See the enum variants for
+    /// the possible reasons.
+    pub fn from_slice(data: &[u8]) -> Result<Self, ReadError> {
+        let mut reader = ProofReader::new(data);
+
+        let header = reader.read_item::<Header>()?;
+        header
+            .validate(Some(ProofType::Change))
+            .map_err(ReadError::InvalidHeader)?;
+
+        if header.version != 0 {
+            return Err(ReadError::InvalidHeader(
+                InvalidHeader::UnsupportedVersion {
+                    found: header.version,
+                },
+            ));
+        }
+
+        let mut reader = V0Reader::new(reader, header);
+        let this = reader.read_v0_item()?;
+        if reader.remainder().is_empty() {
+            Ok(this)
+        } else {
+            Err(reader.invalid_item(
+                "trailing bytes",
+                "no data after the proof",
+                format!("{} bytes", reader.remainder().len()),
+            ))
+        }
+    }
+}
+
 impl Version0 for FrozenRangeProof {
     fn read_v0_item(reader: &mut V0Reader<'_>) -> Result<Self, ReadError> {
         let start_proof = reader.read_v0_item()?;
@@ -86,6 +131,41 @@ impl Version0 for FrozenRangeProof {
     }
 }
 
+impl Version0 for FrozenChangeProof {
+    fn read_v0_item(reader: &mut V0Reader<'_>) -> Result<Self, ReadError> {
+        let start_proof = reader.read_v0_item()?;
+        let end_proof = reader.read_v0_item()?;
+        let key_values = reader.read_v0_item()?;
+
+        Ok(Self::new(
+            Proof::new(start_proof),
+            Proof::new(end_proof),
+            key_values,
+        ))
+    }
+}
+
+impl Version0 for BatchOp<Key, Value> {
+    fn read_v0_item(reader: &mut V0Reader<'_>) -> Result<Self, ReadError> {
+        match reader
+            .read_item::<u8>()
+            .map_err(|err| err.set_item("option discriminant"))?
+        {
+            BATCH_PUT => Ok(BatchOp::Put {
+                key: reader.read_item()?,
+                value: reader.read_item()?,
+            }),
+            BATCH_DELETE => Ok(BatchOp::Delete {
+                key: reader.read_item()?,
+            }),
+            BATCH_DELETE_RANGE => Ok(BatchOp::DeleteRange {
+                prefix: reader.read_item()?,
+            }),
+            found => Err(reader.invalid_item("option discriminant", "0, 1, or 2", found)),
+        }
+    }
+}
+
 impl Version0 for ProofNode {
     fn read_v0_item(reader: &mut V0Reader<'_>) -> Result<Self, ReadError> {
         let key = reader.read_v0_item()?;

firewood/src/proofs/de.rs

@@ -114,4 +114,9 @@ pub(super) mod magic {
 
     /// Branching factor identifier for branch factor 16
     pub const BRANCH_FACTOR: u8 = 16;
+
+    /// `BatchOp` constants for serialization
+    pub const BATCH_PUT: u8 = 0;
+    pub const BATCH_DELETE: u8 = 1;
+    pub const BATCH_DELETE_RANGE: u8 = 2;
 }

firewood/src/proofs/mod.rs

@@ -15,7 +15,12 @@ use super::{
     header::Header,
     types::{ProofNode, ProofType},
 };
-use crate::v2::api::FrozenRangeProof;
+use crate::{
+    db::BatchOp,
+    merkle::{Key, Value},
+    proofs::magic::{BATCH_DELETE, BATCH_DELETE_RANGE, BATCH_PUT},
+    v2::api::{FrozenChangeProof, FrozenRangeProof},
+};
 
 impl FrozenRangeProof {
     /// Serializes this proof into the provided byte vector.
@@ -78,6 +83,13 @@ impl FrozenRangeProof {
     }
 }
 
+impl FrozenChangeProof {
+    pub fn write_to_vec(&self, out: &mut Vec<u8>) {
+        Header::from(ProofType::Change).write_item(out);
+        self.write_item(out);
+    }
+}
+
 trait PushVarInt {
     fn push_var_int<VI: VarInt>(&mut self, v: VI);
 }
@@ -103,6 +115,37 @@ impl WriteItem for FrozenRangeProof {
     }
 }
 
+impl WriteItem for FrozenChangeProof {
+    fn write_item(&self, out: &mut Vec<u8>) {
+        self.start_proof().write_item(out);
+        self.end_proof().write_item(out);
+        self.batch_ops().write_item(out);
+    }
+}
+
+impl WriteItem for [BatchOp<Key, Value>] {
+    fn write_item(&self, out: &mut Vec<u8>) {
+        out.push_var_int(self.len());
+        for item in self {
+            match item {
+                BatchOp::Put { key, value } => {
+                    out.push(BATCH_PUT);
+                    key.write_item(out);
+                    value.write_item(out);
+                }
+                BatchOp::Delete { key } => {
+                    out.push(BATCH_DELETE);
+                    key.write_item(out);
+                }
+                BatchOp::DeleteRange { prefix } => {
+                    out.push(BATCH_DELETE_RANGE);
+                    prefix.write_item(out);
+                }
+            }
+        }
+    }
+}
+
 impl WriteItem for ProofNode {
     fn write_item(&self, out: &mut Vec<u8>) {
         self.key.write_item(out);