Issues/1310 shrink snapshot file (#1648)

### What

Add a compact format for `LedgerSnapshot` to prevent having to write
duplicate `LedgerKey` data to files.

On serialization, the `LedgerKey` is derived from the `LedgerEntry` to
keep the public interface the same.

### Why

Fixes #1310 .

Tested against a large snapshot (82mb). File was reduced to 53mb, or
~35% reduction.

Roundtrip to serialize the large snapshot and write it back takes 60s
with the old way, and 22s for the new way. A good half of this
optimization comes from using `BufReader`.

The memory footprint does increase by the size of the snapshot due to
conversions between the compact and ledgerSnapshot type, and up to 2x
the snapshot if we are reading a snapshot of the old format. I assume
this is due to how serde manages the deserialization retry logic for
enums.

### Known limitations

N/A

---------

Co-authored-by: Leigh <351529+leighmcculloch@users.noreply.github.com>

Author: mootz12

soroban-ledger-snapshot/src/lib.rs

@@ -1,7 +1,8 @@
-use serde_with::serde_as;
+use serde::Deserialize;
+use serde_with::{serde_as, DeserializeAs, SerializeAs};
 use std::{
     fs::{create_dir_all, File},
-    io::{self, Read, Write},
+    io::{self, BufReader, Read, Write},
     path::Path,
     rc::Rc,
 };
@@ -12,6 +13,9 @@ use soroban_env_host::{
     HostError, LedgerInfo,
 };
 
+#[cfg(test)]
+mod tests;
+
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
     #[error("io")]
@@ -35,11 +39,80 @@ pub struct LedgerSnapshot {
     pub min_persistent_entry_ttl: u32,
     pub min_temp_entry_ttl: u32,
     pub max_entry_ttl: u32,
+    #[serde_as(as = "LedgerEntryVec")]
     pub ledger_entries: Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>,
 }
 
+/// Extended ledger entry that includes the live util ledger sequence. Provides a more compact
+/// form of the tuple used in [`LedgerSnapshot::ledger_entries`], to reduce the size of the snapshot
+/// when serialized to JSON.
+#[derive(Debug, Clone, serde::Deserialize)]
+struct LedgerEntryExt {
+    entry: Box<LedgerEntry>,
+    live_until: Option<u32>,
+}
+
+/// Extended ledger entry that includes the live util ledger sequence, and the entry by reference.
+/// Used to reduce memory usage during serialization.
+#[derive(serde::Serialize)]
+struct LedgerEntryExtRef<'a> {
+    entry: &'a Box<LedgerEntry>, // Reference = no clone
+    live_until: Option<u32>,
+}
+
+struct LedgerEntryVec;
+
+impl<'a> SerializeAs<Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>> for LedgerEntryVec {
+    fn serialize_as<S>(
+        source: &Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>,
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        use serde::ser::SerializeSeq;
+        let mut seq = serializer.serialize_seq(Some(source.len()))?;
+        for (_, (entry, live_until)) in source {
+            seq.serialize_element(&LedgerEntryExtRef {
+                entry,
+                live_until: *live_until,
+            })?;
+        }
+        seq.end()
+    }
+}
+
+impl<'de> DeserializeAs<'de, Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>>
+    for LedgerEntryVec
+{
+    fn deserialize_as<D>(
+        deserializer: D,
+    ) -> Result<Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        #[derive(serde::Deserialize)]
+        #[serde(untagged)]
+        enum Format {
+            V2(Vec<LedgerEntryExt>),
+            V1(Vec<(Box<LedgerKey>, (Box<LedgerEntry>, Option<u32>))>),
+        }
+
+        match Format::deserialize(deserializer)? {
+            Format::V2(entries) => Ok(entries
+                .into_iter()
+                .map(|LedgerEntryExt { entry, live_until }| {
+                    let key = Box::new(entry.to_key());
+                    (key, (entry, live_until))
+                })
+                .collect()),
+            Format::V1(entries) => Ok(entries),
+        }
+    }
+}
+
 impl LedgerSnapshot {
-    // Create a ledger snapshot from ledger info and a set of entries.
+    /// Create a [`LedgerSnapshot`] from [`LedgerInfo`] and a set of entries.
     pub fn from<'a>(
         info: LedgerInfo,
         entries: impl IntoIterator<Item = (&'a Box<LedgerKey>, (&'a Box<LedgerEntry>, Option<u32>))>,
@@ -65,7 +138,7 @@ impl LedgerSnapshot {
         self.update_entries(&host.get_stored_entries().unwrap());
     }
 
-    // Get the ledger info in the snapshot.
+    /// Get the ledger info in the snapshot.
     pub fn ledger_info(&self) -> LedgerInfo {
         LedgerInfo {
             protocol_version: self.protocol_version,
@@ -136,22 +209,23 @@ impl LedgerSnapshot {
 }
 
 impl LedgerSnapshot {
-    // Read in a [`LedgerSnapshot`] from a reader.
+    /// Read in a [`LedgerSnapshot`] from a reader.
     pub fn read(r: impl Read) -> Result<LedgerSnapshot, Error> {
         Ok(serde_json::from_reader::<_, LedgerSnapshot>(r)?)
     }
 
-    // Read in a [`LedgerSnapshot`] from a file.
+    /// Read in a [`LedgerSnapshot`] from a file.
     pub fn read_file(p: impl AsRef<Path>) -> Result<LedgerSnapshot, Error> {
-        Self::read(File::open(p)?)
+        let reader = BufReader::new(File::open(p)?);
+        Self::read(reader)
     }
 
-    // Write a [`LedgerSnapshot`] to a writer.
+    /// Write a [`LedgerSnapshot`] to a writer.
     pub fn write(&self, w: impl Write) -> Result<(), Error> {
         Ok(serde_json::to_writer_pretty(w, self)?)
     }
 
-    // Write a [`LedgerSnapshot`] to file.
+    /// Write a [`LedgerSnapshot`] to file.
     pub fn write_file(&self, p: impl AsRef<Path>) -> Result<(), Error> {
         let p = p.as_ref();
         if let Some(dir) = p.parent() {

soroban-ledger-snapshot/src/tests.rs

@@ -0,0 +1,57 @@
+use soroban_env_host::{
+    storage::SnapshotSource,
+    xdr::{LedgerKey, Limits, ReadXdr},
+};
+
+use crate::LedgerSnapshot;
+use std::rc::Rc;
+
+// Tuple of (ledger key in xdr base64, live_until_ledger_seq u32) for the entries in the test snapshots
+const TEST_SNAPSHOT_XDR: [(&str, u32); 4] = [
+    ("AAAABgAAAAEltPzYWa7C+mNIQ4xImzw8EMmLbSG+T9PLMMtolT75dwAAABQAAAAB", 10000000),
+    ("AAAABgAAAAEltPzYWa7C+mNIQ4xImzw8EMmLbSG+T9PLMMtolT75dwAAABAAAAABAAAAAgAAAA8AAAAHQmFsYW5jZQAAAAASAAAAAa3vzlmu5Slo92Bh1JTCUlt1ZZ+kKWpl9JnvKeVkd+SWAAAAAQ==", 10000001),
+    ("AAAACAAAAAA=", 10000002),
+    ("AAAAAAAAAAA7mRE4Dv6Yi6CokA6xz+RPNm99vpRr7QdyQPf2JN8VxQ==", 10000003)
+];
+
+#[test]
+fn test_snapshot_from_v1() {
+    let snapshot = LedgerSnapshot::read_file("./test_data/snapshot_v1.json").unwrap();
+
+    for (xdr_key, live_until_ledger_seq) in TEST_SNAPSHOT_XDR.iter() {
+        let ledger_key = LedgerKey::from_xdr_base64(xdr_key, Limits::none()).unwrap();
+        let entry = snapshot.get(&Rc::new(ledger_key)).unwrap();
+        assert!(entry.is_some());
+        assert_eq!(entry.unwrap().1, Some(live_until_ledger_seq.clone()));
+    }
+
+    let expected_str = std::fs::read_to_string("./test_data/snapshot_v2.json").unwrap();
+    let mut written = Vec::new();
+    snapshot.write(&mut written).unwrap();
+    let written_str = String::from_utf8(written).unwrap();
+
+    let expected_normalized = expected_str.replace("\r\n", "\n");
+    let written_normalized = written_str.replace("\r\n", "\n");
+    assert_eq!(written_normalized, expected_normalized);
+}
+
+#[test]
+fn test_snapshot_roundtrip() {
+    let snapshot = LedgerSnapshot::read_file("./test_data/snapshot_v2.json").unwrap();
+
+    for (xdr_key, live_until_ledger_seq) in TEST_SNAPSHOT_XDR.iter() {
+        let ledger_key = LedgerKey::from_xdr_base64(xdr_key, Limits::none()).unwrap();
+        let entry = snapshot.get(&Rc::new(ledger_key)).unwrap();
+        assert!(entry.is_some());
+        assert_eq!(entry.unwrap().1, Some(live_until_ledger_seq.clone()));
+    }
+
+    let expected_str = std::fs::read_to_string("./test_data/snapshot_v2.json").unwrap();
+    let mut written = Vec::new();
+    snapshot.write(&mut written).unwrap();
+    let written_str = String::from_utf8(written).unwrap();
+
+    let expected_normalized = expected_str.replace("\r\n", "\n");
+    let written_normalized = written_str.replace("\r\n", "\n");
+    assert_eq!(written_normalized, expected_normalized);
+}