Store a bitset of transmissions in compact subdag.

New types:
- CompactCertificate
- CompactHeader
- Subdag enum can hold either
- NarwhalCertificate trait provides an interface
- We use BitSet, which used to be part of standard library

Co-authored-by ljedrz <ljedrz@users.noreply.github.com>

Author: vicsn

.circleci/config.yml

@@ -484,6 +484,20 @@ jobs:
       - run_test:
           workspace_member: snarkvm-ledger-narwhal-batch-header
 
+  ledger-narwhal-compact-certificate:
+    executor: rust-docker
+    resource_class: << pipeline.parameters.twoxlarge >>
+    steps:
+      - run_test:
+          workspace_member: snarkvm-ledger-narwhal-compact-certificate
+
+  ledger-narwhal-compact-header:
+    executor: rust-docker
+    resource_class: << pipeline.parameters.twoxlarge >>
+    steps:
+      - run_test:
+          workspace_member: snarkvm-ledger-narwhal-compact-header
+
   ledger-narwhal-data:
     executor: rust-docker
     resource_class: << pipeline.parameters.small >>
@@ -891,6 +905,8 @@ workflows:
       - ledger-narwhal
       - ledger-narwhal-batch-certificate
       - ledger-narwhal-batch-header
+      - ledger-narwhal-compact-certificate
+      - ledger-narwhal-compact-header
       - ledger-narwhal-data
       - ledger-narwhal-subdag
       - ledger-narwhal-transmission

Cargo.lock

@@ -68,6 +68,8 @@ members = [
   "ledger/narwhal",
   "ledger/narwhal/batch-certificate",
   "ledger/narwhal/batch-header",
+  "ledger/narwhal/compact-certificate",
+  "ledger/narwhal/compact-header",
   "ledger/narwhal/data",
   "ledger/narwhal/subdag",
   "ledger/narwhal/transmission",
@@ -365,6 +367,18 @@ version = "=4.1.0"
 path = "ledger/narwhal/batch-certificate"
 version = "=4.1.0"
 
+[workspace.dependencies.snarkvm-ledger-narwhal-compact-certificate]
+path = "ledger/narwhal/compact-certificate"
+version = "=4.0.0"
+
+[workspace.dependencies.snarkvm-ledger-narwhal-compact-header]
+path = "ledger/narwhal/compact-header"
+version = "=4.0.0"
+
+[workspace.dependencies.snarkvm-ledger-narwhal-traits]
+path = "ledger/narwhal/traits"
+version = "=4.0.0"
+
 [workspace.dependencies.snarkvm-ledger-narwhal-subdag]
 path = "ledger/narwhal/subdag"
 version = "=4.1.0"

Cargo.toml

@@ -50,6 +50,12 @@ features = [ "default" ]
 [dependencies.snarkvm-ledger-narwhal-batch-header]
 workspace = true
 
+[dependencies.snarkvm-ledger-narwhal-batch-certificate]
+workspace = true
+
+[dependencies.snarkvm-ledger-narwhal-compact-certificate]
+workspace = true
+
 [dependencies.snarkvm-ledger-narwhal-data]
 workspace = true
 
@@ -68,6 +74,9 @@ workspace = true
 [dependencies.snarkvm-synthesizer-snark]
 workspace = true
 
+[dependencies.bit-set]
+version = "0.8"
+
 [dependencies.indexmap]
 workspace = true
 features = [ "serde" ]

ledger/block/Cargo.toml

@@ -22,7 +22,7 @@ impl<N: Network> FromBytes for Block<N> {
         // Read the version.
         let version = u8::read_le(&mut reader)?;
         // Ensure the version is valid.
-        if version != 1 {
+        if ![1, 2].contains(&version) {
             return Err(error("Invalid block version"));
         }
 
@@ -42,32 +42,78 @@ impl<N: Network> FromBytes for Block<N> {
         // Read the solutions.
         let solutions: Solutions<N> = FromBytes::read_le(&mut reader)?;
 
-        // Read the number of aborted solution IDs.
-        let num_aborted_solutions = u32::read_le(&mut reader)?;
-        // Ensure the number of aborted solutions IDs is within bounds (this is an early safety check).
-        if num_aborted_solutions as usize > Solutions::<N>::max_aborted_solutions().map_err(error)? {
-            return Err(error("Invalid number of aborted solutions IDs in the block"));
-        }
-        // Read the aborted solution IDs.
-        let mut aborted_solution_ids = Vec::with_capacity(num_aborted_solutions as usize);
-        for _ in 0..num_aborted_solutions {
-            aborted_solution_ids.push(FromBytes::read_le(&mut reader)?);
-        }
+        let (prior_solution_transmission_ids, aborted_solution_transmission_ids, aborted_solution_ids) = if version >= 2
+        {
+            // Read the number of prior solution transmission IDs.
+            let num_prior_solutions = u32::read_le(&mut reader)?; // TODO: what is a sane bound on this value?
+            // Read the prior solution transmission IDs.
+            let mut prior_solution_transmission_ids = Vec::with_capacity(num_prior_solutions as usize);
+            for _ in 0..num_prior_solutions {
+                prior_solution_transmission_ids.push(FromBytes::read_le(&mut reader)?);
+            }
+            // Read the number of aborted solution transmission IDs.
+            let num_aborted_solutions = u32::read_le(&mut reader)?;
+            if num_aborted_solutions as usize > Solutions::<N>::max_aborted_solutions().map_err(error)? {
+                return Err(error("Invalid number of aborted solutions IDs in the block"));
+            }
+            // Read the aborted solution transmission IDs.
+            let mut aborted_solution_transmission_ids = Vec::with_capacity(num_aborted_solutions as usize);
+            for _ in 0..num_aborted_solutions {
+                aborted_solution_transmission_ids.push(FromBytes::read_le(&mut reader)?);
+            }
+            (Some(prior_solution_transmission_ids), Some(aborted_solution_transmission_ids), vec![])
+        } else {
+            // Read the number of aborted solution IDs.
+            let num_aborted_solutions = u32::read_le(&mut reader)?;
+            // Ensure the number of aborted solutions IDs is within bounds (this is an early safety check).
+            if num_aborted_solutions as usize > Solutions::<N>::max_aborted_solutions().map_err(error)? {
+                return Err(error("Invalid number of aborted solutions IDs in the block"));
+            }
+            // Read the aborted solution IDs.
+            let mut aborted_solution_ids = Vec::with_capacity(num_aborted_solutions as usize);
+            for _ in 0..num_aborted_solutions {
+                aborted_solution_ids.push(FromBytes::read_le(&mut reader)?);
+            }
+            (None, None, aborted_solution_ids)
+        };
 
         // Read the transactions.
         let transactions = FromBytes::read_le(&mut reader)?;
 
-        // Read the number of aborted transaction IDs.
-        let num_aborted_transactions = u32::read_le(&mut reader)?;
-        // Ensure the number of aborted transaction IDs is within bounds (this is an early safety check).
-        if num_aborted_transactions as usize > Transactions::<N>::max_aborted_transactions().map_err(error)? {
-            return Err(error("Invalid number of aborted transaction IDs in the block"));
-        }
-        // Read the aborted transaction IDs.
-        let mut aborted_transaction_ids = Vec::with_capacity(num_aborted_transactions as usize);
-        for _ in 0..num_aborted_transactions {
-            aborted_transaction_ids.push(FromBytes::read_le(&mut reader)?);
-        }
+        let (prior_transaction_transmission_ids, aborted_transaction_transmission_ids, aborted_transaction_ids) =
+            if version >= 2 {
+                // Read the number of prior transaction transmission IDs.
+                let num_prior_transactions = u32::read_le(&mut reader)?; // TODO: what is a sane bound on this value?
+                // Read the prior transaction transmission IDs.
+                let mut prior_transaction_transmission_ids = Vec::with_capacity(num_prior_transactions as usize);
+                for _ in 0..num_prior_transactions {
+                    prior_transaction_transmission_ids.push(FromBytes::read_le(&mut reader)?);
+                }
+                // Read the number of aborted transaction transmission IDs.
+                let num_aborted_transactions = u32::read_le(&mut reader)?;
+                if num_aborted_transactions as usize > Transactions::<N>::max_aborted_transactions().map_err(error)? {
+                    return Err(error("Invalid number of aborted transaction IDs in the block"));
+                }
+                // Read the aborted transaction transmission IDs.
+                let mut aborted_transaction_transmission_ids = Vec::with_capacity(num_aborted_transactions as usize);
+                for _ in 0..num_aborted_transactions {
+                    aborted_transaction_transmission_ids.push(FromBytes::read_le(&mut reader)?);
+                }
+                (Some(prior_transaction_transmission_ids), Some(aborted_transaction_transmission_ids), vec![])
+            } else {
+                // Read the number of aborted transaction IDs.
+                let num_aborted_transactions = u32::read_le(&mut reader)?;
+                // Ensure the number of aborted transaction IDs is within bounds (this is an early safety check).
+                if num_aborted_transactions as usize > Transactions::<N>::max_aborted_transactions().map_err(error)? {
+                    return Err(error("Invalid number of aborted transaction IDs in the block"));
+                }
+                // Read the aborted transaction IDs.
+                let mut aborted_transaction_ids = Vec::with_capacity(num_aborted_transactions as usize);
+                for _ in 0..num_aborted_transactions {
+                    aborted_transaction_ids.push(FromBytes::read_le(&mut reader)?);
+                }
+                (None, None, aborted_transaction_ids)
+            };
 
         // Construct the block.
         let block = Self::from(
@@ -76,8 +122,12 @@ impl<N: Network> FromBytes for Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_transmission_ids,
+            aborted_solution_transmission_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_transmission_ids,
+            aborted_transaction_transmission_ids,
             aborted_transaction_ids,
         )
         .map_err(error)?;
@@ -95,7 +145,9 @@ impl<N: Network> ToBytes for Block<N> {
     #[inline]
     fn write_le<W: Write>(&self, mut writer: W) -> IoResult<()> {
         // Write the version.
-        1u8.write_le(&mut writer)?;
+        let version =
+            if N::CONSENSUS_VERSION(self.height()).map_err(error)? >= ConsensusVersion::V10 { 2u8 } else { 1 };
+        version.write_le(&mut writer)?;
 
         // Write the block hash.
         self.block_hash.write_le(&mut writer)?;
@@ -113,16 +165,57 @@ impl<N: Network> ToBytes for Block<N> {
         // Write the solutions.
         self.solutions.write_le(&mut writer)?;
 
-        // Write the aborted solution IDs.
-        (u32::try_from(self.aborted_solution_ids.len()).map_err(error))?.write_le(&mut writer)?;
-        self.aborted_solution_ids.write_le(&mut writer)?;
+        match version {
+            1 => {
+                // Write the aborted solution IDs.
+                (u32::try_from(self.aborted_solution_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                self.aborted_solution_ids.write_le(&mut writer)?;
+            }
+            _ => {
+                // Write the prior solution transmission ids.
+                let prior_solution_transmission_ids = self
+                    .prior_solution_transmission_ids
+                    .as_ref()
+                    .ok_or(error("missing prior_solution_transmission_ids"))?;
+                (u32::try_from(prior_solution_transmission_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                prior_solution_transmission_ids.write_le(&mut writer)?;
+                // Write the aborted solution transmission ids.
+                let aborted_solution_transmission_ids = self
+                    .aborted_solution_transmission_ids
+                    .as_ref()
+                    .ok_or(error("missing aborted_solution_transmission_ids"))?;
+                (u32::try_from(aborted_solution_transmission_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                aborted_solution_transmission_ids.write_le(&mut writer)?;
+            }
+        }
 
         // Write the transactions.
         self.transactions.write_le(&mut writer)?;
 
-        // Write the aborted transaction IDs.
-        (u32::try_from(self.aborted_transaction_ids.len()).map_err(error))?.write_le(&mut writer)?;
-        self.aborted_transaction_ids.write_le(&mut writer)
+        match version {
+            1 => {
+                // Write the aborted transaction ids.
+                (u32::try_from(self.aborted_transaction_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                self.aborted_transaction_ids.write_le(&mut writer)?;
+            }
+            _ => {
+                // Write the prior transaction transmission ids.
+                let prior_transaction_transmission_ids = self
+                    .prior_transaction_transmission_ids
+                    .as_ref()
+                    .ok_or(error("missing prior_transaction_transmission_ids"))?;
+                (u32::try_from(prior_transaction_transmission_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                prior_transaction_transmission_ids.write_le(&mut writer)?;
+                // Write the aborted transaction transmission ids.
+                let aborted_transaction_transmission_ids = self
+                    .aborted_transaction_transmission_ids
+                    .as_ref()
+                    .ok_or(error("missing aborted_transaction_transmission_ids"))?;
+                (u32::try_from(aborted_transaction_transmission_ids.len()).map_err(error))?.write_le(&mut writer)?;
+                aborted_transaction_transmission_ids.write_le(&mut writer)?;
+            }
+        }
+        Ok(())
     }
 }
 
@@ -138,6 +231,7 @@ mod tests {
         let rng = &mut TestRng::default();
 
         for expected in [crate::test_helpers::sample_genesis_block(rng)].into_iter() {
+            // todo: sample compact too for the Block tests
             // Check the byte representation.
             let expected_bytes = expected.to_bytes_le()?;
             assert_eq!(expected, Block::read_le(&expected_bytes[..])?);