Store a bitset of transmissions in compact subdag.

New types:
- CompactCertificate
- CompactHeader
- Subdag enum can hold either
- NarwhalCertificate trait provides an interface
- We use BitSet, which used to be part of standard library

Co-authored-by ljedrz <ljedrz@users.noreply.github.com>

Author: vicsn

.circleci/config.yml

@@ -484,6 +484,20 @@ jobs:
       - run_test:
           workspace_member: snarkvm-ledger-narwhal-batch-header
 
+  ledger-narwhal-compact-certificate:
+    executor: rust-docker
+    resource_class: << pipeline.parameters.twoxlarge >>
+    steps:
+      - run_test:
+          workspace_member: snarkvm-ledger-narwhal-compact-certificate
+
+  ledger-narwhal-compact-header:
+    executor: rust-docker
+    resource_class: << pipeline.parameters.twoxlarge >>
+    steps:
+      - run_test:
+          workspace_member: snarkvm-ledger-narwhal-compact-header
+
   ledger-narwhal-data:
     executor: rust-docker
     resource_class: << pipeline.parameters.small >>
@@ -891,6 +905,8 @@ workflows:
       - ledger-narwhal
       - ledger-narwhal-batch-certificate
       - ledger-narwhal-batch-header
+      - ledger-narwhal-compact-certificate
+      - ledger-narwhal-compact-header
       - ledger-narwhal-data
       - ledger-narwhal-subdag
       - ledger-narwhal-transmission

Cargo.toml

@@ -68,6 +68,8 @@ members = [
   "ledger/narwhal",
   "ledger/narwhal/batch-certificate",
   "ledger/narwhal/batch-header",
+  "ledger/narwhal/compact-certificate",
+  "ledger/narwhal/compact-header",
   "ledger/narwhal/data",
   "ledger/narwhal/subdag",
   "ledger/narwhal/transmission",
@@ -358,6 +360,18 @@ version = "=4.0.0"
 path = "ledger/narwhal/batch-certificate"
 version = "=4.0.0"
 
+[workspace.dependencies.snarkvm-ledger-narwhal-compact-certificate]
+path = "ledger/narwhal/compact-certificate"
+version = "=4.0.0"
+
+[workspace.dependencies.snarkvm-ledger-narwhal-compact-header]
+path = "ledger/narwhal/compact-header"
+version = "=4.0.0"
+
+[workspace.dependencies.snarkvm-ledger-narwhal-traits]
+path = "ledger/narwhal/traits"
+version = "=4.0.0"
+
 [workspace.dependencies.snarkvm-ledger-narwhal-subdag]
 path = "ledger/narwhal/subdag"
 version = "=4.0.0"

ledger/block/src/bytes.rs

@@ -54,9 +54,29 @@ impl<N: Network> FromBytes for Block<N> {
             aborted_solution_ids.push(FromBytes::read_le(&mut reader)?);
         }
 
+        // Read the number of prior solution IDs.
+        let num_prior_solutions = u16::read_le(&mut reader)?;
+        // Read the aborted transaction IDs.
+        let mut prior_solution_ids = Vec::with_capacity(num_prior_solutions as usize);
+        for _ in 0..num_prior_solutions {
+            prior_solution_ids.push(FromBytes::read_le(&mut reader)?);
+        }
+
         // Read the transactions.
         let transactions = FromBytes::read_le(&mut reader)?;
 
+        // Read the number of prior transaction IDs.
+        let num_prior_transactions = u32::read_le(&mut reader)?;
+        // Ensure the number of prior transaction IDs is within bounds (this is an early safety check).
+        if num_prior_transactions as usize > Transactions::<N>::MAX_TRANSACTIONS {
+            return Err(error("Invalid number of aborted transaction IDs in the block"));
+        }
+        // Read the aborted transaction IDs.
+        let mut prior_transaction_ids = Vec::with_capacity(num_prior_transactions as usize);
+        for _ in 0..num_prior_transactions {
+            prior_transaction_ids.push(FromBytes::read_le(&mut reader)?);
+        }
+
         // Read the number of aborted transaction IDs.
         let num_aborted_transactions = u32::read_le(&mut reader)?;
         // Ensure the number of aborted transaction IDs is within bounds (this is an early safety check).
@@ -76,8 +96,10 @@ impl<N: Network> FromBytes for Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_ids,
             aborted_transaction_ids,
         )
         .map_err(error)?;
@@ -113,13 +135,25 @@ impl<N: Network> ToBytes for Block<N> {
         // Write the solutions.
         self.solutions.write_le(&mut writer)?;
 
+        // Write the prior solution ids.
+        (u16::try_from(self.prior_solution_ids.len()).map_err(error))?.write_le(&mut writer)?;
+        self.prior_solution_ids.write_le(&mut writer)?;
+
         // Write the aborted solution IDs.
         (u32::try_from(self.aborted_solution_ids.len()).map_err(error))?.write_le(&mut writer)?;
         self.aborted_solution_ids.write_le(&mut writer)?;
 
+        // Write the prior solution ids.
+        (u16::try_from(self.prior_solution_ids.len()).map_err(error))?.write_le(&mut writer)?;
+        self.prior_solution_ids.write_le(&mut writer)?;
+
         // Write the transactions.
         self.transactions.write_le(&mut writer)?;
 
+        // Write the prior transaction ids.
+        (u32::try_from(self.prior_transaction_ids.len()).map_err(error))?.write_le(&mut writer)?;
+        self.prior_transaction_ids.write_le(&mut writer)?;
+
         // Write the aborted transaction IDs.
         (u32::try_from(self.aborted_transaction_ids.len()).map_err(error))?.write_le(&mut writer)?;
         self.aborted_transaction_ids.write_le(&mut writer)

ledger/block/src/lib.rs

@@ -77,10 +77,14 @@ pub struct Block<N: Network> {
     ratifications: Ratifications<N>,
     /// The solutions in the block.
     solutions: Solutions<N>,
+    /// The prior solution ids in the block.
+    prior_solution_ids: Vec<SolutionID<N>>,
     /// The aborted solution IDs in this block.
     aborted_solution_ids: Vec<SolutionID<N>>,
     /// The transactions in this block.
     transactions: Transactions<N>,
+    /// The prior transaction ids in the block.
+    prior_transaction_ids: Vec<N::TransactionID>,
     /// The aborted transaction IDs in this block.
     aborted_transaction_ids: Vec<N::TransactionID>,
 }
@@ -94,8 +98,10 @@ impl<N: Network> Block<N> {
         header: Header<N>,
         ratifications: Ratifications<N>,
         solutions: Solutions<N>,
+        prior_solution_ids: Vec<SolutionID<N>>,
         aborted_solution_ids: Vec<SolutionID<N>>,
         transactions: Transactions<N>,
+        prior_transaction_ids: Vec<N::TransactionID>,
         aborted_transaction_ids: Vec<N::TransactionID>,
         rng: &mut R,
     ) -> Result<Self> {
@@ -110,8 +116,10 @@ impl<N: Network> Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_ids,
             aborted_transaction_ids,
         )
     }
@@ -124,8 +132,10 @@ impl<N: Network> Block<N> {
         subdag: Subdag<N>,
         ratifications: Ratifications<N>,
         solutions: Solutions<N>,
+        prior_solution_ids: Vec<SolutionID<N>>,
         aborted_solution_ids: Vec<SolutionID<N>>,
         transactions: Transactions<N>,
+        prior_transaction_ids: Vec<N::TransactionID>,
         aborted_transaction_ids: Vec<N::TransactionID>,
     ) -> Result<Self> {
         // Construct the beacon authority.
@@ -137,8 +147,10 @@ impl<N: Network> Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_ids,
             aborted_transaction_ids,
         )
     }
@@ -151,8 +163,10 @@ impl<N: Network> Block<N> {
         authority: Authority<N>,
         ratifications: Ratifications<N>,
         solutions: Solutions<N>,
+        prior_solution_ids: Vec<SolutionID<N>>,
         aborted_solution_ids: Vec<SolutionID<N>>,
         transactions: Transactions<N>,
+        prior_transaction_ids: Vec<N::TransactionID>,
         aborted_transaction_ids: Vec<N::TransactionID>,
     ) -> Result<Self> {
         // Ensure the number of aborted solutions IDs is within the allowed range.
@@ -202,8 +216,10 @@ impl<N: Network> Block<N> {
                 Self::check_subdag_transmissions(
                     subdag,
                     &solutions,
+                    &prior_solution_ids,
                     &aborted_solution_ids,
                     &transactions,
+                    &prior_transaction_ids,
                     &aborted_transaction_ids,
                 )?;
             }
@@ -231,8 +247,10 @@ impl<N: Network> Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_ids,
             aborted_transaction_ids,
         )
     }
@@ -247,8 +265,10 @@ impl<N: Network> Block<N> {
         authority: Authority<N>,
         ratifications: Ratifications<N>,
         solutions: Solutions<N>,
+        prior_solution_ids: Vec<SolutionID<N>>,
         aborted_solution_ids: Vec<SolutionID<N>>,
         transactions: Transactions<N>,
+        prior_transaction_ids: Vec<N::TransactionID>,
         aborted_transaction_ids: Vec<N::TransactionID>,
     ) -> Result<Self> {
         // Return the block.
@@ -259,11 +279,81 @@ impl<N: Network> Block<N> {
             authority,
             ratifications,
             solutions,
+            prior_solution_ids,
             aborted_solution_ids,
             transactions,
+            prior_transaction_ids,
             aborted_transaction_ids,
         })
     }
+
+    /// Consume the Block and return a Subdag with full batch certificates.
+    pub fn into_full_subdag(self) -> Result<Subdag<N>> {
+        let Block {
+            ratifications,
+            solutions,
+            prior_solution_ids,
+            transactions,
+            prior_transaction_ids,
+            aborted_transaction_ids,
+            authority,
+            ..
+        } = self;
+
+        // Check if Authority is a Quorum
+        let Authority::Quorum(subdag) = authority else {
+            bail!("Can only convert block with Quorum Authority to full subdag");
+        };
+
+        // Collect ratification IDs.
+        let ratification_ids = ratifications.ratification_ids().copied().collect_vec();
+        // Collect transaction IDs.
+        let transaction_ids = transactions.transaction_ids().copied().collect_vec();
+
+        // Convert Quorum authority to subdag with full batch certificates.
+        subdag.into_full(
+            ratification_ids,
+            solutions.as_puzzle_solutions().cloned(),
+            prior_solution_ids,
+            transaction_ids,
+            prior_transaction_ids,
+            aborted_transaction_ids,
+        )
+    }
+
+    /// Borrow the Block and return a Subdag with full batch certificates.
+    pub fn to_full_subdag(&self) -> Result<Subdag<N>> {
+        let Block {
+            ratifications,
+            solutions,
+            prior_solution_ids,
+            transactions,
+            prior_transaction_ids,
+            aborted_transaction_ids,
+            authority,
+            ..
+        } = self;
+
+        // Check if Authority is a Quorum
+        let Authority::Quorum(subdag) = authority else {
+            bail!("Can only convert block with Quorum Authority to full subdag");
+        };
+
+        // Collect ratification IDs.
+        let ratification_ids = ratifications.ratification_ids().copied().collect_vec();
+        // Collect transaction IDs.
+        let transaction_ids = transactions.transaction_ids().copied().collect_vec();
+
+        // Convert Quorum authority to subdag with full batch certificates.
+        subdag.clone().into_full(
+            ratification_ids,
+            solutions.as_puzzle_solutions().cloned(),
+            prior_solution_ids.clone(),
+            transaction_ids,
+            prior_transaction_ids.clone(),
+            aborted_transaction_ids.clone(),
+        )
+    }
 }
 
 impl<N: Network> Block<N> {
@@ -297,11 +387,21 @@ impl<N: Network> Block<N> {
         &self.aborted_solution_ids
     }
 
+    /// Returns the prior solution ids in the block.
+    pub const fn prior_solution_ids(&self) -> &Vec<SolutionID<N>> {
+        &self.prior_solution_ids
+    }
+
     /// Returns the transactions in this block.
     pub const fn transactions(&self) -> &Transactions<N> {
         &self.transactions
     }
 
+    /// Returns the prior transactions ids in the block.
+    pub const fn prior_transaction_ids(&self) -> &Vec<N::TransactionID> {
+        &self.prior_transaction_ids
+    }
+
     /// Returns the aborted transaction IDs in this block.
     pub const fn aborted_transaction_ids(&self) -> &Vec<N::TransactionID> {
         &self.aborted_transaction_ids
@@ -709,8 +809,10 @@ pub mod test_helpers {
             ratifications,
             None.into(),
             vec![],
+            vec![],
             transactions,
             vec![],
+            vec![],
             rng,
         )
         .unwrap();

ledger/block/src/serialize.rs

@@ -27,8 +27,10 @@ impl<N: Network> Serialize for Block<N> {
                 block.serialize_field("authority", &self.authority)?;
                 block.serialize_field("ratifications", &self.ratifications)?;
                 block.serialize_field("solutions", &self.solutions)?;
+                block.serialize_field("prior_solution_ids", &self.prior_solution_ids)?;
                 block.serialize_field("aborted_solution_ids", &self.aborted_solution_ids)?;
                 block.serialize_field("transactions", &self.transactions)?;
+                block.serialize_field("prior_transaction_ids", &self.prior_transaction_ids)?;
                 block.serialize_field("aborted_transaction_ids", &self.aborted_transaction_ids)?;
                 block.end()
             }
@@ -52,8 +54,10 @@ impl<'de, N: Network> Deserialize<'de> for Block<N> {
                     DeserializeExt::take_from_value::<D>(&mut block, "authority")?,
                     DeserializeExt::take_from_value::<D>(&mut block, "ratifications")?,
                     DeserializeExt::take_from_value::<D>(&mut block, "solutions")?,
+                    DeserializeExt::take_from_value::<D>(&mut block, "prior_solution_ids")?,
                     DeserializeExt::take_from_value::<D>(&mut block, "aborted_solution_ids")?,
                     DeserializeExt::take_from_value::<D>(&mut block, "transactions")?,
+                    DeserializeExt::take_from_value::<D>(&mut block, "prior_transaction_ids")?,
                     DeserializeExt::take_from_value::<D>(&mut block, "aborted_transaction_ids")?,
                 )
                 .map_err(de::Error::custom)?;

ledger/block/src/solutions/mod.rs

@@ -65,6 +65,10 @@ impl<N: Network> Solutions<N> {
             None => 0,
         }
     }
+
+    pub fn as_puzzle_solutions(&self) -> Option<&PuzzleSolutions<N>> {
+        self.solutions.as_ref()
+    }
 }
 
 impl<N: Network> Solutions<N> {

ledger/block/src/transactions/mod.rs

@@ -221,6 +221,14 @@ impl<N: Network> Transactions<N> {
         self.transactions.keys()
     }
 
+    /// Returns the unconfirmed transaction IDs, for all transactions in `self`.
+    pub fn unconfirmed_transaction_ids(&self) -> Result<Vec<N::TransactionID>> {
+        self.transactions
+            .values()
+            .map(|confirmed_tx| confirmed_tx.to_unconfirmed_transaction_id())
+            .collect::<Result<_>>()
+    }
+
     /// Returns an iterator over all transactions in `self` that are accepted deploy transactions.
     pub fn deployments(&self) -> impl '_ + Iterator<Item = &ConfirmedTransaction<N>> {
         self.iter().filter(|tx| tx.is_accepted() && tx.is_deploy())