chore: refactor TransactionHeader serialization (#1759)

* chore: refactor `TransactionHeader` proto serialization

* docs: add changelog entry

* chore: address PR comments

* docs: add pending TODO

Author: juan518munoz

CHANGELOG.md

@@ -28,6 +28,7 @@
  - Replaced NTX Builder's in-memory state management with SQLite-backed persistence; account states, notes, and transaction effects are now stored in the database and inflight state is purged on startup ([#1662](https://github.com/0xMiden/node/pull/1662)).
 - [BREAKING] Reworked `miden-remote-prover`, removing the `worker`/`proxy` distinction and simplifying to a `worker` with a request queue ([#1688](https://github.com/0xMiden/node/pull/1688)).
 - [BREAKING] Renamed `NoteRoot` protobuf message used in `GetNoteScriptByRoot` gRPC endpoints into `NoteScriptRoot` ([#1722](https://github.com/0xMiden/node/pull/1722)).
+- [BREAKING] Modified `TransactionHeader` serialization to allow converting back into the native type after serialization ([#1759](https://github.com/0xMiden/node/issues/1759)).
 - Removed `chain_tip` requirement from mempool subscription request ([#1771](https://github.com/0xMiden/node/pull/1771)).
 
 ### Fixes

crates/proto/src/domain/note.rs

@@ -5,6 +5,7 @@ use miden_protocol::note::{
     Note,
     NoteAttachment,
     NoteDetails,
+    NoteHeader,
     NoteId,
     NoteInclusionProof,
     NoteMetadata,
@@ -219,6 +220,35 @@ impl TryFrom<proto::note::Note> for Note {
     }
 }
 
+// NOTE HEADER
+// ================================================================================================
+
+impl From<NoteHeader> for proto::note::NoteHeader {
+    fn from(header: NoteHeader) -> Self {
+        Self {
+            note_id: Some((&header.id()).into()),
+            metadata: Some(header.into_metadata().into()),
+        }
+    }
+}
+
+impl TryFrom<proto::note::NoteHeader> for NoteHeader {
+    type Error = ConversionError;
+
+    fn try_from(value: proto::note::NoteHeader) -> Result<Self, Self::Error> {
+        let note_id_word: Word = value
+            .note_id
+            .ok_or_else(|| proto::note::NoteHeader::missing_field(stringify!(note_id)))?
+            .try_into()?;
+        let metadata: NoteMetadata = value
+            .metadata
+            .ok_or_else(|| proto::note::NoteHeader::missing_field(stringify!(metadata)))?
+            .try_into()?;
+
+        Ok(NoteHeader::new(NoteId::from_raw(note_id_word), metadata))
+    }
+}
+
 // NOTE SCRIPT
 // ================================================================================================
 

crates/proto/src/domain/transaction.rs

@@ -1,7 +1,9 @@
 use miden_protocol::Word;
-use miden_protocol::transaction::TransactionId;
+use miden_protocol::note::Nullifier;
+use miden_protocol::transaction::{InputNoteCommitment, TransactionId};
+use miden_protocol::utils::{Deserializable, Serializable};
 
-use crate::errors::ConversionError;
+use crate::errors::{ConversionError, MissingFieldHelper};
 use crate::generated as proto;
 
 // FROM TRANSACTION ID
@@ -56,3 +58,40 @@ impl TryFrom<proto::transaction::TransactionId> for TransactionId {
             .try_into()
     }
 }
+
+// INPUT NOTE COMMITMENT
+// ================================================================================================
+
+impl From<InputNoteCommitment> for proto::transaction::InputNoteCommitment {
+    fn from(value: InputNoteCommitment) -> Self {
+        Self {
+            nullifier: Some(value.nullifier().into()),
+            header: value.header().cloned().map(Into::into),
+        }
+    }
+}
+
+impl TryFrom<proto::transaction::InputNoteCommitment> for InputNoteCommitment {
+    type Error = ConversionError;
+
+    fn try_from(value: proto::transaction::InputNoteCommitment) -> Result<Self, Self::Error> {
+        let nullifier: Nullifier = value
+            .nullifier
+            .ok_or_else(|| {
+                proto::transaction::InputNoteCommitment::missing_field(stringify!(nullifier))
+            })?
+            .try_into()?;
+
+        let header: Option<miden_protocol::note::NoteHeader> =
+            value.header.map(TryInto::try_into).transpose()?;
+
+        // TODO: https://github.com/0xMiden/node/issues/1783
+        // InputNoteCommitment has private fields, so we reconstruct it via
+        // serialization roundtrip using its Serializable/Deserializable impls.
+        let mut bytes = Vec::new();
+        nullifier.write_into(&mut bytes);
+        header.write_into(&mut bytes);
+        InputNoteCommitment::read_from_bytes(&bytes)
+            .map_err(|err| ConversionError::deserialization_error("InputNoteCommitment", err))
+    }
+}

crates/store/src/db/migrations/2025062000000_setup/up.sql

@@ -146,9 +146,10 @@ CREATE TABLE transactions (
     block_num                    INTEGER NOT NULL, -- Block number in which the transaction was included.
     initial_state_commitment     BLOB    NOT NULL, -- State of the account before applying the transaction.
     final_state_commitment       BLOB    NOT NULL, -- State of the account after applying the transaction.
-    nullifiers                   BLOB    NOT NULL, -- Serialized vector with the Nullifier of the input notes.
-    output_notes                 BLOB    NOT NULL, -- Serialized vector with the NoteId of the output notes.
+    input_notes                  BLOB    NOT NULL, -- Serialized Vec<InputNoteCommitment> (nullifier + optional NoteHeader).
+    output_notes                 BLOB    NOT NULL, -- Serialized Vec<NoteHeader> (NoteId + NoteMetadata).
     size_in_bytes                INTEGER NOT NULL, -- Estimated size of the row in bytes, considering the size of the input and output notes.
+    fee                          BLOB    NOT NULL, -- Serialized FungibleAsset representing the fee paid by the transaction.
 
     PRIMARY KEY (transaction_id)
 ) WITHOUT ROWID;

crates/store/src/db/mod.rs

@@ -11,18 +11,19 @@ use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES;
 use miden_node_utils::tracing::OpenTelemetrySpanExt;
 use miden_protocol::Word;
 use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader, StorageMapKey};
-use miden_protocol::asset::{Asset, AssetVaultKey};
+use miden_protocol::asset::{Asset, AssetVaultKey, FungibleAsset};
 use miden_protocol::block::{BlockHeader, BlockNoteIndex, BlockNumber, SignedBlock};
 use miden_protocol::crypto::merkle::SparseMerklePath;
 use miden_protocol::note::{
     NoteDetails,
+    NoteHeader,
     NoteId,
     NoteInclusionProof,
     NoteMetadata,
     NoteScript,
     Nullifier,
 };
-use miden_protocol::transaction::TransactionId;
+use miden_protocol::transaction::{InputNoteCommitment, TransactionId};
 use miden_protocol::utils::{Deserializable, Serializable};
 use tokio::sync::oneshot;
 use tracing::{info, instrument};
@@ -141,27 +142,26 @@ pub struct TransactionRecord {
     pub account_id: AccountId,
     pub initial_state_commitment: Word,
     pub final_state_commitment: Word,
-    pub nullifiers: Vec<Nullifier>, // Store nullifiers for input notes
-    pub output_notes: Vec<NoteId>,  // Store note IDs for output notes
+    pub input_notes: Vec<InputNoteCommitment>,
+    pub output_notes: Vec<NoteHeader>,
+    pub fee: FungibleAsset,
 }
 
 impl TransactionRecord {
-    /// Convert to proto `TransactionRecord`, but requires note sync records for output notes.
-    /// For `sync_transactions` RPC, we need to fetch note sync records separately since we only
-    /// store note IDs in the database.
-    pub fn into_proto_with_note_records(
-        self,
-        note_records: Vec<NoteRecord>,
-    ) -> proto::rpc::TransactionRecord {
-        let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into));
-
+    /// Convert to proto `TransactionRecord`.
+    ///
+    /// The proto `TransactionHeader` is a 1:1 mapping of
+    /// `miden_protocol::transaction::TransactionHeader`.
+    pub fn into_proto(self) -> proto::rpc::TransactionRecord {
         proto::rpc::TransactionRecord {
             header: Some(proto::transaction::TransactionHeader {
+                transaction_id: Some(self.transaction_id.into()),
                 account_id: Some(self.account_id.into()),
                 initial_state_commitment: Some(self.initial_state_commitment.into()),
                 final_state_commitment: Some(self.final_state_commitment.into()),
-                nullifiers: self.nullifiers.into_iter().map(From::from).collect(),
-                output_notes,
+                input_notes: self.input_notes.into_iter().map(Into::into).collect(),
+                output_notes: self.output_notes.into_iter().map(Into::into).collect(),
+                fee: Some(Asset::from(self.fee).into()),
             }),
             block_num: self.block_num.as_u32(),
         }

crates/store/src/db/models/queries/transactions.rs

@@ -19,8 +19,8 @@ use miden_node_utils::limiter::{
 };
 use miden_protocol::account::AccountId;
 use miden_protocol::block::BlockNumber;
-use miden_protocol::note::{NoteId, Nullifier};
-use miden_protocol::transaction::{OrderedTransactionHeaders, TransactionId};
+use miden_protocol::note::NoteHeader;
+use miden_protocol::transaction::{InputNoteCommitment, OrderedTransactionHeaders, TransactionId};
 use miden_protocol::utils::{Deserializable, Serializable};
 
 use super::DatabaseError;
@@ -38,33 +38,32 @@ pub struct TransactionRecordRaw {
     transaction_id: Vec<u8>,
     initial_state_commitment: Vec<u8>,
     final_state_commitment: Vec<u8>,
-    nullifiers: Vec<u8>,
+    input_notes: Vec<u8>,
     output_notes: Vec<u8>,
     size_in_bytes: i64,
+    fee: Vec<u8>,
 }
 
 impl TryInto<crate::db::TransactionRecord> for TransactionRecordRaw {
     type Error = DatabaseError;
     fn try_into(self) -> Result<crate::db::TransactionRecord, Self::Error> {
         use miden_protocol::Word;
+        use miden_protocol::asset::FungibleAsset;
 
-        let initial_state_commitment = self.initial_state_commitment;
-        let final_state_commitment = self.final_state_commitment;
-        let nullifiers_binary = self.nullifiers;
-        let output_notes_binary = self.output_notes;
-
-        // Deserialize input notes as nullifiers and output notes as note IDs
-        let nullifiers: Vec<Nullifier> = Deserializable::read_from_bytes(&nullifiers_binary)?;
-        let output_notes: Vec<NoteId> = Deserializable::read_from_bytes(&output_notes_binary)?;
+        let input_notes: Vec<InputNoteCommitment> =
+            Deserializable::read_from_bytes(&self.input_notes)?;
+        let output_notes: Vec<NoteHeader> = Deserializable::read_from_bytes(&self.output_notes)?;
+        let fee = FungibleAsset::read_from_bytes(&self.fee)?;
 
         Ok(crate::db::TransactionRecord {
             account_id: AccountId::read_from_bytes(&self.account_id[..])?,
             block_num: BlockNumber::from_raw_sql(self.block_num)?,
             transaction_id: TransactionId::read_from_bytes(&self.transaction_id[..])?,
-            initial_state_commitment: Word::read_from_bytes(&initial_state_commitment)?,
-            final_state_commitment: Word::read_from_bytes(&final_state_commitment)?,
-            nullifiers,
+            initial_state_commitment: Word::read_from_bytes(&self.initial_state_commitment)?,
+            final_state_commitment: Word::read_from_bytes(&self.final_state_commitment)?,
+            input_notes,
             output_notes,
+            fee,
         })
     }
 }
@@ -108,9 +107,10 @@ pub struct TransactionSummaryRowInsert {
     block_num: i64,
     initial_state_commitment: Vec<u8>,
     final_state_commitment: Vec<u8>,
-    nullifiers: Vec<u8>,
+    input_notes: Vec<u8>,
     output_notes: Vec<u8>,
     size_in_bytes: i64,
+    fee: Vec<u8>,
 }
 
 impl TransactionSummaryRowInsert {
@@ -124,50 +124,37 @@ impl TransactionSummaryRowInsert {
     ) -> Self {
         const HEADER_BASE_SIZE: usize = 4 + 32 + 16 + 64; // block_num + tx_id + account_id + commitments
 
-        // Extract nullifiers from input notes and serialize them.
-        // We only store the nullifiers (not the full `InputNoteCommitment`) since
-        // that's all that's needed when reading back `TransactionRecords`.
-        let nullifiers: Vec<Nullifier> = transaction_header
-            .input_notes()
-            .iter()
-            .map(miden_protocol::transaction::InputNoteCommitment::nullifier)
-            .collect();
-        let nullifiers_binary = nullifiers.to_bytes();
+        // Serialize input notes as full InputNoteCommitments (nullifier + optional NoteHeader).
+        let input_notes: Vec<InputNoteCommitment> =
+            transaction_header.input_notes().iter().cloned().collect();
+        let input_notes_binary = input_notes.to_bytes();
 
-        // Extract note IDs from output note headers and serialize them.
-        // We only store the `NoteId`s (not the full `NoteHeader` with metadata) since
-        // that's all that's needed when reading back `TransactionRecords`.
-        let output_note_ids: Vec<NoteId> = transaction_header
-            .output_notes()
-            .iter()
-            .map(miden_protocol::note::NoteHeader::id)
-            .collect();
-        let output_notes_binary = output_note_ids.to_bytes();
+        // Serialize output notes as full NoteHeaders (NoteId + NoteMetadata).
+        let output_notes: Vec<NoteHeader> = transaction_header.output_notes().to_vec();
+        let output_notes_binary = output_notes.to_bytes();
 
         // Manually calculate the estimated size of the transaction header to avoid
         // the cost of serialization. The size estimation includes:
         // - 4 bytes for block number
         // - 32 bytes for transaction ID
         // - 16 bytes for account ID
         // - 64 bytes for initial + final state commitments (32 bytes each)
-        // - 32 bytes per input note (nullifier size)
-        // - 500 bytes per output note (estimated size when converted to NoteSyncRecord)
-        //
-        // Note: 500 bytes per output note is an over-estimate but ensures we don't
-        // exceed memory limits when these transactions are later converted to proto records.
-        let nullifiers_size = (transaction_header.input_notes().num_notes() * 32) as usize;
-        let output_notes_size = transaction_header.output_notes().len() * 500;
-        let size_in_bytes = (HEADER_BASE_SIZE + nullifiers_size + output_notes_size) as i64;
+        // - ~64 bytes per input note (nullifier + optional NoteHeader)
+        // - ~64 bytes per output note (NoteHeader = NoteId + NoteMetadata)
+        let input_notes_size = (transaction_header.input_notes().num_notes() as usize) * 64;
+        let output_notes_size = transaction_header.output_notes().len() * 64;
+        let size_in_bytes = (HEADER_BASE_SIZE + input_notes_size + output_notes_size) as i64;
 
         Self {
             transaction_id: transaction_header.id().to_bytes(),
             account_id: transaction_header.account_id().to_bytes(),
             block_num: block_num.to_raw_sql(),
             initial_state_commitment: transaction_header.initial_state_commitment().to_bytes(),
             final_state_commitment: transaction_header.final_state_commitment().to_bytes(),
-            nullifiers: nullifiers_binary,
+            input_notes: input_notes_binary,
             output_notes: output_notes_binary,
             size_in_bytes,
+            fee: transaction_header.fee().to_bytes(),
         }
     }
 }

crates/store/src/db/schema.rs

@@ -97,9 +97,10 @@ diesel::table! {
         block_num -> BigInt,
         initial_state_commitment -> Binary,
         final_state_commitment -> Binary,
-        nullifiers -> Binary,
+        input_notes -> Binary,
         output_notes -> Binary,
         size_in_bytes -> BigInt,
+        fee -> Binary,
     }
 }