Separate bytes for experimental TLVs

When constructing UnsignedInvoiceRequest or UnsignedBolt12Invoice, use a
separate field for experimental TLV bytes. This allows for properly
inserting the signature TLVs before the experimental TLVs when signing.

Author: jkczyz

lightning/src/offers/invoice.rs

@@ -121,10 +121,10 @@ use crate::ln::msgs::DecodeError;
 use crate::offers::invoice_macros::{invoice_accessors_common, invoice_builder_methods_common};
 #[cfg(test)]
 use crate::offers::invoice_macros::invoice_builder_methods_test;
-use crate::offers::invoice_request::{INVOICE_REQUEST_PAYER_ID_TYPE, INVOICE_REQUEST_TYPES, IV_BYTES as INVOICE_REQUEST_IV_BYTES, InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
-use crate::offers::merkle::{SignError, SignFn, SignatureTlvStream, SignatureTlvStreamRef, TaggedHash, TlvStream, WithoutSignatures, self};
+use crate::offers::invoice_request::{EXPERIMENTAL_INVOICE_REQUEST_TYPES, INVOICE_REQUEST_PAYER_ID_TYPE, INVOICE_REQUEST_TYPES, IV_BYTES as INVOICE_REQUEST_IV_BYTES, InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
+use crate::offers::merkle::{SignError, SignFn, SignatureTlvStream, SignatureTlvStreamRef, TaggedHash, TlvStream, self, SIGNATURE_TLV_RECORD_SIZE};
 use crate::offers::nonce::Nonce;
-use crate::offers::offer::{Amount, OFFER_TYPES, OfferTlvStream, OfferTlvStreamRef, Quantity};
+use crate::offers::offer::{Amount, EXPERIMENTAL_OFFER_TYPES, OFFER_TYPES, OfferTlvStream, OfferTlvStreamRef, Quantity};
 use crate::offers::parse::{Bolt12ParseError, Bolt12SemanticError, ParsedMessage};
 use crate::offers::payer::{PAYER_METADATA_TYPE, PayerTlvStream, PayerTlvStreamRef};
 use crate::offers::refund::{IV_BYTES_WITH_METADATA as REFUND_IV_BYTES_WITH_METADATA, IV_BYTES_WITHOUT_METADATA as REFUND_IV_BYTES_WITHOUT_METADATA, Refund, RefundContents};
@@ -461,6 +461,7 @@ for InvoiceBuilder<'a, DerivedSigningPubkey> {
 #[derive(Clone)]
 pub struct UnsignedBolt12Invoice {
 	bytes: Vec<u8>,
+	experimental_bytes: Vec<u8>,
 	contents: InvoiceContents,
 	tagged_hash: TaggedHash,
 }
@@ -491,19 +492,55 @@ where
 
 impl UnsignedBolt12Invoice {
 	fn new(invreq_bytes: &[u8], contents: InvoiceContents) -> Self {
+		// TLV record ranges applicable to invreq_bytes.
+		const NON_EXPERIMENTAL_TYPES: core::ops::Range<u64> = 0..INVOICE_REQUEST_TYPES.end;
+		const EXPERIMENTAL_TYPES: core::ops::Range<u64> =
+			EXPERIMENTAL_OFFER_TYPES.start..EXPERIMENTAL_INVOICE_REQUEST_TYPES.end;
+
+		let (_, _, _, invoice_tlv_stream) = contents.as_tlv_stream();
+
+		// Allocate enough space for the invoice, which will include:
+		// - all TLV records from `invreq_bytes` except signatures,
+		// - all invoice-specific TLV records, and
+		// - a signature TLV record once the invoice is signed.
+		//
+		// This assumes both the invoice request and the invoice will each only have one signature
+		// using SIGNATURE_TYPES.start as the TLV record. Thus, it is accounted for by invreq_bytes.
+		let mut bytes = Vec::with_capacity(
+			invreq_bytes.len()
+				+ invoice_tlv_stream.serialized_length()
+				+ if contents.is_for_offer() { 0 } else { SIGNATURE_TLV_RECORD_SIZE }
+		);
+
 		// Use the invoice_request bytes instead of the invoice_request TLV stream as the latter may
 		// have contained unknown TLV records, which are not stored in `InvoiceRequestContents` or
 		// `RefundContents`.
-		let (_, _, _, invoice_tlv_stream) = contents.as_tlv_stream();
-		let invoice_request_bytes = WithoutSignatures(invreq_bytes);
-		let unsigned_tlv_stream = (invoice_request_bytes, invoice_tlv_stream);
+		for record in TlvStream::new(invreq_bytes).range(NON_EXPERIMENTAL_TYPES) {
+			record.write(&mut bytes).unwrap();
+		}
 
-		let mut bytes = Vec::new();
-		unsigned_tlv_stream.write(&mut bytes).unwrap();
+		let remaining_bytes = &invreq_bytes[bytes.len()..];
 
-		let tagged_hash = TaggedHash::from_valid_tlv_stream_bytes(SIGNATURE_TAG, &bytes);
+		invoice_tlv_stream.write(&mut bytes).unwrap();
+
+		let mut experimental_tlv_stream = TlvStream::new(remaining_bytes)
+			.range(EXPERIMENTAL_TYPES)
+			.peekable();
+		let mut experimental_bytes = Vec::with_capacity(
+			remaining_bytes.len()
+				- experimental_tlv_stream
+					.peek()
+					.map_or(remaining_bytes.len(), |first_record| first_record.start)
+		);
 
-		Self { bytes, contents, tagged_hash }
+		for record in experimental_tlv_stream {
+			record.write(&mut experimental_bytes).unwrap();
+		}
+
+		let tlv_stream = TlvStream::new(&bytes).chain(TlvStream::new(&experimental_bytes));
+		let tagged_hash = TaggedHash::from_tlv_stream(SIGNATURE_TAG, tlv_stream);
+
+		Self { bytes, experimental_bytes, contents, tagged_hash }
 	}
 
 	/// Returns the [`TaggedHash`] of the invoice to sign.
@@ -528,6 +565,9 @@ macro_rules! unsigned_invoice_sign_method { ($self: ident, $self_type: ty $(, $s
 		};
 		signature_tlv_stream.write(&mut $self.bytes).unwrap();
 
+		// Append the experimental bytes after the signature.
+		WithoutLength(&$self.experimental_bytes).write(&mut $self.bytes).unwrap();
+
 		Ok(Bolt12Invoice {
 			#[cfg(not(c_bindings))]
 			bytes: $self.bytes,
@@ -882,6 +922,13 @@ impl Hash for Bolt12Invoice {
 }
 
 impl InvoiceContents {
+	fn is_for_offer(&self) -> bool {
+		match self {
+			InvoiceContents::ForOffer { .. } => true,
+			InvoiceContents::ForRefund { .. } => false,
+		}
+	}
+
 	/// Whether the original offer or refund has expired.
 	#[cfg(feature = "std")]
 	fn is_offer_or_refund_expired(&self) -> bool {
@@ -1211,7 +1258,7 @@ impl TryFrom<Vec<u8>> for UnsignedBolt12Invoice {
 
 	fn try_from(bytes: Vec<u8>) -> Result<Self, Self::Error> {
 		let invoice = ParsedMessage::<PartialInvoiceTlvStream>::try_from(bytes)?;
-		let ParsedMessage { bytes, tlv_stream } = invoice;
+		let ParsedMessage { mut bytes, tlv_stream } = invoice;
 		let (
 			payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream,
 		) = tlv_stream;
@@ -1221,7 +1268,13 @@ impl TryFrom<Vec<u8>> for UnsignedBolt12Invoice {
 
 		let tagged_hash = TaggedHash::from_valid_tlv_stream_bytes(SIGNATURE_TAG, &bytes);
 
-		Ok(UnsignedBolt12Invoice { bytes, contents, tagged_hash })
+		let offset = TlvStream::new(&bytes)
+			.range(0..INVOICE_TYPES.end)
+			.last()
+			.map_or(0, |last_record| last_record.end);
+		let experimental_bytes = bytes.split_off(offset);
+
+		Ok(UnsignedBolt12Invoice { bytes, experimental_bytes, contents, tagged_hash })
 	}
 }
 

lightning/src/offers/invoice_request.rs

@@ -69,9 +69,9 @@ use crate::ln::channelmanager::PaymentId;
 use crate::ln::features::InvoiceRequestFeatures;
 use crate::ln::inbound_payment::{ExpandedKey, IV_LEN};
 use crate::ln::msgs::DecodeError;
-use crate::offers::merkle::{SignError, SignFn, SignatureTlvStream, SignatureTlvStreamRef, TaggedHash, self};
+use crate::offers::merkle::{SignError, SignFn, SignatureTlvStream, SignatureTlvStreamRef, TaggedHash, TlvStream, self, SIGNATURE_TLV_RECORD_SIZE};
 use crate::offers::nonce::Nonce;
-use crate::offers::offer::{Offer, OfferContents, OfferId, OfferTlvStream, OfferTlvStreamRef};
+use crate::offers::offer::{EXPERIMENTAL_OFFER_TYPES, OFFER_TYPES, Offer, OfferContents, OfferId, OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{Bolt12ParseError, ParsedMessage, Bolt12SemanticError};
 use crate::offers::payer::{PayerContents, PayerTlvStream, PayerTlvStreamRef};
 use crate::offers::signer::{Metadata, MetadataMaterial};
@@ -488,6 +488,7 @@ for InvoiceRequestBuilder<'a, 'b, DerivedPayerSigningPubkey, secp256k1::All> {
 #[derive(Clone)]
 pub struct UnsignedInvoiceRequest {
 	bytes: Vec<u8>,
+	experimental_bytes: Vec<u8>,
 	contents: InvoiceRequestContents,
 	tagged_hash: TaggedHash,
 }
@@ -520,17 +521,49 @@ impl UnsignedInvoiceRequest {
 	fn new(offer: &Offer, contents: InvoiceRequestContents) -> Self {
 		// Use the offer bytes instead of the offer TLV stream as the offer may have contained
 		// unknown TLV records, which are not stored in `OfferContents`.
-		let (payer_tlv_stream, _offer_tlv_stream, invoice_request_tlv_stream) =
-			contents.as_tlv_stream();
-		let offer_bytes = WithoutLength(&offer.bytes);
-		let unsigned_tlv_stream = (payer_tlv_stream, offer_bytes, invoice_request_tlv_stream);
+		let (
+			payer_tlv_stream, _offer_tlv_stream, invoice_request_tlv_stream,
+		) = contents.as_tlv_stream();
+
+		// Allocate enough space for the invoice_request, which will include:
+		// - all TLV records from `offer.bytes`,
+		// - all invoice_request-specific TLV records, and
+		// - a signature TLV record once the invoice_request is signed.
+		let mut bytes = Vec::with_capacity(
+			offer.bytes.len()
+				+ payer_tlv_stream.serialized_length()
+				+ invoice_request_tlv_stream.serialized_length()
+				+ SIGNATURE_TLV_RECORD_SIZE
+		);
 
-		let mut bytes = Vec::new();
-		unsigned_tlv_stream.write(&mut bytes).unwrap();
+		payer_tlv_stream.write(&mut bytes).unwrap();
 
-		let tagged_hash = TaggedHash::from_valid_tlv_stream_bytes(SIGNATURE_TAG, &bytes);
+		for record in TlvStream::new(&offer.bytes).range(OFFER_TYPES) {
+			record.write(&mut bytes).unwrap();
+		}
+
+		let remaining_bytes = &offer.bytes[bytes.len() - payer_tlv_stream.serialized_length()..];
+
+		invoice_request_tlv_stream.write(&mut bytes).unwrap();
+
+		let mut experimental_tlv_stream = TlvStream::new(remaining_bytes)
+			.range(EXPERIMENTAL_OFFER_TYPES)
+			.peekable();
+		let mut experimental_bytes = Vec::with_capacity(
+			remaining_bytes.len()
+				- experimental_tlv_stream
+					.peek()
+					.map_or(remaining_bytes.len(), |first_record| first_record.start)
+		);
+
+		for record in experimental_tlv_stream {
+			record.write(&mut experimental_bytes).unwrap();
+		}
+
+		let tlv_stream = TlvStream::new(&bytes).chain(TlvStream::new(&experimental_bytes));
+		let tagged_hash = TaggedHash::from_tlv_stream(SIGNATURE_TAG, tlv_stream);
 
-		Self { bytes, contents, tagged_hash }
+		Self { bytes, experimental_bytes, contents, tagged_hash }
 	}
 
 	/// Returns the [`TaggedHash`] of the invoice to sign.
@@ -557,6 +590,9 @@ macro_rules! unsigned_invoice_request_sign_method { (
 		};
 		signature_tlv_stream.write(&mut $self.bytes).unwrap();
 
+		// Append the experimental bytes after the signature.
+		WithoutLength(&$self.experimental_bytes).write(&mut $self.bytes).unwrap();
+
 		Ok(InvoiceRequest {
 			#[cfg(not(c_bindings))]
 			bytes: $self.bytes,
@@ -1072,6 +1108,10 @@ tlv_stream!(InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef<'a>, INVOICE_REQ
 	(90, paths: (Vec<BlindedMessagePath>, WithoutLength)),
 });
 
+/// Valid type range for experimental invoice_request TLV records.
+pub(super) const EXPERIMENTAL_INVOICE_REQUEST_TYPES: core::ops::Range<u64> =
+	2_000_000_000..3_000_000_000;
+
 type FullInvoiceRequestTlvStream =
 	(PayerTlvStream, OfferTlvStream, InvoiceRequestTlvStream, SignatureTlvStream);
 
@@ -1106,7 +1146,7 @@ impl TryFrom<Vec<u8>> for UnsignedInvoiceRequest {
 
 	fn try_from(bytes: Vec<u8>) -> Result<Self, Self::Error> {
 		let invoice_request = ParsedMessage::<PartialInvoiceRequestTlvStream>::try_from(bytes)?;
-		let ParsedMessage { bytes, tlv_stream } = invoice_request;
+		let ParsedMessage { mut bytes, tlv_stream } = invoice_request;
 		let (
 			payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream,
 		) = tlv_stream;
@@ -1116,7 +1156,13 @@ impl TryFrom<Vec<u8>> for UnsignedInvoiceRequest {
 
 		let tagged_hash = TaggedHash::from_valid_tlv_stream_bytes(SIGNATURE_TAG, &bytes);
 
-		Ok(UnsignedInvoiceRequest { bytes, contents, tagged_hash })
+		let offset = TlvStream::new(&bytes)
+			.range(0..INVOICE_REQUEST_TYPES.end)
+			.last()
+			.map_or(0, |last_record| last_record.end);
+		let experimental_bytes = bytes.split_off(offset);
+
+		Ok(UnsignedInvoiceRequest { bytes, experimental_bytes, contents, tagged_hash })
 	}
 }
 

lightning/src/offers/merkle.rs

@@ -11,6 +11,7 @@
 
 use bitcoin::hashes::{Hash, HashEngine, sha256};
 use bitcoin::secp256k1::{Message, PublicKey, Secp256k1, self};
+use bitcoin::secp256k1::constants::SCHNORR_SIGNATURE_SIZE;
 use bitcoin::secp256k1::schnorr::Signature;
 use crate::io;
 use crate::util::ser::{BigSize, Readable, Writeable, Writer};
@@ -19,12 +20,16 @@ use crate::util::ser::{BigSize, Readable, Writeable, Writer};
 use crate::prelude::*;
 
 /// Valid type range for signature TLV records.
-const SIGNATURE_TYPES: core::ops::RangeInclusive<u64> = 240..=1000;
+pub(super) const SIGNATURE_TYPES: core::ops::RangeInclusive<u64> = 240..=1000;
 
 tlv_stream!(SignatureTlvStream, SignatureTlvStreamRef<'a>, SIGNATURE_TYPES, {
 	(240, signature: Signature),
 });
 
+/// Size of a TLV record in `SIGNATURE_TYPES` when the type is 1000. TLV types are encoded using
+/// BigSize, so a TLV record with type 240 will use two less bytes.
+pub(super) const SIGNATURE_TLV_RECORD_SIZE: usize = 3 + 1 + SCHNORR_SIGNATURE_SIZE;
+
 /// A hash for use in a specific context by tweaking with a context-dependent tag as per [BIP 340]
 /// and computed over the merkle root of a TLV stream to sign as defined in [BOLT 12].
 ///
@@ -164,7 +169,7 @@ fn root_hash<'a, I: core::iter::Iterator<Item = TlvRecord<'a>>>(tlv_stream: I) -
 	let branch_tag = tagged_hash_engine(sha256::Hash::hash("LnBranch".as_bytes()));
 
 	let mut leaves = Vec::new();
-	for record in TlvStream::skip_signatures(tlv_stream) {
+	for record in tlv_stream.filter(|record| !SIGNATURE_TYPES.contains(&record.r#type)) {
 		leaves.push(tagged_hash_from_engine(leaf_tag.clone(), &record.record_bytes));
 		leaves.push(tagged_hash_from_engine(nonce_tag.clone(), &record.type_bytes));
 	}
@@ -240,12 +245,6 @@ impl<'a> TlvStream<'a> {
 		self.skip_while(move |record| !types.contains(&record.r#type))
 			.take_while(move |record| take_range.contains(&record.r#type))
 	}
-
-	fn skip_signatures(
-		tlv_stream: impl core::iter::Iterator<Item = TlvRecord<'a>>
-	) -> impl core::iter::Iterator<Item = TlvRecord<'a>> {
-		tlv_stream.filter(|record| !SIGNATURE_TYPES.contains(&record.r#type))
-	}
 }
 
 /// A slice into a [`TlvStream`] for a record.
@@ -254,6 +253,8 @@ pub(super) struct TlvRecord<'a> {
 	type_bytes: &'a [u8],
 	// The entire TLV record.
 	pub(super) record_bytes: &'a [u8],
+	pub(super) start: usize,
+	pub(super) end: usize,
 }
 
 impl<'a> Iterator for TlvStream<'a> {
@@ -276,32 +277,25 @@ impl<'a> Iterator for TlvStream<'a> {
 
 			self.data.set_position(end);
 
-			Some(TlvRecord { r#type, type_bytes, record_bytes })
+			Some(TlvRecord {
+				r#type, type_bytes, record_bytes, start: start as usize, end: end as usize,
+			})
 		} else {
 			None
 		}
 	}
 }
 
-/// Encoding for a pre-serialized TLV stream that excludes any signature TLV records.
-///
-/// Panics if the wrapped bytes are not a well-formed TLV stream.
-pub(super) struct WithoutSignatures<'a>(pub &'a [u8]);
-
-impl<'a> Writeable for WithoutSignatures<'a> {
+impl<'a> Writeable for TlvRecord<'a> {
 	#[inline]
 	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-		let tlv_stream = TlvStream::new(self.0);
-		for record in TlvStream::skip_signatures(tlv_stream) {
-			writer.write_all(record.record_bytes)?;
-		}
-		Ok(())
+		writer.write_all(self.record_bytes)
 	}
 }
 
 #[cfg(test)]
 mod tests {
-	use super::{SIGNATURE_TYPES, TlvStream, WithoutSignatures};
+	use super::{SIGNATURE_TYPES, TlvStream};
 
 	use bitcoin::hashes::{Hash, sha256};
 	use bitcoin::hex::FromHex;
@@ -411,7 +405,11 @@ mod tests {
 			.unwrap();
 
 		let mut bytes_without_signature = Vec::new();
-		WithoutSignatures(&invoice_request.bytes).write(&mut bytes_without_signature).unwrap();
+		let tlv_stream_without_signatures = TlvStream::new(&invoice_request.bytes)
+			.filter(|record| !SIGNATURE_TYPES.contains(&record.r#type));
+		for record in tlv_stream_without_signatures {
+			record.write(&mut bytes_without_signature).unwrap();
+		}
 
 		assert_ne!(bytes_without_signature, invoice_request.bytes);
 		assert_eq!(

lightning/src/offers/offer.rs

@@ -1091,6 +1091,9 @@ tlv_stream!(OfferTlvStream, OfferTlvStreamRef<'a>, OFFER_TYPES, {
 	(OFFER_ISSUER_ID_TYPE, issuer_id: PublicKey),
 });
 
+/// Valid type range for experimental offer TLV records.
+pub(super) const EXPERIMENTAL_OFFER_TYPES: core::ops::Range<u64> = 1_000_000_000..2_000_000_000;
+
 impl Bech32Encode for Offer {
 	const BECH32_HRP: &'static str = "lno";
 }