htlcswitch+routing: use attributable failures

We now call into the new encryption/decryption methods and provide the
received attribution data as an argument. The result is then also
returned to our upstream peer.

Author: GeorgeTsagk

htlcswitch/failure.go

@@ -3,6 +3,7 @@ package htlcswitch
 import (
 	"bytes"
 	"fmt"
+	"strings"
 
 	sphinx "github.com/lightningnetwork/lightning-onion"
 	"github.com/lightningnetwork/lnd/htlcswitch/hop"
@@ -92,6 +93,13 @@ type ForwardingError struct {
 	// be nil in the case where we fail to decode failure message sent by
 	// a peer.
 	msg lnwire.FailureMessage
+
+	// HoldTimes is an array of hold times (in ms) as reported from the
+	// nodes of the route. It is the time for which a node held the HTLC for
+	// from that nodes local perspective. The first element corresponds to
+	// the first node after the sender node, with greater indices indicating
+	// nodes further down the route.
+	HoldTimes []uint32
 }
 
 // WireMessage extracts a valid wire failure message from an internal
@@ -116,11 +124,12 @@ func (f *ForwardingError) Error() string {
 // NewForwardingError creates a new payment error which wraps a wire error
 // with additional metadata.
 func NewForwardingError(failure lnwire.FailureMessage,
-	index int) *ForwardingError {
+	index int, holdTimes []uint32) *ForwardingError {
 
 	return &ForwardingError{
 		FailureSourceIdx: index,
 		msg:              failure,
+		HoldTimes:        holdTimes,
 	}
 }
 
@@ -140,7 +149,7 @@ type ErrorDecrypter interface {
 	// hop, to the source of the error. A fully populated
 	// lnwire.FailureMessage is returned along with the source of the
 	// error.
-	DecryptError(lnwire.OpaqueReason) (*ForwardingError, error)
+	DecryptError(lnwire.OpaqueReason, []byte) (*ForwardingError, error)
 }
 
 // UnknownEncrypterType is an error message used to signal that an unexpected
@@ -160,37 +169,70 @@ type OnionErrorDecrypter interface {
 	// node where error have occurred. As a result, in order to decrypt the
 	// error we need get all shared secret and apply decryption in the
 	// reverse order.
-	DecryptError(encryptedData, _ []byte, _ bool) (*sphinx.DecryptedError, error)
+	DecryptError(encryptedData, attrData []byte) (*sphinx.DecryptedError,
+		error)
 }
 
 // SphinxErrorDecrypter wraps the sphinx data SphinxErrorDecrypter and maps the
 // returned errors to concrete lnwire.FailureMessage instances.
 type SphinxErrorDecrypter struct {
-	OnionErrorDecrypter
+	decrypter *sphinx.OnionErrorDecrypter
+}
+
+// NewSphinxErrorDecrypter instantiates a new error decrypter.
+func NewSphinxErrorDecrypter(circuit *sphinx.Circuit) *SphinxErrorDecrypter {
+	return &SphinxErrorDecrypter{
+		decrypter: sphinx.NewOnionErrorDecrypter(
+			circuit, hop.AttrErrorStruct,
+		),
+	}
 }
 
 // DecryptError peels off each layer of onion encryption from the first hop, to
 // the source of the error. A fully populated lnwire.FailureMessage is returned
 // along with the source of the error.
 //
 // NOTE: Part of the ErrorDecrypter interface.
-func (s *SphinxErrorDecrypter) DecryptError(reason lnwire.OpaqueReason) (
-	*ForwardingError, error) {
-
-	failure, err := s.OnionErrorDecrypter.DecryptError(reason, nil, false)
+func (s *SphinxErrorDecrypter) DecryptError(reason lnwire.OpaqueReason,
+	attrData []byte) (*ForwardingError, error) {
+
+	// We do not set the strict attribution flag, as we want to account for
+	// the grace period during which nodes are still upgrading to support
+	// this feature. If set prematurely it can lead to early blame of our
+	// direct peers that may not support this feature yet, blacklisting our
+	// channels and failing our payments.
+	attrErr, err := s.decrypter.DecryptError(reason, attrData, false)
 	if err != nil {
 		return nil, err
 	}
 
+	var holdTimes []string
+	for _, payload := range attrErr.HoldTimes {
+		// Read hold time.
+		holdTimeMs := payload
+
+		holdTimes = append(
+			holdTimes,
+			fmt.Sprintf("%v", holdTimeMs),
+		)
+	}
+
+	// For now just log the hold times, the collector of the payment result
+	// should handle this in a more sophisticated way.
+	log.Tracef("Extracted hold times from onion error: %v",
+		strings.Join(holdTimes, "/"))
+
 	// Decode the failure. If an error occurs, we leave the failure message
 	// field nil.
-	r := bytes.NewReader(failure.Message)
+	r := bytes.NewReader(attrErr.Message)
 	failureMsg, err := lnwire.DecodeFailure(r, 0)
 	if err != nil {
-		return NewUnknownForwardingError(failure.SenderIdx), nil
+		return NewUnknownForwardingError(attrErr.SenderIdx), nil
 	}
 
-	return NewForwardingError(failureMsg, failure.SenderIdx), nil
+	return NewForwardingError(
+		failureMsg, attrErr.SenderIdx, attrErr.HoldTimes,
+	), nil
 }
 
 // A compile time check to ensure ErrorDecrypter implements the Deobfuscator

htlcswitch/hop/error_encryptor.go

@@ -2,12 +2,16 @@ package hop
 
 import (
 	"bytes"
+	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/btcsuite/btcd/btcec/v2"
 	sphinx "github.com/lightningnetwork/lightning-onion"
 	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/tlv"
 )
 
 // EncrypterType establishes an enum used in serialization to indicate how to
@@ -37,6 +41,25 @@ const (
 	// the same functionality as a EncrypterTypeSphinx, but is used to mark
 	// our special-case error handling.
 	EncrypterTypeRelaying = 4
+
+	// A set of tlv type definitions used to serialize the encrypter to the
+	// database.
+	//
+	// NOTE: A migration should be added whenever this list changes. This
+	// prevents against the database being rolled back to an older
+	// format where the surrounding logic might assume a different set of
+	// fields are known.
+	creationTimeType tlv.Type = 0
+)
+
+// AttrErrorStruct defines the message structure for an attributable error. Use
+// a maximum route length of 20, a fixed payload length of 4 bytes to
+// accommodate the a 32-bit hold time in milliseconds and use 4 byte hmacs.
+// Total size including a 256 byte message from the error source works out to
+// 1200 bytes.
+var (
+	AttrErrorStruct = sphinx.NewAttrErrorStructure(20, 4, 4)
+	byteOrder       = binary.BigEndian
 )
 
 // IsBlinded returns a boolean indicating whether the error encrypter belongs
@@ -58,8 +81,8 @@ type ErrorEncrypter interface {
 	// encrypted opaque failure reason. This method will be used at the
 	// source that the error occurs. It differs from IntermediateEncrypt
 	// slightly, in that it computes a proper MAC over the error.
-	EncryptFirstHop(lnwire.FailureMessage) (lnwire.OpaqueReason, []byte,
-		error)
+	EncryptFirstHop(lnwire.FailureMessage) (lnwire.OpaqueReason,
+		[]byte, error)
 
 	// EncryptMalformedError is similar to EncryptFirstHop (it adds the
 	// MAC), but it accepts an opaque failure reason rather than a failure
@@ -104,6 +127,7 @@ type SphinxErrorEncrypter struct {
 	*sphinx.OnionErrorEncrypter
 
 	EphemeralKey *btcec.PublicKey
+	CreatedAt    time.Time
 }
 
 // NewSphinxErrorEncrypterUninitialized initializes a blank sphinx error
@@ -115,8 +139,7 @@ type SphinxErrorEncrypter struct {
 //     OnionProcessor.
 func NewSphinxErrorEncrypterUninitialized() *SphinxErrorEncrypter {
 	return &SphinxErrorEncrypter{
-		OnionErrorEncrypter: nil,
-		EphemeralKey:        &btcec.PublicKey{},
+		EphemeralKey: &btcec.PublicKey{},
 	}
 }
 
@@ -131,15 +154,35 @@ func NewSphinxErrorEncrypter(ephemeralKey *btcec.PublicKey,
 		EphemeralKey: ephemeralKey,
 	}
 
+	// Set creation time rounded to nanosecond to avoid differences after
+	// serialization.
+	encrypter.CreatedAt = time.Now().Truncate(time.Nanosecond)
+
 	encrypter.initialize(sharedSecret)
 
 	return encrypter
 }
 
+// getHoldTimeMs returns the hold time in milliseconds since the first
+// instantiation of this sphinx error encrypter.
+func (s *SphinxErrorEncrypter) getHoldTimeMs() uint32 {
+	return uint32(time.Since(s.CreatedAt).Milliseconds())
+}
+
+// encrypt is a thin wrapper around the main encryption method, mainly used to
+// automatically derive the hold time to encode in the attribution structure.
+func (s *SphinxErrorEncrypter) encrypt(initial bool,
+	data, attrData []byte) (lnwire.OpaqueReason, []byte, error) {
+
+	holdTimeMs := s.getHoldTimeMs()
+
+	return s.EncryptError(initial, data, attrData, holdTimeMs)
+}
+
 // initialize creates the underlying instance of the sphinx error encrypter.
 func (s *SphinxErrorEncrypter) initialize(sharedSecret sphinx.Hash256) {
 	s.OnionErrorEncrypter = sphinx.NewOnionErrorEncrypter(
-		sharedSecret, nil,
+		sharedSecret, AttrErrorStruct,
 	)
 }
 
@@ -157,9 +200,7 @@ func (s *SphinxErrorEncrypter) EncryptFirstHop(
 		return nil, nil, err
 	}
 
-	// We pass a true as the first parameter to indicate that a MAC should
-	// be added.
-	return s.EncryptError(true, b.Bytes(), nil, 0)
+	return s.encrypt(true, b.Bytes(), nil)
 }
 
 // EncryptMalformedError is similar to EncryptFirstHop (it adds the MAC), but
@@ -172,7 +213,7 @@ func (s *SphinxErrorEncrypter) EncryptFirstHop(
 func (s *SphinxErrorEncrypter) EncryptMalformedError(
 	reason lnwire.OpaqueReason) (lnwire.OpaqueReason, []byte, error) {
 
-	return s.EncryptError(true, reason, nil, 0)
+	return s.encrypt(true, reason, nil)
 }
 
 // IntermediateEncrypt wraps an already encrypted opaque reason error in an
@@ -183,10 +224,25 @@ func (s *SphinxErrorEncrypter) EncryptMalformedError(
 //
 // NOTE: Part of the ErrorEncrypter interface.
 func (s *SphinxErrorEncrypter) IntermediateEncrypt(
-	reason lnwire.OpaqueReason, _ []byte) (lnwire.OpaqueReason, []byte,
-	error) {
+	reason lnwire.OpaqueReason, attrData []byte) (lnwire.OpaqueReason,
+	[]byte, error) {
+
+	encrypted, attrData, err := s.encrypt(false, reason, attrData)
+
+	switch {
+	// If the structure of the error received from downstream is invalid,
+	// then generate a new attribution structure so that the sender is able
+	// to penalize the offending node.
+	case errors.Is(err, sphinx.ErrInvalidAttrStructure):
+		// Preserve the error message and initialize fresh attribution
+		// data.
+		return s.encrypt(true, reason, nil)
+
+	case err != nil:
+		return lnwire.OpaqueReason{}, nil, err
+	}
 
-	return s.EncryptError(false, reason, nil, 0)
+	return encrypted, attrData, nil
 }
 
 // Type returns the identifier for a sphinx error encrypter.
@@ -199,7 +255,20 @@ func (s *SphinxErrorEncrypter) Type() EncrypterType {
 func (s *SphinxErrorEncrypter) Encode(w io.Writer) error {
 	ephemeral := s.EphemeralKey.SerializeCompressed()
 	_, err := w.Write(ephemeral)
-	return err
+	if err != nil {
+		return err
+	}
+
+	var creationTime = uint64(s.CreatedAt.UnixNano())
+
+	tlvStream, err := tlv.NewStream(
+		tlv.MakePrimitiveRecord(creationTimeType, &creationTime),
+	)
+	if err != nil {
+		return err
+	}
+
+	return tlvStream.Encode(w)
 }
 
 // Decode reconstructs the error encrypter's ephemeral public key from the
@@ -216,6 +285,29 @@ func (s *SphinxErrorEncrypter) Decode(r io.Reader) error {
 		return err
 	}
 
+	// Try decode attributable error structure.
+	var creationTime uint64
+
+	tlvStream, err := tlv.NewStream(
+		tlv.MakePrimitiveRecord(creationTimeType, &creationTime),
+	)
+	if err != nil {
+		return err
+	}
+
+	typeMap, err := tlvStream.DecodeWithParsedTypes(r)
+	if err != nil {
+		return err
+	}
+
+	// Return early if this encrypter is not for attributable errors.
+	if len(typeMap) == 0 {
+		return nil
+	}
+
+	// Set attributable error creation time.
+	s.CreatedAt = time.Unix(0, int64(creationTime))
+
 	return nil
 }
 

htlcswitch/interceptable_switch.go

@@ -809,13 +809,19 @@ func (f *interceptedForward) FailWithCode(code lnwire.FailCode) error {
 
 	// Encrypt the failure for the first hop. This node will be the origin
 	// of the failure.
-	reason, _, err := f.packet.obfuscator.EncryptFirstHop(failureMsg)
+	reason, attrData, err := f.packet.obfuscator.EncryptFirstHop(failureMsg)
 	if err != nil {
 		return fmt.Errorf("failed to encrypt failure reason %w", err)
 	}
 
+	extraData, err := lnwire.AttrDataToExtraData(attrData)
+	if err != nil {
+		return err
+	}
+
 	return f.resolve(&lnwire.UpdateFailHTLC{
-		Reason: reason,
+		Reason:    reason,
+		ExtraData: extraData,
 	})
 }
 

htlcswitch/link.go

@@ -4361,13 +4361,20 @@ func (l *channelLink) sendHTLCError(add lnwire.UpdateAddHTLC,
 	sourceRef channeldb.AddRef, failure *LinkError,
 	e hop.ErrorEncrypter, isReceive bool) {
 
-	reason, _, err := e.EncryptFirstHop(failure.WireMessage())
+	reason, attrData, err := e.EncryptFirstHop(failure.WireMessage())
 	if err != nil {
 		l.log.Errorf("unable to obfuscate error: %v", err)
 		return
 	}
 
-	err = l.channel.FailHTLC(add.ID, reason, nil, &sourceRef, nil, nil)
+	extraData, err := lnwire.AttrDataToExtraData(attrData)
+	if err != nil {
+		return
+	}
+
+	err = l.channel.FailHTLC(
+		add.ID, reason, extraData, &sourceRef, nil, nil,
+	)
 	if err != nil {
 		l.log.Errorf("unable cancel htlc: %v", err)
 		return
@@ -4376,7 +4383,7 @@ func (l *channelLink) sendHTLCError(add lnwire.UpdateAddHTLC,
 	// Send the appropriate failure message depending on whether we're
 	// in a blinded route or not.
 	if err := l.sendIncomingHTLCFailureMsg(
-		add.ID, e, reason, nil,
+		add.ID, e, reason, extraData,
 	); err != nil {
 		l.log.Errorf("unable to send HTLC failure: %v", err)
 		return