partial merge 1: f321x's "lightning: refactor htlc switch"

split-off from #10230

Author: SomberNight

electrum/commands.py

@@ -1438,7 +1438,7 @@ async def settle_hold_invoice(self, preimage: str, wallet: Abstract_Wallet = Non
         assert payment_hash in wallet.lnworker.payment_info, \
             f"Couldn't find lightning invoice for {payment_hash=}"
         assert payment_hash in wallet.lnworker.dont_settle_htlcs, f"Invoice {payment_hash=} not a hold invoice?"
-        assert wallet.lnworker.is_accepted_mpp(bfh(payment_hash)), \
+        assert wallet.lnworker.is_complete_mpp(bfh(payment_hash)), \
             f"MPP incomplete, cannot settle hold invoice {payment_hash} yet"
         info: Optional['PaymentInfo'] = wallet.lnworker.get_payment_info(bfh(payment_hash))
         assert (wallet.lnworker.get_payment_mpp_amount_msat(bfh(payment_hash)) or 0) >= (info.amount_msat or 0)
@@ -1465,7 +1465,7 @@ async def cancel_hold_invoice(self, payment_hash: str, wallet: Abstract_Wallet =
         wallet.lnworker.set_payment_status(bfh(payment_hash), PR_UNPAID)
         wallet.lnworker.delete_payment_info(payment_hash)
         wallet.set_label(payment_hash, None)
-        while wallet.lnworker.is_accepted_mpp(bfh(payment_hash)):
+        while wallet.lnworker.is_complete_mpp(bfh(payment_hash)):
             # wait until the htlcs got failed so the payment won't get settled accidentally in a race
             await asyncio.sleep(0.1)
         del wallet.lnworker.dont_settle_htlcs[payment_hash]
@@ -1490,18 +1490,18 @@ async def check_hold_invoice(self, payment_hash: str, wallet: Abstract_Wallet =
         """
         assert len(payment_hash) == 64, f"Invalid payment_hash length: {len(payment_hash)} != 64"
         info: Optional['PaymentInfo'] = wallet.lnworker.get_payment_info(bfh(payment_hash))
-        is_accepted_mpp: bool = wallet.lnworker.is_accepted_mpp(bfh(payment_hash))
+        is_complete_mpp: bool = wallet.lnworker.is_complete_mpp(bfh(payment_hash))
         amount_sat = (wallet.lnworker.get_payment_mpp_amount_msat(bfh(payment_hash)) or 0) // 1000
         result = {
             "status": "unknown",
             "received_amount_sat": amount_sat,
         }
         if info is None:
             pass
-        elif not is_accepted_mpp and not wallet.lnworker.get_preimage_hex(payment_hash):
-            # is_accepted_mpp is False for settled payments
+        elif not is_complete_mpp and not wallet.lnworker.get_preimage_hex(payment_hash):
+            # is_complete_mpp is False for settled payments
             result["status"] = "unpaid"
-        elif is_accepted_mpp and payment_hash in wallet.lnworker.dont_settle_htlcs:
+        elif is_complete_mpp and payment_hash in wallet.lnworker.dont_settle_htlcs:
             result["status"] = "paid"
             payment_key: str = wallet.lnworker._get_payment_key(bfh(payment_hash)).hex()
             htlc_status = wallet.lnworker.received_mpp_htlcs[payment_key]

electrum/lnchannel.py

@@ -17,6 +17,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
+import dataclasses
 import enum
 from collections import defaultdict
 from enum import IntEnum, Enum
@@ -1202,12 +1203,10 @@ def add_htlc(self, htlc: UpdateAddHtlc) -> UpdateAddHtlc:
         """Adds a new LOCAL HTLC to the channel.
         Action must be initiated by LOCAL.
         """
-        if isinstance(htlc, dict):  # legacy conversion  # FIXME remove
-            htlc = UpdateAddHtlc(**htlc)
         assert isinstance(htlc, UpdateAddHtlc)
         self._assert_can_add_htlc(htlc_proposer=LOCAL, amount_msat=htlc.amount_msat)
         if htlc.htlc_id is None:
-            htlc = attr.evolve(htlc, htlc_id=self.hm.get_next_htlc_id(LOCAL))
+            htlc = dataclasses.replace(htlc, htlc_id=self.hm.get_next_htlc_id(LOCAL))
         with self.db_lock:
             self.hm.send_htlc(htlc)
         self.logger.info("add_htlc")
@@ -1217,15 +1216,13 @@ def receive_htlc(self, htlc: UpdateAddHtlc, onion_packet:bytes = None) -> Update
         """Adds a new REMOTE HTLC to the channel.
         Action must be initiated by REMOTE.
         """
-        if isinstance(htlc, dict):  # legacy conversion  # FIXME remove
-            htlc = UpdateAddHtlc(**htlc)
         assert isinstance(htlc, UpdateAddHtlc)
         try:
             self._assert_can_add_htlc(htlc_proposer=REMOTE, amount_msat=htlc.amount_msat)
         except PaymentFailure as e:
             raise RemoteMisbehaving(e) from e
         if htlc.htlc_id is None:  # used in unit tests
-            htlc = attr.evolve(htlc, htlc_id=self.hm.get_next_htlc_id(REMOTE))
+            htlc = dataclasses.replace(htlc, htlc_id=self.hm.get_next_htlc_id(REMOTE))
         with self.db_lock:
             self.hm.recv_htlc(htlc)
             if onion_packet:

electrum/lnonion.py

@@ -36,6 +36,7 @@
                      NUM_MAX_EDGES_IN_PAYMENT_PATH, ShortChannelID, OnionFailureCodeMetaFlag)
 from .lnmsg import OnionWireSerializer, read_bigsize_int, write_bigsize_int
 from . import lnmsg
+from . import util
 
 if TYPE_CHECKING:
     from .lnrouter import LNPaymentRoute
@@ -113,11 +114,11 @@ def __repr__(self):
 
 class OnionPacket:
 
-    def __init__(self, public_key: bytes, hops_data: bytes, hmac: bytes):
+    def __init__(self, public_key: bytes, hops_data: bytes, hmac: bytes, version: int = 0):
         assert len(public_key) == 33
         assert len(hops_data) in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE, ONION_MESSAGE_LARGE_SIZE]
         assert len(hmac) == PER_HOP_HMAC_SIZE
-        self.version = 0
+        self.version = version
         self.public_key = public_key
         self.hops_data = hops_data  # also called RoutingInfo in bolt-04
         self.hmac = hmac
@@ -140,13 +141,11 @@ def to_bytes(self) -> bytes:
     def from_bytes(cls, b: bytes):
         if len(b) - 66 not in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE, ONION_MESSAGE_LARGE_SIZE]:
             raise Exception('unexpected length {}'.format(len(b)))
-        version = b[0]
-        if version != 0:
-            raise UnsupportedOnionPacketVersion('version {} is not supported'.format(version))
         return OnionPacket(
             public_key=b[1:34],
             hops_data=b[34:-32],
-            hmac=b[-32:]
+            hmac=b[-32:],
+            version=b[0],
         )
 
 
@@ -361,6 +360,9 @@ def process_onion_packet(
         associated_data: bytes = b'',
         is_trampoline=False,
         tlv_stream_name='payload') -> ProcessedOnionPacket:
+    # TODO: check Onion features ( PERM|NODE|3 (required_node_feature_missing )
+    if onion_packet.version != 0:
+        raise UnsupportedOnionPacketVersion()
     if not ecc.ECPubkey.is_pubkey_bytes(onion_packet.public_key):
         raise InvalidOnionPubkey()
     shared_secret = get_ecdh(our_onion_private_key, onion_packet.public_key)
@@ -369,7 +371,7 @@ def process_onion_packet(
     calculated_mac = hmac_oneshot(
         mu_key, msg=onion_packet.hops_data+associated_data,
         digest=hashlib.sha256)
-    if onion_packet.hmac != calculated_mac:
+    if not util.constant_time_compare(onion_packet.hmac, calculated_mac):
         raise InvalidOnionMac()
     # peel an onion layer off
     rho_key = get_bolt04_onion_key(b'rho', shared_secret)
@@ -484,23 +486,38 @@ def obfuscate_onion_error(error_packet, their_public_key, our_onion_private_key)
 
 def _decode_onion_error(error_packet: bytes, payment_path_pubkeys: Sequence[bytes],
                         session_key: bytes) -> Tuple[bytes, int]:
-    """Returns the decoded error bytes, and the index of the sender of the error."""
+    """
+    Returns the decoded error bytes, and the index of the sender of the error.
+    https://github.com/lightning/bolts/blob/14272b1bd9361750cfdb3e5d35740889a6b510b5/04-onion-routing.md?plain=1#L1096
+    """
     num_hops = len(payment_path_pubkeys)
     hop_shared_secrets, _ = get_shared_secrets_along_route(payment_path_pubkeys, session_key)
-    for i in range(num_hops):
-        ammag_key = get_bolt04_onion_key(b'ammag', hop_shared_secrets[i])
-        um_key = get_bolt04_onion_key(b'um', hop_shared_secrets[i])
+    result = None
+    dummy_secret = bytes(32)
+    # SHOULD continue decrypting, until the loop has been repeated 27 times
+    for i in range(27):
+        if i < num_hops:
+            ammag_key = get_bolt04_onion_key(b'ammag', hop_shared_secrets[i])
+            um_key = get_bolt04_onion_key(b'um', hop_shared_secrets[i])
+        else:
+            # SHOULD use constant `ammag` and `um` keys to obfuscate the route length.
+            ammag_key = get_bolt04_onion_key(b'ammag', dummy_secret)
+            um_key = get_bolt04_onion_key(b'um', dummy_secret)
+
         stream_bytes = generate_cipher_stream(ammag_key, len(error_packet))
         error_packet = xor_bytes(error_packet, stream_bytes)
         hmac_computed = hmac_oneshot(um_key, msg=error_packet[32:], digest=hashlib.sha256)
         hmac_found = error_packet[:32]
-        if hmac_computed == hmac_found:
-            return error_packet, i
+        if util.constant_time_compare(hmac_found, hmac_computed) and i < num_hops:
+            result = error_packet, i
+
+    if result is not None:
+        return result
     raise FailedToDecodeOnionError()
 
 
 def decode_onion_error(error_packet: bytes, payment_path_pubkeys: Sequence[bytes],
-                       session_key: bytes) -> (OnionRoutingFailure, int):
+                       session_key: bytes) -> Tuple[OnionRoutingFailure, int]:
     """Returns the failure message, and the index of the sender of the error."""
     decrypted_error, sender_index = _decode_onion_error(error_packet, payment_path_pubkeys, session_key)
     failure_msg = get_failure_msg_from_onion_error(decrypted_error)

electrum/lnpeer.py

@@ -24,7 +24,7 @@
 from .crypto import sha256, sha256d, privkey_to_pubkey
 from . import bitcoin, util
 from . import constants
-from .util import (bfh, log_exceptions, ignore_exceptions, chunks, OldTaskGroup,
+from .util import (log_exceptions, ignore_exceptions, chunks, OldTaskGroup,
                    UnrelatedTransactionException, error_text_bytes_to_safe_str, AsyncHangDetector,
                    NoDynamicFeeEstimates, event_listener, EventListener)
 from . import transaction
@@ -36,7 +36,7 @@
                       OnionPacket, construct_onion_error, obfuscate_onion_error, OnionRoutingFailure,
                       ProcessedOnionPacket, UnsupportedOnionPacketVersion, InvalidOnionMac, InvalidOnionPubkey,
                       OnionFailureCodeMetaFlag)
-from .lnchannel import Channel, RevokeAndAck, RemoteCtnTooFarInFuture, ChannelState, PeerState, ChanCloseOption, CF_ANNOUNCE_CHANNEL
+from .lnchannel import Channel, RevokeAndAck, ChannelState, PeerState, ChanCloseOption, CF_ANNOUNCE_CHANNEL
 from . import lnutil
 from .lnutil import (Outpoint, LocalConfig, RECEIVED, UpdateAddHtlc, ChannelConfig,
                      RemoteConfig, OnlyPubkeyKeypair, ChannelConstraints, RevocationStore,
@@ -45,19 +45,17 @@
                      LOCAL, REMOTE, HTLCOwner,
                      ln_compare_features, MIN_FINAL_CLTV_DELTA_ACCEPTED,
                      RemoteMisbehaving, ShortChannelID,
-                     IncompatibleLightningFeatures, derive_payment_secret_from_payment_preimage,
-                     ChannelType, LNProtocolWarning, validate_features,
+                     IncompatibleLightningFeatures, ChannelType, LNProtocolWarning, validate_features,
                      IncompatibleOrInsaneFeatures, FeeBudgetExceeded,
-                     GossipForwardingMessage, GossipTimestampFilter)
-from .lnutil import FeeUpdate, channel_id_from_funding_tx, PaymentFeeBudget
-from .lnutil import serialize_htlc_key, Keypair
+                     GossipForwardingMessage, GossipTimestampFilter, channel_id_from_funding_tx,
+                     PaymentFeeBudget, serialize_htlc_key, Keypair, RecvMPPResolution)
 from .lntransport import LNTransport, LNTransportBase, LightningPeerConnectionClosed, HandshakeFailed
 from .lnmsg import encode_msg, decode_msg, UnknownOptionalMsgType, FailedToParseMsg
 from .interface import GracefulDisconnect
 from .lnrouter import fee_for_edge_msat
 from .json_db import StoredDict
 from .invoices import PR_PAID
-from .fee_policy import FEE_LN_ETA_TARGET, FEE_LN_MINIMUM_ETA_TARGET, FEERATE_PER_KW_MIN_RELAY_LIGHTNING
+from .fee_policy import FEE_LN_ETA_TARGET, FEERATE_PER_KW_MIN_RELAY_LIGHTNING
 from .trampoline import decode_routing_info
 
 if TYPE_CHECKING:
@@ -525,12 +523,13 @@ async def wrapper_func(self, *args, **kwargs):
     @handle_disconnect
     async def main_loop(self):
         async with self.taskgroup as group:
-            await group.spawn(self.htlc_switch())
             await group.spawn(self._message_loop())
             await group.spawn(self._query_gossip())
             await group.spawn(self._process_gossip())
             await group.spawn(self._send_own_gossip())
             await group.spawn(self._forward_gossip())
+            if self.network.lngossip != self.lnworker:
+                await group.spawn(self.htlc_switch())
 
     async def _process_gossip(self):
         while True:
@@ -2467,7 +2466,6 @@ def check_mpp_is_waiting(
         exc_incorrect_or_unknown_pd: OnionRoutingFailure,
         log_fail_reason: Callable[[str], None],
     ) -> bool:
-        from .lnworker import RecvMPPResolution
         mpp_resolution = self.lnworker.check_mpp_status(
             payment_secret=payment_secret,
             short_channel_id=short_channel_id,
@@ -2482,7 +2480,7 @@ def check_mpp_is_waiting(
         elif mpp_resolution == RecvMPPResolution.FAILED:
             log_fail_reason(f"mpp_resolution is FAILED")
             raise exc_incorrect_or_unknown_pd
-        elif mpp_resolution == RecvMPPResolution.ACCEPTED:
+        elif mpp_resolution == RecvMPPResolution.COMPLETE:
             return False
         else:
             raise Exception(f"unexpected {mpp_resolution=}")
@@ -2592,11 +2590,9 @@ def log_fail_reason(reason: str):
             raise exc_incorrect_or_unknown_pd
 
         preimage = self.lnworker.get_preimage(payment_hash)
-        expected_payment_secrets = [self.lnworker.get_payment_secret(htlc.payment_hash)]
-        if preimage:
-            expected_payment_secrets.append(derive_payment_secret_from_payment_preimage(preimage)) # legacy secret for old invoices
-        if payment_secret_from_onion not in expected_payment_secrets:
-            log_fail_reason(f'incorrect payment secret {payment_secret_from_onion.hex()} != {expected_payment_secrets[0].hex()}')
+        expected_payment_secret = self.lnworker.get_payment_secret(htlc.payment_hash)
+        if payment_secret_from_onion != expected_payment_secret:
+            log_fail_reason(f'incorrect payment secret {payment_secret_from_onion.hex()} != {expected_payment_secret.hex()}')
             raise exc_incorrect_or_unknown_pd
         invoice_msat = info.amount_msat
         if channel_opening_fee:

electrum/lnsweep.py

@@ -432,7 +432,7 @@ def txs_htlc(
         ctn=ctn)
     for (direction, htlc), (ctx_output_idx, htlc_relative_idx) in htlc_to_ctx_output_idx_map.items():
         if direction == RECEIVED:
-            if not chan.lnworker.is_accepted_mpp(htlc.payment_hash):
+            if not chan.lnworker.is_complete_mpp(htlc.payment_hash):
                 # do not redeem this, it might publish the preimage of an incomplete MPP
                 continue
             preimage = chan.lnworker.get_preimage(htlc.payment_hash)
@@ -727,7 +727,7 @@ def tx_htlc(
     for (direction, htlc), (ctx_output_idx, htlc_relative_idx) in htlc_to_ctx_output_idx_map.items():
         is_received_htlc = direction == RECEIVED
         if not is_received_htlc and not is_revocation:
-            if not chan.lnworker.is_accepted_mpp(htlc.payment_hash):
+            if not chan.lnworker.is_complete_mpp(htlc.payment_hash):
                 # do not redeem this, it might publish the preimage of an incomplete MPP
                 continue
             preimage = chan.lnworker.get_preimage(htlc.payment_hash)