ln: move constants to common file and refactor miner funds distro

Author: pinheadmz

resources/scenarios/ln_framework/ln.py

@@ -15,6 +15,27 @@
 INSECURE_CONTEXT.check_hostname = False
 INSECURE_CONTEXT.verify_mode = ssl.CERT_NONE
 
+# These values may need to be tweaked depending on the network being deployed.
+# Currently passes all tests and ln_init succeeds on these examples:
+#  test/data/LN_10.json
+#  test/data/LN_50.json
+#  test/data/LN_100.json
+# If any values are changed, you may need to re-build network.yaml with import-network.
+# Issues I encountered while setting on these values:
+# - Too many blocks generated, ln_init takes too long
+# - TX that distributes miner funds to LN wallets exceeds standard weight limit
+# - Too many miner distribution TXs result in too-long-mempool-chain
+# - Not enough UTXO value, forcing LN nodes to combine UTXOs to open large channels
+#   which results in the change output being too big which results in the tx
+#   outputs being ordered unexpectedly (which change at 0 and channel open at 1)
+# - LND actual fee rate ends up way off from the expected value
+# LN networks with more than 100 nodes and 500 channels may also need to tweak ln_init.py
+CHANNEL_OPEN_START_HEIGHT = 500
+CHANNEL_OPENS_PER_BLOCK = 200
+MAX_FEE_RATE = 40006  # s/vB
+FEE_RATE_DECREMENT = 200
+assert MAX_FEE_RATE - (FEE_RATE_DECREMENT * CHANNEL_OPENS_PER_BLOCK) > 1
+
 
 # https://github.com/lightningcn/lightning-rfc/blob/master/07-routing-gossip.md#the-channel_update-message
 # We use the field names as written in the BOLT as our canonical, internal field names.

resources/scenarios/ln_init.py

@@ -4,7 +4,19 @@
 from time import sleep
 
 from commander import Commander
-from ln_framework.ln import Policy
+from ln_framework.ln import (
+    Policy,
+    CHANNEL_OPEN_START_HEIGHT,
+    CHANNEL_OPENS_PER_BLOCK,
+    MAX_FEE_RATE,
+    FEE_RATE_DECREMENT
+)
+from test_framework.messages import (
+    COIN,
+    CTransaction,
+    CTxOut,
+)
+from test_framework.address import address_to_scriptpubkey
 
 
 class LNInit(Commander):
@@ -39,13 +51,13 @@ def gen(n):
         # WALLET ADDRESSES
         ##
         self.log.info("Getting LN wallet addresses...")
-        ln_addrs = []
+        ln_addrs = {}
 
         def get_ln_addr(self, ln):
             while True:
                 try:
                     address = ln.newaddress()
-                    ln_addrs.append(address)
+                    ln_addrs[ln.name] = address
                     self.log.info(f"Got wallet address {address} from {ln.name}")
                     break
                 except Exception as e:
@@ -67,45 +79,71 @@ def get_ln_addr(self, ln):
         # FUNDS
         ##
         self.log.info("Funding LN wallets...")
-        # 298 block base for miner wallet
-        gen(297)
+        # One past block generated already to lock out IBD
+        # One next block to consolidate the miner's coins
+        # One next block to confirm the distributed coins
+        # Then the channel open TXs go in the expected block height
+        gen(CHANNEL_OPEN_START_HEIGHT - 4)
         # divvy up the goods, except fee.
-        # 10 UTXOs per node means 10 channel opens per node per block
-        split = (miner.getbalance() - 1) // len(ln_addrs) // 10
-        sends = {}
-        for _ in range(10):
-            for addr in ln_addrs:
-                sends[addr] = split
-            miner.sendmany("", sends)
-        # confirm funds in block 299
+        # Multiple UTXOs per LN wallet so multiple channels can be opened per block
+        miner_balance = int(miner.getbalance())
+        # To reduce individual TX weight, consolidate all outputs before distribution
+        miner.sendtoaddress(miner_addr, miner_balance - 1)
         gen(1)
+        helicopter = CTransaction()
 
+        # Provide the source LN node for each channel with a UTXO just big enough
+        # to open that channel with its capacity plus fee.
+        channel_openers = []
+        for ch in self.channels:
+            if ch["source"] not in channel_openers:
+                channel_openers.append(ch["source"])
+            addr = ln_addrs[ch["source"]]
+            # More than enough to open the channel plus fee and cover LND's "maxFeeRatio"
+            # As long as all channel capacities are < 4 BTC the change output will be
+            # larger and occupy tx output 1, leaving the actual channel open at output 0
+            sat_amt = 10 * COIN
+            helicopter.vout.append(CTxOut(sat_amt, address_to_scriptpubkey(addr)))
+        rawtx = miner.fundrawtransaction(helicopter.serialize().hex())
+        signed_tx = miner.signrawtransactionwithwallet(rawtx['hex'])['hex']
+        txid = miner.sendrawtransaction(signed_tx)
+        # confirm funds in last block before channel opens
+        gen(1)
+
+        txstats = miner.gettransaction(txid)
         self.log.info(
-            f"Waiting for funds to be spendable: 10x{split} BTC UTXOs each for {len(ln_addrs)} LN nodes"
+            "Funds distribution from miner:\n  "
+            + f"txid: {txid}\n  "
+            + f"# outputs: {len(txstats['details'])}\n  "
+            + f"total amount: {txstats['amount']}\n  "
+            + f"remaining miner balance: {miner.getbalance()}"
         )
 
-        def confirm_ln_balance(self, ln):
+        self.log.info("Waiting for funds to be spendable by channel-openers")
+
+        def confirm_ln_balance(self, ln_name):
+            ln = self.lns[ln_name]
             while True:
                 try:
                     bal = ln.walletbalance()
-                    if bal >= (split * 100000000):
-                        self.log.info(f"LN node {ln.name} confirmed funds")
+                    if bal >= 0:
+                        self.log.info(f"LN node {ln_name} confirmed funds")
                         break
                     else:
-                        self.log.info(f"Got balance from {ln.name} but less than expected, retrying in 5 seconds...")
+                        self.log.info(f"Got 0 balance from {ln_name} retrying in 5 seconds...")
                         sleep(5)
                 except Exception as e:
-                    self.log.info(f"Couldn't get balance from {ln.name} because {e}, retrying in 5 seconds...")
+                    self.log.info(f"Couldn't get balance from {ln_name} because {e}, retrying in 5 seconds...")
                     sleep(5)
 
         fund_threads = [
-            threading.Thread(target=confirm_ln_balance, args=(self, ln)) for ln in self.lns.values()
+            threading.Thread(target=confirm_ln_balance, args=(self, ln_name)) for ln_name in channel_openers
         ]
         for thread in fund_threads:
             thread.start()
 
         all(thread.join() is None for thread in fund_threads)
-        self.log.info("All LN nodes are funded")
+        self.log.info("All channel-opening LN nodes are funded")
 
         ##
         # URIs
@@ -247,12 +285,14 @@ def open_channel(self, ch, fee_rate):
                         sleep(5)
 
             channels = sorted(ch_by_block[target_block], key=lambda ch: ch["id"]["index"])
+            if len(channels) > CHANNEL_OPENS_PER_BLOCK:
+                raise Exception(f"Too many channels in block {target_block}: {len(channels)} / Maximum: {CHANNEL_OPENS_PER_BLOCK}")
             index = 0
-            fee_rate = 5006  # s/vB, decreases by 20 per tx for up to 250 txs per block
+            fee_rate = MAX_FEE_RATE
             ch_threads = []
             for ch in channels:
                 index += 1  # noqa
-                fee_rate -= 20
+                fee_rate -= FEE_RATE_DECREMENT
                 assert index == ch["id"]["index"], "Channel ID indexes are not consecutive"
                 assert fee_rate >= 1, "Too many TXs in block, out of fee range"
                 t = threading.Thread(target=open_channel, args=(self, ch, fee_rate))

src/warnet/graph.py

@@ -11,7 +11,11 @@
 from rich.console import Console
 from rich.table import Table
 
-from resources.scenarios.ln_framework.ln import Policy
+from resources.scenarios.ln_framework.ln import (
+    Policy,
+    CHANNEL_OPEN_START_HEIGHT,
+    CHANNEL_OPENS_PER_BLOCK
+)
 
 from .constants import (
     DEFAULT_IMAGE_REPO,
@@ -339,8 +343,8 @@ def _import_network(graph_file_path, output_path):
 
     sorted_edges = sorted(graph["edges"], key=lambda x: int(x["channel_id"]))
 
-    # By default we start including channel open txs in block 300
-    block = 300
+    # Start including channel open txs at this block height
+    block = CHANNEL_OPEN_START_HEIGHT
     # Coinbase occupies the 0 position!
     index = 1
     count = 0
@@ -356,7 +360,7 @@ def _import_network(graph_file_path, output_path):
         }
         tanks[source]["lnd"]["channels"].append(channel)
         index += 1
-        if index > 250:
+        if index > CHANNEL_OPENS_PER_BLOCK:
             index = 1
             block += 1
         count += 1