ln: open rest api and ensure scenarios can open channels

Author: pinheadmz

resources/charts/bitcoincore/charts/lnd/templates/service.yaml

@@ -16,5 +16,9 @@ spec:
       targetPort: p2p
       protocol: TCP
       name: p2p
+    - port: {{ .Values.RestPort }}
+      targetPort: rest
+      protocol: TCP
+      name: rest
   selector:
     {{- include "lnd.selectorLabels" . | nindent 4 }}

resources/charts/bitcoincore/charts/lnd/values.yaml

@@ -115,6 +115,7 @@ affinity: {}
 
 baseConfig: |
   norest=false
+  restlisten=0.0.0.0:8080
   debuglevel=debug
   accept-keysend=true
   bitcoin.active=true

resources/scenarios/commander.py

@@ -1,11 +1,14 @@
 import argparse
+import base64
 import configparser
+import http.client
 import json
 import logging
 import os
 import pathlib
 import random
 import signal
+import ssl
 import sys
 import tempfile
 from typing import Dict
@@ -22,6 +25,13 @@
 
 WARNET_FILE = "/shared/warnet.json"
 
+# hard-coded deterministic lnd credentials
+ADMIN_MACAROON_HEX = "0201036c6e6402f801030a1062beabbf2a614b112128afa0c0b4fdd61201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e657261746512047265616400000620b17be53e367290871681055d0de15587f6d1cd47d1248fe2662ae27f62cfbdc6"
+# Don't worry about lnd's self-signed certificates
+INSECURE_CONTEXT = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+INSECURE_CONTEXT.check_hostname = False
+INSECURE_CONTEXT.verify_mode = ssl.CERT_NONE
+
 try:
     with open(WARNET_FILE) as file:
         WARNET = json.load(file)
@@ -39,6 +49,45 @@ def auth_proxy_request(self, method, path, postdata):
 AuthServiceProxy._request = auth_proxy_request
 
 
+class LND:
+    def __init__(self, tank_name):
+        self.conn = http.client.HTTPSConnection(
+            host=f"{tank_name}-ln", port=8080, timeout=5, context=INSECURE_CONTEXT
+        )
+
+    def get(self, uri):
+        self.conn.request(
+            method="GET", url=uri, headers={"Grpc-Metadata-macaroon": ADMIN_MACAROON_HEX}
+        )
+        return self.conn.getresponse().read().decode("utf8")
+
+    def post(self, uri, data):
+        body = json.dumps(data)
+        self.conn.request(
+            method="POST",
+            url=uri,
+            body=body,
+            headers={
+                "Content-Type": "application/json",
+                "Content-Length": str(len(body)),
+                "Grpc-Metadata-macaroon": ADMIN_MACAROON_HEX,
+            },
+        )
+        # Stream output, otherwise we get a timeout error
+        res = self.conn.getresponse()
+        stream = ""
+        while True:
+            try:
+                data = res.read(1)
+                if len(data) == 0:
+                    break
+                else:
+                    stream += data.decode("utf8")
+            except Exception:
+                break
+        return stream
+
+
 class Commander(BitcoinTestFramework):
     # required by subclasses of BitcoinTestFramework
     def set_test_params(self):
@@ -55,6 +104,10 @@ def ensure_miner(node):
             node.createwallet("miner", descriptors=True)
         return node.get_wallet_rpc("miner")
 
+    @staticmethod
+    def hex_to_b64(hex):
+        return base64.b64encode(bytes.fromhex(hex)).decode()
+
     def handle_sigterm(self, signum, frame):
         print("SIGTERM received, stopping...")
         self.shutdown()
@@ -108,6 +161,12 @@ def setup(self):
             )
             node.rpc_connected = True
             node.init_peers = tank["init_peers"]
+
+            # Tank might not even have an ln node, that's
+            # not our problem, it'll just 404 if scenario tries
+            # to connect to it
+            node.lnd = LND(tank["tank"])
+
             self.nodes.append(node)
             self.tanks[tank["tank"]] = node
 

resources/scenarios/test_scenarios/ln_basic.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+import json
+
+from commander import Commander
+
+
+class LNBasic(Commander):
+    def set_test_params(self):
+        self.num_nodes = None
+
+    def add_options(self, parser):
+        parser.description = "Open a channel between two LN nodes using REST + macaroon"
+        parser.usage = "warnet run /path/to/ln_init.py"
+
+    def run_test(self):
+        info = json.loads(self.tanks["tank-0003"].lnd.get("/v1/getinfo"))
+        uri = info["uris"][0]
+        pk3, host = uri.split("@")
+
+        print(
+            self.tanks["tank-0002"].lnd.post(
+                "/v1/peers", data={"addr": {"pubkey": pk3, "host": host}}
+            )
+        )
+
+        print(
+            self.tanks["tank-0002"].lnd.post(
+                "/v1/channels/stream",
+                data={"local_funding_amount": 100000, "node_pubkey": self.hex_to_b64(pk3)},
+            )
+        )
+
+        # Mine it ourself
+        self.wait_until(lambda: self.tanks["tank-0002"].getmempoolinfo()["size"] == 1)
+        print(self.tanks["tank-0002"].generate(5, invalid_call=False))
+
+
+def main():
+    LNBasic().main()
+
+
+if __name__ == "__main__":
+    main()

test/data/ln/network.yaml

@@ -51,11 +51,4 @@ nodes:
     addnode:
       - tank-0000
     ln:
-      lnd: true
-    lnd:
-      channels:
-        - id:
-            block: 301
-            index: 1
-          target: tank-0000-ln
-          local_amt: 25000
+      lnd: true

test/ln_basic_test.py

@@ -12,6 +12,7 @@ class LNBasicTest(TestBase):
     def __init__(self):
         super().__init__()
         self.network_dir = Path(os.path.dirname(__file__)) / "data" / "ln"
+        self.scen_dir = Path(os.path.dirname(__file__)).parent / "resources" / "scenarios"
         self.lns = [
             "tank-0000-ln",
             "tank-0001-ln",
@@ -29,16 +30,21 @@ def run_test(self):
             self.fund_wallets()
 
             # Manually open two channels between first three nodes
-            # and send a payment
+            # and send a payment using warnet RPC
             self.manual_open_channels()
             self.wait_for_gossip_sync(self.lns[:3], 2)
             self.pay_invoice(sender="tank-0000-ln", recipient="tank-0002-ln")
 
-            # Automatically open channels from network.yaml
+            # Automatically open channels from network.yaml using warnet RPC
             self.automatic_open_channels()
-            self.wait_for_gossip_sync(self.lns[3:], 3)
+            self.wait_for_gossip_sync(self.lns[3:], 2)
             # push_amt should enable payments from target to source
             self.pay_invoice(sender="tank-0005-ln", recipient="tank-0003-ln")
+
+            # Automatically open channels from inside a scenario commander
+            self.scenario_open_channels()
+            self.pay_invoice(sender="tank-0002-ln", recipient="tank-0003-ln")
+
         finally:
             self.cleanup()
 
@@ -75,6 +81,11 @@ def fund_wallets(self):
         self.warnet("bitcoin rpc tank-0000 sendmany '' '{" + outputs + "}'")
         self.warnet("bitcoin rpc tank-0000 -generate 1")
 
+    def wait_for_two_txs(self):
+        self.wait_for_predicate(
+            lambda: json.loads(self.warnet("bitcoin rpc tank-0000 getmempoolinfo"))["size"] == 2
+        )
+
     def manual_open_channels(self):
         # 0 -> 1 -> 2
         pk1 = self.warnet("ln pubkey tank-0001-ln")
@@ -101,10 +112,7 @@ def manual_open_channels(self):
             )
         )
 
-        def wait_for_two_txs():
-            return json.loads(self.warnet("bitcoin rpc tank-0000 getmempoolinfo"))["size"] == 2
-
-        self.wait_for_predicate(wait_for_two_txs)
+        self.wait_for_two_txs()
 
         self.warnet("bitcoin rpc tank-0000 -generate 10")
 
@@ -131,14 +139,20 @@ def wait_for_success():
         self.wait_for_predicate(wait_for_success)
 
     def automatic_open_channels(self):
+        # 3 -> 4 -> 5
         self.warnet("ln open-all-channels")
 
-        def wait_for_three_txs():
-            return json.loads(self.warnet("bitcoin rpc tank-0000 getmempoolinfo"))["size"] == 3
+        self.wait_for_two_txs()
 
-        self.wait_for_predicate(wait_for_three_txs)
         self.warnet("bitcoin rpc tank-0000 -generate 10")
 
+    def scenario_open_channels(self):
+        # 2 -> 3
+        # connecting all six ln nodes in the graph
+        scenario_file = self.scen_dir / "test_scenarios" / "ln_basic.py"
+        self.log.info(f"Running scenario from: {scenario_file}")
+        self.warnet(f"run {scenario_file} --source_dir={self.scen_dir} --debug")
+
 
 if __name__ == "__main__":
     test = LNBasicTest()