Integrate VLSD into integration test framework

Author: ksedgwic

Makefile

@@ -412,6 +412,10 @@ else
 PYTEST_OPTS += -x
 endif
 
+ifneq ($(PYTEST_MOREOPTS),)
+PYTEST_OPTS += $(PYTEST_MOREOPTS)
+endif
+
 check-units:
 
 check: check-units installcheck check-protos pytest

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -91,7 +91,7 @@ def directory(request, test_base_dir, test_name):
     outcome = 'passed' if rep_call is None else rep_call.outcome
     failed = not outcome or request.node.has_errors or outcome != 'passed'
 
-    if not failed:
+    if not failed and not bool(int(os.getenv('TEST_KEEPDIR', '0'))):
         try:
             shutil.rmtree(directory)
         except (OSError, Exception):

contrib/pyln-testing/pyln/testing/utils.py

@@ -79,6 +79,7 @@ def env(name, default=None):
 SLOW_MACHINE = env("SLOW_MACHINE", "0") == "1"
 DEPRECATED_APIS = env("DEPRECATED_APIS", "0") == "1"
 TIMEOUT = int(env("TIMEOUT", 180 if SLOW_MACHINE else 60))
+SUBDAEMON = env("SUBDAEMON", "")
 EXPERIMENTAL_DUAL_FUND = env("EXPERIMENTAL_DUAL_FUND", "0") == "1"
 
 
@@ -529,6 +530,40 @@ def getnewaddress(self):
         return info['unconfidential']
 
 
+class ValidatingLightningSignerD(TailableProc):
+    def __init__(self, vlsd_dir, vlsd_port):
+        TailableProc.__init__(self, vlsd_dir)
+        self.executable = env("REMOTE_SIGNER_CMD", 'vlsd')
+        self.opts = [
+            '--log-level-console=DEBUG',
+            '--log-level-disk=DEBUG',
+            '--network={}'.format(TEST_NETWORK),
+            '--datadir={}'.format(vlsd_dir),
+            '--port={}'.format(vlsd_port),
+            '--initial-allowlist-file={}'.format(env('REMOTE_SIGNER_ALLOWLIST',
+                                                     'contrib/remote_hsmd/TESTING_ALLOWLIST')),
+        ]
+        self.prefix = 'vlsd'
+        self.vlsd_port = vlsd_port
+
+    @property
+    def cmd_line(self):
+        return [self.executable] + self.opts
+
+    def start(self, stdin=None, stdout=None, stderr=None,
+              wait_for_initialized=True):
+        TailableProc.start(self, stdin, stdout, stderr)
+        # We need to always wait for initialization
+        self.wait_for_log("vlsd [0-9]* ready on .*:{}".format(self.vlsd_port))
+        logging.info("vlsd started")
+
+    def stop(self, timeout=10):
+        logging.info("stopping vlsd")
+        rc = TailableProc.stop(self, timeout)
+        logging.info("vlsd stopped")
+        return rc
+
+
 class LightningD(TailableProc):
     def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, node_id=0):
         TailableProc.__init__(self, lightning_dir)
@@ -537,6 +572,9 @@ def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, no
         self.port = port
         self.cmd_prefix = []
         self.disconnect_file = None
+        self.lightning_dir = lightning_dir
+        self.use_vlsd = False;
+        self.vlsd_dir = os.path.join(lightning_dir, "vlsd")
 
         self.rpcproxy = bitcoindproxy
 
@@ -555,12 +593,21 @@ def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, no
             'bitcoin-datadir': lightning_dir,
         }
 
+        if SUBDAEMON:
+            opts['subdaemon'] = SUBDAEMON
+            if SUBDAEMON == 'hsmd:remote_hsmd':
+                self.use_vlsd = True
+                
         for k, v in opts.items():
             self.opts[k] = v
 
         if not os.path.exists(os.path.join(lightning_dir, TEST_NETWORK)):
             os.makedirs(os.path.join(lightning_dir, TEST_NETWORK))
 
+        if self.use_vlsd:
+            if not os.path.exists(self.vlsd_dir):
+                os.makedirs(self.vlsd_dir)
+
         # Last 32-bytes of final part of dir -> seed.
         seed = (bytes(re.search('([^/]+)/*$', lightning_dir).group(1), encoding='utf-8') + bytes(32))[:32]
         if not random_hsm:
@@ -572,6 +619,11 @@ def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, no
         self.prefix = 'lightningd-%d' % (node_id)
 
     def cleanup(self):
+        if self.use_vlsd:
+            if self.use_vlsd:
+                # Make sure the remotesigner is shutdown
+                self.vlsd.stop()
+
         # To force blackhole to exit, disconnect file must be truncated!
         if self.disconnect_file:
             with open(self.disconnect_file, "w") as f:
@@ -594,11 +646,38 @@ def cmd_line(self):
 
     def start(self, stdin=None, stdout=None, stderr=None,
               wait_for_initialized=True):
-        self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport
-        TailableProc.start(self, stdin, stdout, stderr)
-        if wait_for_initialized:
-            self.wait_for_log("Server started with public key")
-        logging.info("LightningD started")
+        try:
+            if self.use_vlsd:
+                # Start the remote signer first
+                vlsd_port = reserve()
+                self.vlsd = ValidatingLightningSignerD(self.vlsd_dir, vlsd_port)
+                self.vlsd.start(stdin, stdout, stderr, wait_for_initialized)
+
+                # We can't do this in the constructor because we need a new port on each restart.
+                self.env['REMOTE_HSMD_ENDPOINT'] = '127.0.0.1:{}'.format(vlsd_port)
+
+            self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport
+            TailableProc.start(self, stdin, stdout, stderr)
+            if wait_for_initialized:
+                self.wait_for_log("Server started with public key")
+            logging.info("LightningD started")
+        except Exception:
+            if self.use_vlsd:
+                # LightningD didn't start, stop the remotesigner
+                self.vlsd.stop()
+            raise
+
+    def stop(self, timeout=10):
+        if self.use_vlsd:
+            # Stop the remote signer first
+            self.vlsd.stop(timeout)
+        return TailableProc.stop(self, timeout)
+
+    def kill(self):
+        if self.use_vlsd:
+            # Kill the remote signer first
+            self.vlsd.kill()
+        TailableProc.kill(self)
 
     def wait(self, timeout=10):
         """Wait for the daemon to stop for up to timeout seconds

contrib/remote_hsmd/TESTING_ALLOWLIST

@@ -0,0 +1,41 @@
+bcrt1qeyyk6sl5pr49ycpqyckvmttus5ttj25pd0zpvg
+bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080
+bcrt1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qzf4jry
+bcrt1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k0ylj56
+bcrt1sw50qt2uwha
+bcrt1zw508d6qejxtdg4y5r3zarvaryv2wuatf
+bcrt1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvseswlauz7
+bcrt1pqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesyga46z
+bcrt1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqc8gma6
+bcrt1pqqqqrldkrl
+bcrt1pqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqwekeum
+bcrt1sqqqq4wstyw
+bcrt1sqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq8v496j
+bcrt1zqqqq0u8uvu
+bcrt1zqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqla7vv2
+bcrt1rqqqqtap6fa
+bcrt1rqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqsxpcm9
+bcrt1yqqqqh6ngj6
+bcrt1yqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq54w09p
+bcrt19qqqqnm4whm
+bcrt19qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqmw3mjw
+bcrt1xqqqqlclycc
+bcrt1xqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq22ewzl
+bcrt18qqqqmeezae
+bcrt18qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq93x64s
+bcrt1gqqqqwkjf8k
+bcrt1gqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqz98fhh
+bcrt1fqqqq2h50zh
+bcrt1fqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqd7caqc
+bcrt12qqqqx579d5
+bcrt12qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqu6sgsf
+bcrt1tqqqqz4crg4
+bcrt1tqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqnp0u8x
+bcrt1vqqqq7j23nj
+bcrt1vqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhjqtez
+bcrt1dqqqq6nvhkn
+bcrt1dqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqcfllwd
+bcrt1wqqqqksxaes
+bcrt1wqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfdh27u
+bcrt10qqqqj3qmu3
+bcrt10qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqxkg7fn

contrib/remote_hsmd/scripts/cleanup

@@ -0,0 +1,10 @@
+#!/usr/bin/sh
+
+rm -rf /tmp/ltests-*
+rm -rf /tmp/lnpt-cl-*
+rm -rf /tmp/tmp*
+rm -rf /tmp/.tmp*
+rm -rf /tmp/vgdb-pipe-*
+rm -rf /tmp/git-difftool.*
+killall bitcoind
+killall vlsd

contrib/remote_hsmd/scripts/rerun-failed-tests

@@ -0,0 +1,23 @@
+#!/usr/bin/sh
+
+echo "accepts log w/ failures on stdin"
+
+TESTS=`awk '/^FAILED tests/ {print $2}'`
+
+# scale the parallelism to number of processors
+NPROC=$(nproc)
+TPAR=$((NPROC * 2))
+
+PYTHONPATH=\
+$PWD/contrib/pyln-client:\
+$PWD/contrib/pyln-testing:\
+$PWD/contrib/pyln-proto \
+TEST_DEBUG=1 \
+DEVELOPER=1 \
+VALGRIND=0 \
+SUBDAEMON='hsmd:remote_hsmd' \
+REMOTE_SIGNER_CMD=$(pwd)/../validating-lightning-signer/target/debug/vlsd \
+REMOTE_SIGNER_ALLOWLIST=$(pwd)/contrib/remote_hsmd/TESTING_ALLOWLIST \
+pytest \
+$TESTS \
+-n=$TPAR --timeout=300 --timeout_method=thread -v

contrib/remote_hsmd/scripts/run-all-tests

@@ -0,0 +1,17 @@
+#!/usr/bin/sh
+
+# scale the parallelism to number of processors
+JPAR=$(nproc)
+TPAR=$((JPAR * 2))
+
+SUBDAEMON='hsmd:remote_hsmd' \
+REMOTE_SIGNER_CMD=$(pwd)/../validating-lightning-signer/target/debug/vlsd \
+REMOTE_SIGNER_ALLOWLIST=$(pwd)/contrib/remote_hsmd/TESTING_ALLOWLIST \
+make \
+-j$JPAR PYTEST_PAR=$TPAR \
+DEVELOPER=1 \
+VALGRIND=0 \
+SLOW_MACHINE=0 \
+PYTEST_MOREOPTS='--timeout=300 --timeout_method=thread' \
+pytest
+

contrib/remote_hsmd/scripts/run-one-test

@@ -0,0 +1,19 @@
+#!/usr/bin/sh
+
+THETEST=$1
+
+PYTHONPATH=\
+$PWD/contrib/pyln-client:\
+$PWD/contrib/pyln-testing:\
+$PWD/contrib/pyln-proto \
+TEST_KEEPDIR=1 \
+TEST_DEBUG=1 \
+DEVELOPER=1 \
+VALGRIND=0 \
+SLOW_MACHINE=0 \
+SUBDAEMON='hsmd:remote_hsmd' \
+REMOTE_SIGNER_CMD=$(pwd)/../validating-lightning-signer/target/debug/vlsd \
+REMOTE_SIGNER_ALLOWLIST=$(pwd)/contrib/remote_hsmd/TESTING_ALLOWLIST \
+pytest \
+$THETEST \
+-v --timeout=300 --timeout_method=thread -x -s