Merge pull request #51 from lightning-signer/2022-03-autoscale-test-runs

Autoscale remote-hsmd test script runs to machine size

Author: ksedgwic

contrib/remote_hsmd/NOTES.md

@@ -5,8 +5,7 @@ Setup
 
     # Add an "upstream" reference and fetch all tags (needed for pyln-{proto,client,testing})
     git remote add upstream [email protected]:ElementsProject/lightning.git
-    git fetch --all --recurse-submodules
-    git fetch --tags
+    git fetch upstream --tags
 
     (cd contrib/remote_hsmd && \
     ln -s ../../../validating-lightning-signer/lightning-signer-server/src/server/remotesigner.proto)

contrib/remote_hsmd/scripts/rerun-failed-tests

@@ -4,6 +4,10 @@ echo "accepts log w/ failures on stdin"
 
 TESTS=`awk '/^FAILED tests/ {print $2}'`
 
+# scale the parallelism to number of processors
+NPROC=$(nproc)
+TPAR=$((NPROC * 2))
+
 PYTHONPATH=\
 $PWD/contrib/pyln-client:\
 $PWD/contrib/pyln-testing:\
@@ -16,4 +20,4 @@ REMOTE_SIGNER_CMD=$(pwd)/../validating-lightning-signer/target/debug/server \
 REMOTE_SIGNER_ALLOWLIST=$(pwd)/contrib/remote_hsmd/TESTING_ALLOWLIST \
 pytest \
 $TESTS \
--n=32 --timeout=300 --timeout_method=thread -v
+-n=$TPAR --timeout=300 --timeout_method=thread -v

contrib/remote_hsmd/scripts/run-all-tests

@@ -1,10 +1,14 @@
 #!/usr/bin/sh
 
+# scale the parallelism to number of processors
+JPAR=$(nproc)
+TPAR=$((JPAR * 2))
+
 SUBDAEMON='hsmd:remote_hsmd' \
 REMOTE_SIGNER_CMD=$(pwd)/../validating-lightning-signer/target/debug/server \
 REMOTE_SIGNER_ALLOWLIST=$(pwd)/contrib/remote_hsmd/TESTING_ALLOWLIST \
 make \
--j16 PYTEST_PAR=32 \
+-j$JPAR PYTEST_PAR=$TPAR \
 DEVELOPER=1 \
 VALGRIND=0 \
 SLOW_MACHINE=0 \