Merge pull request #739 from pinheadmz/signet-miner

Support signet mining in scenarios

Author: pinheadmz

docker-bake.hcl

@@ -59,7 +59,7 @@ target "cmake-base" {
   inherits = ["maintained-base"]
   dockerfile = "./Dockerfile.dev"
   args = {
-    BUILD_ARGS = "-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON"
+    BUILD_ARGS = "-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON"
   }
 }
 

resources/charts/bitcoincore/templates/pod.yaml

@@ -65,6 +65,8 @@ spec:
         {{- toYaml .Values.readinessProbe | nindent 8 }}
         tcpSocket:
           port: {{ index .Values.global .Values.global.chain "RPCPort" }}
+      startupProbe:
+        {{- toYaml .Values.startupProbe | nindent 8 }}
       resources:
         {{- toYaml .Values.resources | nindent 8 }}
       volumeMounts:

resources/charts/commander/templates/rbac.yaml

@@ -44,7 +44,7 @@ metadata:
     app.kubernetes.io/name: {{ .Chart.Name }}
 rules:
   - apiGroups: [""]
-    resources: ["pods", "namespaces", "configmaps", "pods/log"]
+    resources: ["pods", "namespaces", "configmaps", "pods/log", "pods/exec"]
     verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1

resources/scenarios/commander.py

@@ -7,15 +7,42 @@
 import pathlib
 import random
 import signal
+import struct
 import sys
 import tempfile
 import threading
 from time import sleep
 
 from kubernetes import client, config
+from kubernetes.stream import stream
 from ln_framework.ln import CLN, LND, LNNode
 from test_framework.authproxy import AuthServiceProxy
+from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height
+from test_framework.messages import (
+    CBlock,
+    CBlockHeader,
+    COutPoint,
+    CTransaction,
+    CTxIn,
+    CTxInWitness,
+    CTxOut,
+    from_binary,
+    from_hex,
+    ser_string,
+    ser_uint256,
+    tx_from_hex,
+)
 from test_framework.p2p import NetworkThread
+from test_framework.psbt import (
+    PSBT,
+    PSBT_GLOBAL_UNSIGNED_TX,
+    PSBT_IN_FINAL_SCRIPTSIG,
+    PSBT_IN_FINAL_SCRIPTWITNESS,
+    PSBT_IN_NON_WITNESS_UTXO,
+    PSBT_IN_SIGHASH_TYPE,
+    PSBTMap,
+)
+from test_framework.script import CScriptOp
 from test_framework.test_framework import (
     TMPDIR_PREFIX,
     BitcoinTestFramework,
@@ -24,6 +51,11 @@
 from test_framework.test_node import TestNode
 from test_framework.util import PortSeed, get_rpc_proxy
 
+SIGNET_HEADER = b"\xec\xc7\xda\xa2"
+PSBT_SIGNET_BLOCK = (
+    b"\xfc\x06signetb"  # proprietary PSBT global field holding the block being signed
+)
+
 NAMESPACE = None
 pods = client.V1PodList(items=[])
 cmaps = client.V1ConfigMapList(items=[])
@@ -502,3 +534,144 @@ def connect_nodes(self, a, b, *, peer_advertises_v2=None, wait_for_connect: bool
             )
             == to_num_peers
         )
+
+    def generatetoaddress(self, generator, n, addr, sync_fun=None, **kwargs):
+        if generator.chain == "regtest":
+            blocks = generator.generatetoaddress(n, addr, invalid_call=False, **kwargs)
+            sync_fun() if sync_fun else self.sync_all()
+            return blocks
+        if generator.chain == "signet":
+            mined_blocks = 0
+            block_hashes = []
+
+            def bcli(method, *args, **kwargs):
+                return generator.__getattr__(method)(*args, **kwargs)
+
+            while mined_blocks < n:
+                # gbt
+                tmpl = bcli("getblocktemplate", {"rules": ["signet", "segwit"]})
+                # address for reward
+                reward_spk = bytes.fromhex(bcli("getaddressinfo", addr)["scriptPubKey"])
+                # create coinbase tx
+                cbtx = CTransaction()
+                cbtx.vin = [
+                    CTxIn(
+                        COutPoint(0, 0xFFFFFFFF),
+                        script_BIP34_coinbase_height(tmpl["height"]),
+                        0xFFFFFFFF,
+                    )
+                ]
+                cbtx.vout = [CTxOut(tmpl["coinbasevalue"], reward_spk)]
+                cbtx.vin[0].nSequence = 2**32 - 2
+                cbtx.rehash()
+                # assemble block
+                block = CBlock()
+                block.nVersion = tmpl["version"]
+                block.hashPrevBlock = int(tmpl["previousblockhash"], 16)
+                block.nTime = tmpl["curtime"]
+                if block.nTime < tmpl["mintime"]:
+                    block.nTime = tmpl["mintime"]
+                block.nBits = int(tmpl["bits"], 16)
+                block.nNonce = 0
+                block.vtx = [cbtx] + [tx_from_hex(t["data"]) for t in tmpl["transactions"]]
+                witnonce = 0
+                witroot = block.calc_witness_merkle_root()
+                cbwit = CTxInWitness()
+                cbwit.scriptWitness.stack = [ser_uint256(witnonce)]
+                block.vtx[0].wit.vtxinwit = [cbwit]
+                block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce))))
+                # create signet txs for signing
+                signet_spk = tmpl["signet_challenge"]
+                signet_spk_bin = bytes.fromhex(signet_spk)
+                txs = block.vtx[:]
+                txs[0] = CTransaction(txs[0])
+                txs[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER)
+                hashes = []
+                for tx in txs:
+                    tx.rehash()
+                    hashes.append(ser_uint256(tx.sha256))
+                mroot = block.get_merkle_root(hashes)
+                sd = b""
+                sd += struct.pack("<i", block.nVersion)
+                sd += ser_uint256(block.hashPrevBlock)
+                sd += ser_uint256(mroot)
+                sd += struct.pack("<I", block.nTime)
+                to_spend = CTransaction()
+                to_spend.nVersion = 0
+                to_spend.nLockTime = 0
+                to_spend.vin = [
+                    CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)
+                ]
+                to_spend.vout = [CTxOut(0, signet_spk_bin)]
+                to_spend.rehash()
+                spend = CTransaction()
+                spend.nVersion = 0
+                spend.nLockTime = 0
+                spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)]
+                spend.vout = [CTxOut(0, b"\x6a")]
+                # create PSBT for miner wallet signing
+                psbt = PSBT()
+                psbt.g = PSBTMap(
+                    {
+                        PSBT_GLOBAL_UNSIGNED_TX: spend.serialize(),
+                        PSBT_SIGNET_BLOCK: block.serialize(),
+                    }
+                )
+                psbt.i = [
+                    PSBTMap(
+                        {
+                            PSBT_IN_NON_WITNESS_UTXO: to_spend.serialize(),
+                            PSBT_IN_SIGHASH_TYPE: bytes([1, 0, 0, 0]),
+                        }
+                    )
+                ]
+                psbt.o = [PSBTMap()]
+                psbt = psbt.to_base64()
+                # sign PSBT
+                psbt_signed = bcli("walletprocesspsbt", psbt=psbt, sign=True, sighashtype="ALL")
+                if not psbt_signed.get("complete", False):
+                    self.log.error("PSBT signing failed, aborting...")
+                    return block_hashes
+                # decode signed PSBT
+                signed_psbt = PSBT.from_base64(psbt_signed["psbt"])
+                scriptSig = signed_psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"")
+                scriptWitness = signed_psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00")
+                signed_block = from_binary(CBlock, signed_psbt.g.map[PSBT_SIGNET_BLOCK])
+                signet_solution = ser_string(scriptSig) + scriptWitness
+                # finish block
+                signed_block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(
+                    SIGNET_HEADER + signet_solution
+                )
+                signed_block.vtx[0].rehash()
+                signed_block.hashMerkleRoot = signed_block.calc_merkle_root()
+                try:
+                    headhex = CBlockHeader.serialize(signed_block).hex()
+                    cmd = ["bitcoin-util", "grind", headhex]
+                    newheadhex = stream(
+                        sclient.connect_get_namespaced_pod_exec,
+                        name=generator.tank,
+                        namespace=NAMESPACE,
+                        command=cmd,
+                        stderr=True,
+                        stdin=False,
+                        stdout=True,
+                        tty=False,
+                    )
+                    if "not found" in newheadhex:
+                        raise Exception(newheadhex)
+                    newhead = from_hex(CBlockHeader(), newheadhex.strip())
+                    signed_block.nNonce = newhead.nNonce
+                    signed_block.rehash()
+                except Exception as e:
+                    self.log.info(
+                        f"Error grinding signet PoW with bitcoin-util in {generator.tank}: {e}".strip()
+                    )
+                    self.log.info("  re-attempting with a single python thread...")
+                    signed_block.solve()
+                # submit block
+                bcli("submitblock", signed_block.serialize().hex())
+                block_hashes.append(signed_block.hash)
+                mined_blocks += 1
+                self.log.info(f"Generated {mined_blocks} signet blocks")
+
+            return block_hashes

resources/scenarios/ln_init.py

@@ -26,6 +26,12 @@ def set_test_params(self):
     def add_options(self, parser):
         parser.description = "Fund LN wallets and open channels"
         parser.usage = "warnet run /path/to/ln_init.py"
+        parser.add_argument(
+            "--miner",
+            dest="miner",
+            type=str,
+            help="Select one tank by name as the blockchain miner",
+        )
 
     def run_test(self):
         ##
@@ -38,7 +44,18 @@ def run_test(self):
         # MINER
         ##
         self.log.info("Setting up miner...")
-        miner = self.ensure_miner(self.nodes[0])
+        if self.options.miner:
+            self.log.info(f"Parsed 'miner' argument: {self.options.miner}")
+            mining_tank = self.tanks[self.options.miner]
+        elif "miner" in self.tanks:
+            # or choose the tank with the right name
+            self.log.info("Found tank named 'miner'")
+            mining_tank = self.tanks["miner"]
+        else:
+            mining_tank = self.nodes[0]
+            self.log.info(f"Using tank {mining_tank.tank} as miner")
+
+        miner = self.ensure_miner(mining_tank)
         miner_addr = miner.getnewaddress()
 
         def gen(n):

resources/scenarios/test_scenarios/signet_grinder.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+from commander import Commander
+
+
+class SignetGrinder(Commander):
+    def set_test_params(self):
+        self.num_nodes = 0
+
+    def run_test(self):
+        self.generatetoaddress(self.tanks["miner"], 1, "tb1qjfplwf7a2dpjj04cx96rysqeastvycc0j50cch")
+
+
+def main():
+    SignetGrinder().main()
+
+
+if __name__ == "__main__":
+    main()

src/warnet/deploy.py

@@ -401,7 +401,7 @@ def deploy_network(directory: Path, debug: bool = False, namespace: Optional[str
             debug=False,
             source_dir=SCENARIOS_DIR,
             additional_args=None,
-            admin=False,
+            admin=True,
             namespace=namespace,
         )
         wait_for_pod_ready(name, namespace=namespace)

src/warnet/image_build.py

@@ -23,7 +23,7 @@ def build_image(
     action: str,
 ):
     if not build_args:
-        build_args = '"-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON "'
+        build_args = '"-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON "'
     else:
         build_args = f'"{build_args}"'
 

test/data/signet/network.yaml

@@ -1,5 +1,7 @@
 nodes:
   - name: miner
+    image:
+      tag: "29.0-util"
   - name: tank-1
     image:
       tag: "0.16.1"

test/signet_test.py

@@ -22,6 +22,7 @@ def run_test(self):
             self.setup_network()
             self.check_signet_miner()
             self.check_signet_recon()
+            self.check_signet_scenario_miner()
         finally:
             self.cleanup()
 
@@ -60,6 +61,28 @@ def check_scenario_clean_exit():
 
         self.wait_for_predicate(check_scenario_clean_exit)
 
+    def check_signet_scenario_miner(self):
+        before_count = int(self.warnet("bitcoin rpc tank-1 getblockcount"))
+
+        self.log.info("Generate 1 signet block from a scenario using the bitcoin-util grinder")
+        self.scen_dir = Path(os.path.dirname(__file__)).parent / "resources" / "scenarios"
+        scenario_file = self.scen_dir / "test_scenarios" / "signet_grinder.py"
+        self.log.info(f"Running scenario from: {scenario_file}")
+        self.warnet(f"run {scenario_file} --source_dir={self.scen_dir} --admin")
+        self.wait_for_all_scenarios()
+
+        after_count = int(self.warnet("bitcoin rpc tank-1 getblockcount"))
+        assert after_count - before_count == 1
+
+        deployed = scenarios_deployed()
+        found = False
+        for sc in deployed:
+            if "grinder" in sc["name"]:
+                found = True
+                log = self.warnet(f"logs {sc['name']}")
+                assert "Error grinding" not in log
+        assert found
+
 
 if __name__ == "__main__":
     test = SignetTest()