Merge bitcoin/bitcoin#27452: test: cover addrv2 anchors by adding TorV3 to CAddress in messages.py

ba8ab4f test: cover addrv2 support in anchors.dat with a TorV3 address (Matthew Zipkin)
b4bee4b test: add keep_alive option to socks5 proxy in test_framework (Matthew Zipkin)
5aaf988 test: cover TorV3 address in p2p_addrv2_relay (Matthew Zipkin)
80f64a3 test: add support for all networks in CAddress in messages.py (brunoerg)

Pull request description:

  Closes bitcoin/bitcoin#27140

  Adds test coverage for bitcoin/bitcoin#20516 to ensure that bitcoin/bitcoin#20511 is completed and may be closed.

  This PR adds a test case to `feature_anchors.py` where an onion v3 address is set as a blocks-only relay peer and then shutdown, ensuring that the address is saved to anchors.dat in addrv2 format. We then ensure that bitcoin attempts to reconnect to that anchor address on restart.

  To compute the addrv2 serialization of the onion v3 address, I added logic to `CAddress` in `messages.py`. This new logic is covered by extending `p2p_addrv2_relay.py` to include an onion v3 address. Future work will be adding coverage for ipv6, torv2 and cjdns in these modules and also `feature_proxy.py`

  Also includes de/serialization unit test for `CAddress` in test framework.

ACKs for top commit:
  jonatack:
    ACK ba8ab4f
  brunoerg:
    crACK ba8ab4f
  willcl-ark:
    ACK ba8ab4f

Tree-SHA512: 7220e30d7cb975903d9ac575a7215a08e8f784c24c5741561affcbde12fb92cbf8704cb42e66494b788ba6ed4bb255fb0cc327e4f2190fae50c0ed9f336c0ff0

Author: fanquake

test/functional/feature_anchors.py

@@ -6,12 +6,15 @@
 
 import os
 
-from test_framework.p2p import P2PInterface
+from test_framework.p2p import P2PInterface, P2P_SERVICES
+from test_framework.socks5 import Socks5Configuration, Socks5Server
+from test_framework.messages import CAddress, hash256
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import check_node_connections
+from test_framework.util import check_node_connections, assert_equal, p2p_port
 
 INBOUND_CONNECTIONS = 5
 BLOCK_RELAY_CONNECTIONS = 2
+ONION_ADDR = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:8333"
 
 
 class AnchorsTest(BitcoinTestFramework):
@@ -54,7 +57,7 @@ def run_test(self):
             else:
                 inbound_nodes_port.append(hex(int(addr_split[1]))[2:])
 
-        self.log.info("Stop node 0")
+        self.log.debug("Stop node")
         self.stop_node(0)
 
         # It should contain only the block-relay-only addresses
@@ -78,12 +81,64 @@ def run_test(self):
                 tweaked_contents[20:20] = b'1'
                 out_file_handler.write(bytes(tweaked_contents))
 
-            self.log.info("Start node")
+            self.log.debug("Start node")
             self.start_node(0)
 
         self.log.info("When node starts, check if anchors.dat doesn't exist anymore")
         assert not os.path.exists(node_anchors_path)
 
+        self.log.info("Ensure addrv2 support")
+        # Use proxies to catch outbound connections to networks with 256-bit addresses
+        onion_conf = Socks5Configuration()
+        onion_conf.auth = True
+        onion_conf.unauth = True
+        onion_conf.addr = ('127.0.0.1', p2p_port(self.num_nodes))
+        onion_conf.keep_alive = True
+        onion_proxy = Socks5Server(onion_conf)
+        onion_proxy.start()
+        self.restart_node(0, extra_args=[f"-onion={onion_conf.addr[0]}:{onion_conf.addr[1]}"])
+
+        self.log.info("Add 256-bit-address block-relay-only connections to node")
+        self.nodes[0].addconnection(ONION_ADDR, 'block-relay-only')
+
+        self.log.debug("Stop node")
+        with self.nodes[0].assert_debug_log([f"DumpAnchors: Flush 1 outbound block-relay-only peer addresses to anchors.dat"]):
+            self.stop_node(0)
+        # Manually close keep_alive proxy connection
+        onion_proxy.stop()
+
+        self.log.info("Check for addrv2 addresses in anchors.dat")
+        caddr = CAddress()
+        caddr.net = CAddress.NET_TORV3
+        caddr.ip, port_str = ONION_ADDR.split(":")
+        caddr.port = int(port_str)
+        # TorV3 addrv2 serialization:
+        # time(4) | services(1) | networkID(1) | address length(1) | address(32)
+        expected_pubkey = caddr.serialize_v2()[7:39].hex()
+
+        # position of services byte of first addr in anchors.dat
+        # network magic, vector length, version, nTime
+        services_index = 4 + 1 + 4 + 4
+        data = bytes()
+        with open(node_anchors_path, "rb") as file_handler:
+            data = file_handler.read()
+            assert_equal(data[services_index], 0x00)  # services == NONE
+            anchors2 = data.hex()
+            assert expected_pubkey in anchors2
+
+        with open(node_anchors_path, "wb") as file_handler:
+            # Modify service flags for this address even though we never connected to it.
+            # This is necessary because on restart we will not attempt an anchor connection
+            # to a host without our required services, even if its address is in the anchors.dat file
+            new_data = bytearray(data)[:-32]
+            new_data[services_index] = P2P_SERVICES
+            new_data_hash = hash256(new_data)
+            file_handler.write(new_data + new_data_hash)
+
+        self.log.info("Restarting node attempts to reconnect to anchors")
+        with self.nodes[0].assert_debug_log([f"Trying to make an anchor connection to {ONION_ADDR}"]):
+            self.start_node(0, extra_args=[f"-onion={onion_conf.addr[0]}:{onion_conf.addr[1]}"])
+
 
 if __name__ == "__main__":
     AnchorsTest().main()

test/functional/p2p_addrv2_relay.py

@@ -20,19 +20,24 @@
 from test_framework.util import assert_equal
 
 I2P_ADDR = "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p"
+ONION_ADDR = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"
 
 ADDRS = []
 for i in range(10):
     addr = CAddress()
     addr.time = int(time.time()) + i
+    addr.port = 8333 + i
     addr.nServices = P2P_SERVICES
-    # Add one I2P address at an arbitrary position.
+    # Add one I2P and one onion V3 address at an arbitrary position.
     if i == 5:
         addr.net = addr.NET_I2P
         addr.ip = I2P_ADDR
+        addr.port = 0
+    elif i == 8:
+        addr.net = addr.NET_TORV3
+        addr.ip = ONION_ADDR
     else:
         addr.ip = f"123.123.123.{i % 256}"
-    addr.port = 8333 + i
     ADDRS.append(addr)
 
 
@@ -52,6 +57,17 @@ def wait_for_addrv2(self):
         self.wait_until(lambda: "addrv2" in self.last_message)
 
 
+def calc_addrv2_msg_size(addrs):
+    size = 1  # vector length byte
+    for addr in addrs:
+        size += 4  # time
+        size += 1  # services, COMPACTSIZE(P2P_SERVICES)
+        size += 1  # network id
+        size += 1  # address length byte
+        size += addr.ADDRV2_ADDRESS_LENGTH[addr.net]  # address
+        size += 2  # port
+    return size
+
 class AddrTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
@@ -71,9 +87,10 @@ def run_test(self):
         self.log.info('Check that addrv2 message content is relayed and added to addrman')
         addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver())
         msg.addrs = ADDRS
+        msg_size = calc_addrv2_msg_size(ADDRS)
         with self.nodes[0].assert_debug_log([
-                'received: addrv2 (159 bytes) peer=0',
-                'sending addrv2 (159 bytes) peer=1',
+                f'received: addrv2 ({msg_size} bytes) peer=0',
+                f'sending addrv2 ({msg_size} bytes) peer=1',
         ]):
             addr_source.send_and_ping(msg)
             self.nodes[0].setmocktime(int(time.time()) + 30 * 60)

test/functional/test_framework/messages.py

@@ -27,6 +27,7 @@
 import socket
 import struct
 import time
+import unittest
 
 from test_framework.siphash import siphash256
 from test_framework.util import assert_equal
@@ -77,6 +78,10 @@ def sha256(s):
     return hashlib.sha256(s).digest()
 
 
+def sha3(s):
+    return hashlib.sha3_256(s).digest()
+
+
 def hash256(s):
     return sha256(sha256(s))
 
@@ -229,16 +234,25 @@ class CAddress:
 
     # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki
     NET_IPV4 = 1
+    NET_IPV6 = 2
+    NET_TORV3 = 4
     NET_I2P = 5
+    NET_CJDNS = 6
 
     ADDRV2_NET_NAME = {
         NET_IPV4: "IPv4",
-        NET_I2P: "I2P"
+        NET_IPV6: "IPv6",
+        NET_TORV3: "TorV3",
+        NET_I2P: "I2P",
+        NET_CJDNS: "CJDNS"
     }
 
     ADDRV2_ADDRESS_LENGTH = {
         NET_IPV4: 4,
-        NET_I2P: 32
+        NET_IPV6: 16,
+        NET_TORV3: 32,
+        NET_I2P: 32,
+        NET_CJDNS: 16
     }
 
     I2P_PAD = "===="
@@ -285,33 +299,54 @@ def deserialize_v2(self, f):
         self.nServices = deser_compact_size(f)
 
         self.net = struct.unpack("B", f.read(1))[0]
-        assert self.net in (self.NET_IPV4, self.NET_I2P)
+        assert self.net in self.ADDRV2_NET_NAME
 
         address_length = deser_compact_size(f)
         assert address_length == self.ADDRV2_ADDRESS_LENGTH[self.net]
 
         addr_bytes = f.read(address_length)
         if self.net == self.NET_IPV4:
             self.ip = socket.inet_ntoa(addr_bytes)
-        else:
+        elif self.net == self.NET_IPV6:
+            self.ip = socket.inet_ntop(socket.AF_INET6, addr_bytes)
+        elif self.net == self.NET_TORV3:
+            prefix = b".onion checksum"
+            version = bytes([3])
+            checksum = sha3(prefix + addr_bytes + version)[:2]
+            self.ip = b32encode(addr_bytes + checksum + version).decode("ascii").lower() + ".onion"
+        elif self.net == self.NET_I2P:
             self.ip = b32encode(addr_bytes)[0:-len(self.I2P_PAD)].decode("ascii").lower() + ".b32.i2p"
+        elif self.net == self.NET_CJDNS:
+            self.ip = socket.inet_ntop(socket.AF_INET6, addr_bytes)
+        else:
+            raise Exception(f"Address type not supported")
 
         self.port = struct.unpack(">H", f.read(2))[0]
 
     def serialize_v2(self):
         """Serialize in addrv2 format (BIP155)"""
-        assert self.net in (self.NET_IPV4, self.NET_I2P)
+        assert self.net in self.ADDRV2_NET_NAME
         r = b""
         r += struct.pack("<I", self.time)
         r += ser_compact_size(self.nServices)
         r += struct.pack("B", self.net)
         r += ser_compact_size(self.ADDRV2_ADDRESS_LENGTH[self.net])
         if self.net == self.NET_IPV4:
             r += socket.inet_aton(self.ip)
-        else:
+        elif self.net == self.NET_IPV6:
+            r += socket.inet_pton(socket.AF_INET6, self.ip)
+        elif self.net == self.NET_TORV3:
+            sfx = ".onion"
+            assert self.ip.endswith(sfx)
+            r += b32decode(self.ip[0:-len(sfx)], True)[0:32]
+        elif self.net == self.NET_I2P:
             sfx = ".b32.i2p"
             assert self.ip.endswith(sfx)
             r += b32decode(self.ip[0:-len(sfx)] + self.I2P_PAD, True)
+        elif self.net == self.NET_CJDNS:
+            r += socket.inet_pton(socket.AF_INET6, self.ip)
+        else:
+            raise Exception(f"Address type not supported")
         r += struct.pack(">H", self.port)
         return r
 
@@ -1852,3 +1887,19 @@ def serialize(self):
     def __repr__(self):
         return "msg_sendtxrcncl(version=%lu, salt=%lu)" %\
             (self.version, self.salt)
+
+class TestFrameworkScript(unittest.TestCase):
+    def test_addrv2_encode_decode(self):
+        def check_addrv2(ip, net):
+            addr = CAddress()
+            addr.net, addr.ip = net, ip
+            ser = addr.serialize_v2()
+            actual = CAddress()
+            actual.deserialize_v2(BytesIO(ser))
+            self.assertEqual(actual, addr)
+
+        check_addrv2("1.65.195.98", CAddress.NET_IPV4)
+        check_addrv2("2001:41f0::62:6974:636f:696e", CAddress.NET_IPV6)
+        check_addrv2("2bqghnldu6mcug4pikzprwhtjjnsyederctvci6klcwzepnjd46ikjyd.onion", CAddress.NET_TORV3)
+        check_addrv2("255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p", CAddress.NET_I2P)
+        check_addrv2("fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa", CAddress.NET_CJDNS)

test/functional/test_framework/socks5.py

@@ -40,6 +40,7 @@ def __init__(self):
         self.af = socket.AF_INET # Bind address family
         self.unauth = False  # Support unauthenticated
         self.auth = False  # Support authentication
+        self.keep_alive = False  # Do not automatically close connections
 
 class Socks5Command():
     """Information about an incoming socks5 command."""
@@ -115,13 +116,14 @@ def handle(self):
 
             cmdin = Socks5Command(cmd, atyp, addr, port, username, password)
             self.serv.queue.put(cmdin)
-            logger.info('Proxy: %s', cmdin)
+            logger.debug('Proxy: %s', cmdin)
             # Fall through to disconnect
         except Exception as e:
             logger.exception("socks5 request handling failed.")
             self.serv.queue.put(e)
         finally:
-            self.conn.close()
+            if not self.serv.keep_alive:
+                self.conn.close()
 
 class Socks5Server():
     def __init__(self, conf):
@@ -133,6 +135,7 @@ def __init__(self, conf):
         self.running = False
         self.thread = None
         self.queue = queue.Queue() # report connections and exceptions to client
+        self.keep_alive = conf.keep_alive
 
     def run(self):
         while self.running:
@@ -157,4 +160,3 @@ def stop(self):
         s.connect(self.conf.addr)
         s.close()
         self.thread.join()
-

test/functional/test_runner.py

@@ -76,6 +76,7 @@
     "blocktools",
     "ellswift",
     "key",
+    "messages",
     "muhash",
     "ripemd160",
     "script",