[functional test] orphan resolution works in the presence of DoSy peers

Co-authored-by: Greg Sanders <[email protected]>

Author: glozow

test/functional/p2p_opportunistic_1p1c.py

@@ -7,12 +7,20 @@
 """
 
 from decimal import Decimal
+import random
 import time
+
+from test_framework.blocktools import MAX_STANDARD_TX_WEIGHT
 from test_framework.mempool_util import (
+    create_large_orphan,
     fill_mempool,
 )
 from test_framework.messages import (
     CInv,
+    COutPoint,
+    CTransaction,
+    CTxIn,
+    CTxOut,
     CTxInWitness,
     MAX_BIP125_RBF_SEQUENCE,
     MSG_WTX,
@@ -21,12 +29,20 @@
     tx_from_hex,
 )
 from test_framework.p2p import (
+    NONPREF_PEER_TX_DELAY,
     P2PInterface,
+    TXID_RELAY_DELAY,
+)
+from test_framework.script import (
+    CScript,
+    OP_NOP,
+    OP_RETURN,
 )
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
+    assert_greater_than_or_equal,
 )
 from test_framework.wallet import (
     MiniWallet,
@@ -373,6 +389,164 @@ def test_other_parent_in_mempool(self):
         result_missing_parent = node.submitpackage(package_hex_missing_parent)
         assert_equal(result_missing_parent["package_msg"], "package-not-child-with-unconfirmed-parents")
 
+    def create_small_orphan(self):
+        """Create small orphan transaction"""
+        tx = CTransaction()
+        # Nonexistent UTXO
+        tx.vin = [CTxIn(COutPoint(random.randrange(1 << 256), random.randrange(1, 100)))]
+        tx.wit.vtxinwit = [CTxInWitness()]
+        tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_NOP] * 5)]
+        tx.vout = [CTxOut(100, CScript([OP_RETURN, b'a' * 3]))]
+        return tx
+
+    @cleanup
+    def test_orphanage_dos_large(self):
+        self.log.info("Test that the node can still resolve orphans when peers use lots of orphanage space")
+        node = self.nodes[0]
+        node.setmocktime(int(time.time()))
+
+        peer_normal = node.add_p2p_connection(P2PInterface())
+        peer_doser = node.add_p2p_connection(P2PInterface())
+
+        self.log.info("Create very large orphans to be sent by DoSy peers (may take a while)")
+        large_orphans = [create_large_orphan() for _ in range(100)]
+        # Check to make sure these are orphans, within max standard size (to be accepted into the orphanage)
+        for large_orphan in large_orphans:
+            assert_greater_than_or_equal(100000, large_orphan.get_vsize())
+            assert_greater_than(MAX_STANDARD_TX_WEIGHT, large_orphan.get_weight())
+            assert_greater_than_or_equal(3 * large_orphan.get_vsize(), 2 * 100000)
+            testres = node.testmempoolaccept([large_orphan.serialize().hex()])
+            assert not testres[0]["allowed"]
+            assert_equal(testres[0]["reject-reason"], "missing-inputs")
+
+        num_individual_dosers = 30
+        self.log.info(f"Connect {num_individual_dosers} peers and send a very large orphan from each one")
+        # This test assumes that unrequested transactions are processed (skipping inv and
+        # getdata steps because they require going through request delays)
+        # Connect 20 peers and have each of them send a large orphan.
+        for large_orphan in large_orphans[:num_individual_dosers]:
+            peer_doser_individual = node.add_p2p_connection(P2PInterface())
+            peer_doser_individual.send_and_ping(msg_tx(large_orphan))
+            node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
+            peer_doser_individual.wait_for_getdata([large_orphan.vin[0].prevout.hash])
+
+        # Make sure that these transactions are going through the orphan handling codepaths.
+        # Subsequent rounds will not wait for getdata because the time mocking will cause the
+        # normal package request to time out.
+        self.wait_until(lambda: len(node.getorphantxs()) == num_individual_dosers)
+
+        self.log.info("Send an orphan from a non-DoSy peer. Its orphan should not be evicted.")
+        low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet)
+        high_fee_child = self.wallet.create_self_transfer(
+            utxo_to_spend=low_fee_parent["new_utxo"],
+            fee_rate=200*FEERATE_1SAT_VB,
+            target_vsize=100000
+        )
+
+        # Announce
+        orphan_tx = high_fee_child["tx"]
+        orphan_inv = CInv(t=MSG_WTX, h=orphan_tx.wtxid_int)
+
+        # Wait for getdata
+        peer_normal.send_and_ping(msg_inv([orphan_inv]))
+        node.bumpmocktime(NONPREF_PEER_TX_DELAY)
+        peer_normal.wait_for_getdata([orphan_tx.wtxid_int])
+        peer_normal.send_and_ping(msg_tx(orphan_tx))
+
+        # Wait for parent request
+        parent_txid_int = int(low_fee_parent["txid"], 16)
+        node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
+        peer_normal.wait_for_getdata([parent_txid_int])
+
+        self.log.info("Send another round of very large orphans from a DoSy peer")
+        for large_orphan in large_orphans[30:]:
+            peer_doser.send_and_ping(msg_tx(large_orphan))
+
+        # Something was evicted; the orphanage does not contain all large orphans + the 1p1c child
+        self.wait_until(lambda: len(node.getorphantxs()) < len(large_orphans) + 1)
+
+        self.log.info("Provide the orphan's parent. This 1p1c package should be successfully accepted.")
+        peer_normal.send_and_ping(msg_tx(low_fee_parent["tx"]))
+        assert_equal(node.getmempoolentry(orphan_tx.txid_hex)["ancestorcount"], 2)
+
+    @cleanup
+    def test_orphanage_dos_many(self):
+        self.log.info("Test that the node can still resolve orphans when peers are sending tons of orphans")
+        node = self.nodes[0]
+        node.setmocktime(int(time.time()))
+
+        peer_normal = node.add_p2p_connection(P2PInterface())
+
+        # 2 sets of peers: the first set all send the same batch_size orphans. The second set each
+        # sends batch_size distinct orphans.
+        batch_size = 51
+        num_peers_shared = 60
+        num_peers_unique = 40
+
+        # 60 peers * 51 orphans = 3060 announcements
+        shared_orphans = [self.create_small_orphan() for _ in range(batch_size)]
+        self.log.info(f"Send the same {batch_size} orphans from {num_peers_shared} DoSy peers (may take a while)")
+        peer_doser_shared = [node.add_p2p_connection(P2PInterface()) for _ in range(num_peers_shared)]
+        for i in range(num_peers_shared):
+            for orphan in shared_orphans:
+                peer_doser_shared[i].send_without_ping(msg_tx(orphan))
+
+        # We sync peers to make sure we have processed as many orphans as possible. Ensure at least
+        # one of the orphans was processed.
+        for peer_doser in peer_doser_shared:
+            peer_doser.sync_with_ping()
+        self.wait_until(lambda: any([tx.txid_hex in node.getorphantxs() for tx in shared_orphans]))
+
+        self.log.info("Send an orphan from a non-DoSy peer. Its orphan should not be evicted.")
+        low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet)
+        high_fee_child = self.wallet.create_self_transfer(
+            utxo_to_spend=low_fee_parent["new_utxo"],
+            fee_rate=200*FEERATE_1SAT_VB,
+        )
+
+        # Announce
+        orphan_tx = high_fee_child["tx"]
+        orphan_inv = CInv(t=MSG_WTX, h=orphan_tx.wtxid_int)
+
+        # Wait for getdata
+        peer_normal.send_and_ping(msg_inv([orphan_inv]))
+        node.bumpmocktime(NONPREF_PEER_TX_DELAY)
+        peer_normal.wait_for_getdata([orphan_tx.wtxid_int])
+        peer_normal.send_and_ping(msg_tx(orphan_tx))
+
+        # Orphan has been entered and evicted something else
+        self.wait_until(lambda: high_fee_child["txid"] in node.getorphantxs())
+
+        # Wait for parent request
+        parent_txid_int = low_fee_parent["tx"].txid_int
+        node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
+        peer_normal.wait_for_getdata([parent_txid_int])
+
+        # Each of the num_peers_unique peers creates a distinct set of orphans
+        many_orphans = [self.create_small_orphan() for _ in range(batch_size * num_peers_unique)]
+
+        self.log.info(f"Send sets of {batch_size} orphans from {num_peers_unique} DoSy peers (may take a while)")
+        for peernum in range(num_peers_unique):
+            peer_doser_batch = node.add_p2p_connection(P2PInterface())
+            this_batch_orphans = many_orphans[batch_size*peernum : batch_size*(peernum+1)]
+            for tx in this_batch_orphans:
+                # Don't wait for responses, because it dramatically increases the runtime of this test.
+                peer_doser_batch.send_without_ping(msg_tx(tx))
+
+            # Ensure at least one of the peer's orphans shows up in getorphantxs. Since each peer is
+            # reserved a portion of orphanage space, this must happen as long as the orphans are not
+            # rejected for some other reason.
+            peer_doser_batch.sync_with_ping()
+            self.wait_until(lambda: any([tx.txid_hex in node.getorphantxs() for tx in this_batch_orphans]))
+
+        self.log.info("Check that orphan from normal peer still exists in orphanage")
+        assert high_fee_child["txid"] in node.getorphantxs()
+
+        self.log.info("Provide the orphan's parent. This 1p1c package should be successfully accepted.")
+        peer_normal.send_and_ping(msg_tx(low_fee_parent["tx"]))
+        assert orphan_tx.txid_hex in node.getrawmempool()
+        assert_equal(node.getmempoolentry(orphan_tx.txid_hex)["ancestorcount"], 2)
+
     def run_test(self):
         node = self.nodes[0]
         # To avoid creating transactions with the same txid (can happen if we set the same feerate
@@ -407,6 +581,9 @@ def run_test(self):
         self.test_multiple_parents()
         self.test_other_parent_in_mempool()
 
+        self.test_orphanage_dos_large()
+        self.test_orphanage_dos_many()
+
 
 if __name__ == '__main__':
     PackageRelayTest(__file__).main()

test/functional/p2p_orphan_handling.py

@@ -5,10 +5,14 @@
 
 import time
 
-from test_framework.mempool_util import tx_in_orphanage
+from test_framework.mempool_util import (
+    create_large_orphan,
+    tx_in_orphanage,
+)
 from test_framework.messages import (
     CInv,
     CTxInWitness,
+    DEFAULT_ANCESTOR_LIMIT,
     MSG_TX,
     MSG_WITNESS_TX,
     MSG_WTX,
@@ -627,6 +631,72 @@ def test_orphan_handling_prefer_outbound(self):
         peer_inbound.sync_with_ping()
         peer_inbound.wait_for_parent_requests([parent_tx.txid_int])
 
+    @cleanup
+    def test_maximal_package_protected(self):
+        self.log.info("Test that a node only announcing a maximally sized ancestor package is protected in orphanage")
+        self.nodes[0].setmocktime(int(time.time()))
+        node = self.nodes[0]
+
+        peer_normal = node.add_p2p_connection(P2PInterface())
+        peer_doser = node.add_p2p_connection(P2PInterface())
+
+        # Each of the num_peers peers creates a distinct set of orphans
+        large_orphans = [create_large_orphan() for _ in range(60)]
+
+        # Check to make sure these are orphans, within max standard size (to be accepted into the orphanage)
+        for large_orphan in large_orphans:
+            testres = node.testmempoolaccept([large_orphan.serialize().hex()])
+            assert not testres[0]["allowed"]
+            assert_equal(testres[0]["reject-reason"], "missing-inputs")
+
+        num_individual_dosers = 20
+        self.log.info(f"Connect {num_individual_dosers} peers and send a very large orphan from each one")
+        # This test assumes that unrequested transactions are processed (skipping inv and
+        # getdata steps because they require going through request delays)
+        # Connect 20 peers and have each of them send a large orphan.
+        for large_orphan in large_orphans[:num_individual_dosers]:
+            peer_doser_individual = node.add_p2p_connection(P2PInterface())
+            peer_doser_individual.send_and_ping(msg_tx(large_orphan))
+            node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + 1)
+            peer_doser_individual.wait_for_getdata([large_orphan.vin[0].prevout.hash])
+
+        # Make sure that these transactions are going through the orphan handling codepaths.
+        # Subsequent rounds will not wait for getdata because the time mocking will cause the
+        # normal package request to time out.
+        self.wait_until(lambda: len(node.getorphantxs()) == num_individual_dosers)
+
+        # Now honest peer will send a maximally sized ancestor package of 24 orphans chaining
+        # off of a single missing transaction, with a total vsize 404,000Wu
+        ancestor_package = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT - 1)
+        sum_ancestor_package_vsize = sum([tx["tx"].get_vsize() for tx in ancestor_package])
+        final_tx = self.wallet.create_self_transfer(utxo_to_spend=ancestor_package[-1]["new_utxo"], target_vsize=101000 - sum_ancestor_package_vsize)
+        ancestor_package.append(final_tx)
+
+        # Peer sends all but first tx to fill up orphange with their orphans
+        for orphan in ancestor_package[1:]:
+            peer_normal.send_and_ping(msg_tx(orphan["tx"]))
+
+        orphan_set = node.getorphantxs()
+        for orphan in ancestor_package[1:]:
+            assert orphan["txid"] in orphan_set
+
+        # Wait for ultimate parent request (the root ancestor transaction)
+        parent_txid_int = ancestor_package[0]["tx"].txid_int
+        node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
+
+        self.wait_until(lambda: "getdata" in peer_normal.last_message and parent_txid_int in [inv.hash for inv in peer_normal.last_message.get("getdata").inv])
+
+        self.log.info("Send another round of very large orphans from a DoSy peer")
+        for large_orphan in large_orphans[num_individual_dosers:]:
+            peer_doser.send_and_ping(msg_tx(large_orphan))
+
+        self.log.info("Provide the top ancestor. The whole package should be re-evaluated after enough time.")
+        peer_normal.send_and_ping(msg_tx(ancestor_package[0]["tx"]))
+
+        # Wait until all transactions have been processed. When the last tx is accepted, it's
+        # guaranteed to have all ancestors.
+        self.wait_until(lambda: node.getmempoolentry(final_tx["txid"])["ancestorcount"] == DEFAULT_ANCESTOR_LIMIT)
+
     @cleanup
     def test_announcers_before_and_after(self):
         self.log.info("Test that the node uses all peers who announced the tx prior to realizing it's an orphan")
@@ -779,6 +849,7 @@ def run_test(self):
         self.test_orphan_handling_prefer_outbound()
         self.test_announcers_before_and_after()
         self.test_parents_change()
+        self.test_maximal_package_protected()
 
 
 if __name__ == '__main__':

test/functional/test_framework/mempool_util.py

@@ -4,11 +4,22 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Helpful routines for mempool testing."""
 from decimal import Decimal
+import random
 
 from .blocktools import (
     COINBASE_MATURITY,
 )
-from .messages import CTransaction
+from .messages import (
+    COutPoint,
+    CTransaction,
+    CTxIn,
+    CTxInWitness,
+    CTxOut,
+)
+from .script import (
+    CScript,
+    OP_RETURN,
+)
 from .util import (
     assert_equal,
     assert_greater_than,
@@ -104,3 +115,13 @@ def tx_in_orphanage(node, tx: CTransaction) -> bool:
     """Returns true if the transaction is in the orphanage."""
     found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.txid_hex and o["wtxid"] == tx.wtxid_hex]
     return len(found) == 1
+
+def create_large_orphan():
+    """Create huge orphan transaction"""
+    tx = CTransaction()
+    # Nonexistent UTXO
+    tx.vin = [CTxIn(COutPoint(random.randrange(1 << 256), random.randrange(1, 100)))]
+    tx.wit.vtxinwit = [CTxInWitness()]
+    tx.wit.vtxinwit[0].scriptWitness.stack = [CScript(b'X' * 390000)]
+    tx.vout = [CTxOut(100, CScript([OP_RETURN, b'a' * 20]))]
+    return tx