partial bitcoin#26341: add BIP158 false-positive element check in rpc_scanblocks.py

excludes:
- fa54d30

Author: kwvg

test/functional/test_framework/blockfilter.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Helper routines relevant for compact block filters (BIP158).
+"""
+from .siphash import siphash
+
+
+def bip158_basic_element_hash(script_pub_key, N, block_hash):
+    """ Calculates the ranged hash of a filter element as defined in BIP158:
+
+    'The first step in the filter construction is hashing the variable-sized
+    raw items in the set to the range [0, F), where F = N * M.'
+
+    'The items are first passed through the pseudorandom function SipHash, which takes a
+    128-bit key k and a variable-sized byte vector and produces a uniformly random 64-bit
+    output. Implementations of this BIP MUST use the SipHash parameters c = 2 and d = 4.'
+
+    'The parameter k MUST be set to the first 16 bytes of the hash (in standard
+    little-endian representation) of the block for which the filter is constructed. This
+    ensures the key is deterministic while still varying from block to block.'
+    """
+    M = 784931
+    block_hash_bytes = bytes.fromhex(block_hash)[::-1]
+    k0 = int.from_bytes(block_hash_bytes[0:8], 'little')
+    k1 = int.from_bytes(block_hash_bytes[8:16], 'little')
+    return (siphash(k0, k1, script_pub_key) * (N * M)) >> 64
+
+
+def bip158_relevant_scriptpubkeys(node, block_hash):
+    """ Determines the basic filter relvant scriptPubKeys as defined in BIP158:
+
+    'A basic filter MUST contain exactly the following items for each transaction in a block:
+       - The previous output script (the script being spent) for each input, except for
+         the coinbase transaction.
+       - The scriptPubKey of each output, aside from all OP_RETURN output scripts.'
+    """
+    spks = set()
+    for tx in node.getblock(blockhash=block_hash, verbosity=3)['tx']:
+        # gather prevout scripts
+        for i in tx['vin']:
+            if 'prevout' in i:
+                spks.add(bytes.fromhex(i['prevout']['scriptPubKey']['hex']))
+        # gather output scripts
+        for o in tx['vout']:
+            if o['scriptPubKey']['type'] != 'nulldata':
+                spks.add(bytes.fromhex(o['scriptPubKey']['hex']))
+    return spks

test/functional/test_framework/siphash.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016 The Bitcoin Core developers
+# Copyright (c) 2016-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Specialized SipHash-2-4 implementations.
+"""SipHash-2-4 implementation.
 
-This implements SipHash-2-4 for 256-bit integers.
+This implements SipHash-2-4. For convenience, an interface taking 256-bit
+integers is provided in addition to the one accepting generic data.
 """
 
 def rotl64(n, b):
     return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
 
+
 def siphash_round(v0, v1, v2, v3):
     v0 = (v0 + v1) & ((1 << 64) - 1)
     v1 = rotl64(v1, 13)
@@ -27,37 +29,37 @@ def siphash_round(v0, v1, v2, v3):
     v2 = rotl64(v2, 32)
     return (v0, v1, v2, v3)
 
-def siphash256(k0, k1, h):
-    n0 = h & ((1 << 64) - 1)
-    n1 = (h >> 64) & ((1 << 64) - 1)
-    n2 = (h >> 128) & ((1 << 64) - 1)
-    n3 = (h >> 192) & ((1 << 64) - 1)
+
+def siphash(k0, k1, data):
+    assert(type(data) == bytes)
     v0 = 0x736f6d6570736575 ^ k0
     v1 = 0x646f72616e646f6d ^ k1
     v2 = 0x6c7967656e657261 ^ k0
-    v3 = 0x7465646279746573 ^ k1 ^ n0
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n0
-    v3 ^= n1
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n1
-    v3 ^= n2
+    v3 = 0x7465646279746573 ^ k1
+    c = 0
+    t = 0
+    for d in data:
+        t |= d << (8 * (c % 8))
+        c = (c + 1) & 0xff
+        if (c & 7) == 0:
+            v3 ^= t
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0 ^= t
+            t = 0
+    t = t | (c << 56)
+    v3 ^= t
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n2
-    v3 ^= n3
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n3
-    v3 ^= 0x2000000000000000
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= 0x2000000000000000
-    v2 ^= 0xFF
+    v0 ^= t
+    v2 ^= 0xff
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     return v0 ^ v1 ^ v2 ^ v3
+
+
+def siphash256(k0, k1, num):
+    assert(type(num) == int)
+    return siphash(k0, k1, num.to_bytes(32, 'little'))