Merge dashpay#5900: backport: merge bitcoin#21000, bitcoin#21586, bitcoin#21599, bitcoin#21604, bitcoin#23859, partial bitcoin#21798, bitcoin#26341 (auxiliary backports: part 13)

d033cd5 partial bitcoin#26341: add BIP158 false-positive element check in rpc_scanblocks.py (Kittywhiskers Van Gogh)
eab94ac merge bitcoin#23859: Add missing suppressions for crypto_diff_fuzz_chacha20.cpp (Kittywhiskers Van Gogh)
0e8d4a1 partial bitcoin#21798: Create a block template in tx_pool targets (Kittywhiskers Van Gogh)
ad71db2 merge bitcoin#21604: Document why no symbol names can be used for suppressions (Kittywhiskers Van Gogh)
c116d84 merge bitcoin#21599: Replace file level integer overflow suppression with function level suppression (Kittywhiskers Van Gogh)
8a0dc8c merge bitcoin#21586: Add missing suppression for signed-integer-overflow:txmempool.cpp (Kittywhiskers Van Gogh)
53cf0d5 merge bitcoin#21000: Add UBSan suppressions needed for fuzz tests to not warn under -fsanitize=integer (Kittywhiskers Van Gogh)

Pull request description:

  ## Additional Information

  * Dependency for dashpay#5902
  * [bitcoin#21586](bitcoin#21586) was listed as DNM in the backports Google Sheet as listed below as it is `reverted as bitcoin#23059`
    ![Listing of bitcoin#21586 as DNM](https://github.com/dashpay/dash/assets/63189531/413c116a-b3cb-4b58-becd-0d731b0e4b0b)
    * [bitcoin#23059](bitcoin#23059) actually reverts [bitcoin#22925](bitcoin#22925) (which deals with `addrman.cpp`), not [bitcoin#21586](bitcoin#21586) (which deals with `txmempool.cpp`).

  ## Breaking Changes

  None, changes are limited to fuzzing, functional tests and undefined behavior exclusions

  ## Checklist:
    _Go over all the following points, and put an `x` in all the boxes that apply._
  - [x] I have performed a self-review of my own code
  - [x] I have commented my code, particularly in hard-to-understand areas **(note: N/A)**
  - [x] I have added or updated relevant unit/integration/functional/e2e tests **(note: N/A)**
  - [x] I have made corresponding changes to the documentation **(note: N/A)**
  - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_

Top commit has no ACKs.

Tree-SHA512: 7b6c3fcb462edc9bbcc12a0c4eab052d3ef3a6048281b162dd5650f311f6e1bae7a958adf1c03aa0dea09c56c1ddbc99d98a44023fd81f83b2325fd79dc32e80

Author: PastaPastaPasta

src/test/fuzz/crypto_chacha20_poly1305_aead.cpp

@@ -46,18 +46,24 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead)
                 assert(ok);
             },
             [&] {
+                if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) {
+                    return;
+                }
                 seqnr_payload += 1;
                 aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN;
                 if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) {
                     aad_pos = 0;
+                    if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) {
+                        return;
+                    }
                     seqnr_aad += 1;
                 }
             },
             [&] {
-                seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>();
+                seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>();
             },
             [&] {
-                seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>();
+                seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>();
             },
             [&] {
                 is_encrypt = fuzzed_data_provider.ConsumeBool();

src/test/fuzz/pow.cpp

@@ -44,7 +44,10 @@ FUZZ_TARGET_INIT(pow, initialize_pow)
                 current_block.nHeight = current_height;
             }
             if (fuzzed_data_provider.ConsumeBool()) {
-                current_block.nTime = fixed_time + current_height * consensus_params.nPowTargetSpacing;
+                const uint32_t seconds = current_height * consensus_params.nPowTargetSpacing;
+                if (!AdditionOverflow(fixed_time, seconds)) {
+                    current_block.nTime = fixed_time + seconds;
+                }
             }
             if (fuzzed_data_provider.ConsumeBool()) {
                 current_block.nBits = fixed_bits;

src/test/fuzz/tx_pool.cpp

@@ -76,13 +76,21 @@ void SetMempoolConstraints(ArgsManager& args, FuzzedDataProvider& fuzzed_data_pr
                      ToString(fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 999)));
 }
 
+void MockTime(FuzzedDataProvider& fuzzed_data_provider, const CChainState& chainstate)
+{
+    const auto time = ConsumeTime(fuzzed_data_provider,
+                                  chainstate.m_chain.Tip()->GetMedianTimePast() + 1,
+                                  std::numeric_limits<decltype(chainstate.m_chain.Tip()->nTime)>::max());
+    SetMockTime(time);
+}
+
 FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool)
 {
     FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
     const auto& node = g_setup->m_node;
     auto& chainstate = node.chainman->ActiveChainstate();
 
-    SetMockTime(ConsumeTime(fuzzed_data_provider));
+    MockTime(fuzzed_data_provider, chainstate);
     SetMempoolConstraints(*node.args, fuzzed_data_provider);
 
     // All RBF-spendable outpoints
@@ -161,7 +169,7 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool)
         }();
 
         if (fuzzed_data_provider.ConsumeBool()) {
-            SetMockTime(ConsumeTime(fuzzed_data_provider));
+            MockTime(fuzzed_data_provider, chainstate);
         }
         if (fuzzed_data_provider.ConsumeBool()) {
             SetMempoolConstraints(*node.args, fuzzed_data_provider);
@@ -262,6 +270,10 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool)
 {
     FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
     const auto& node = g_setup->m_node;
+    auto& chainstate = node.chainman->ActiveChainstate();
+
+    MockTime(fuzzed_data_provider, chainstate);
+    SetMempoolConstraints(*node.args, fuzzed_data_provider);
 
     std::vector<uint256> txids;
     for (const auto& outpoint : g_outpoints_coinbase_init_mature) {
@@ -273,12 +285,30 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool)
         txids.push_back(ConsumeUInt256(fuzzed_data_provider));
     }
 
-    CTxMemPool tx_pool{/* estimator */ nullptr, /* check_ratio */ 1};
+    CTxMemPool tx_pool_{/* estimator */ nullptr, /* check_ratio */ 1};
+    MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_);
 
     LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300)
     {
         const auto mut_tx = ConsumeTransaction(fuzzed_data_provider, txids);
 
+        if (fuzzed_data_provider.ConsumeBool()) {
+            MockTime(fuzzed_data_provider, chainstate);
+        }
+        if (fuzzed_data_provider.ConsumeBool()) {
+            SetMempoolConstraints(*node.args, fuzzed_data_provider);
+        }
+        if (fuzzed_data_provider.ConsumeBool()) {
+            tx_pool.RollingFeeUpdate();
+        }
+        if (fuzzed_data_provider.ConsumeBool()) {
+            const auto& txid = fuzzed_data_provider.ConsumeBool() ?
+                                   mut_tx.GetHash() :
+                                   PickValue(fuzzed_data_provider, txids);
+            const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN);
+            tx_pool.PrioritiseTransaction(txid, delta);
+        }
+
         const auto tx = MakeTransactionRef(mut_tx);
         const bool bypass_limits = fuzzed_data_provider.ConsumeBool();
         ::fRequireStandard = fuzzed_data_provider.ConsumeBool();

src/test/fuzz/util.cpp

@@ -6,6 +6,7 @@
 
 #include <test/util/script.h>
 #include <util/overflow.h>
+#include <util/time.h>
 #include <version.h>
 
 FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider)
@@ -217,6 +218,14 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_v
     }
 }
 
+int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min, const std::optional<int64_t>& max) noexcept
+{
+    // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) disables mocktime.
+    static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z");
+    static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z");
+    return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(min.value_or(time_min), max.value_or(time_max));
+}
+
 CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in, const int max_num_out) noexcept
 {
     CMutableTransaction tx_mut;
@@ -248,7 +257,7 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider,
     return tx_mut;
 }
 
-CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length) noexcept
+CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length) noexcept
 {
     const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length);
     return {b.begin(), b.end()};

src/test/fuzz/util.h

@@ -28,7 +28,6 @@
 #include <txmempool.h>
 #include <uint256.h>
 #include <util/overflow.h>
-#include <util/time.h>
 #include <util/vector.h>
 #include <version.h>
 
@@ -61,18 +60,20 @@ auto& PickValue(FuzzedDataProvider& fuzzed_data_provider, Collection& col)
     return *it;
 }
 
-[[ nodiscard ]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept
+[[ nodiscard ]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
 {
-    const std::string s = fuzzed_data_provider.ConsumeRandomLengthString(max_length);
+    const std::string s = max_length ?
+                              fuzzed_data_provider.ConsumeRandomLengthString(*max_length) :
+                              fuzzed_data_provider.ConsumeRandomLengthString();
     return {s.begin(), s.end()};
 }
 
-[[ nodiscard ]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept
+[[ nodiscard ]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
 {
     return BytesToBits(ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length));
 }
 
-[[ nodiscard ]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept
+[[ nodiscard ]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
 {
     return CDataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION};
 }
@@ -99,7 +100,7 @@ template <typename T>
 }
 
 template <typename T>
-[[ nodiscard ]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept
+[[ nodiscard ]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
 {
     const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length);
     CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION};
@@ -113,7 +114,7 @@ template <typename T>
 }
 
 template <typename WeakEnumType, size_t size>
-[[nodiscard]] WeakEnumType ConsumeWeakEnum(FuzzedDataProvider& fuzzed_data_provider, const WeakEnumType (&all_types)[size]) noexcept
+[[ nodiscard ]] WeakEnumType ConsumeWeakEnum(FuzzedDataProvider& fuzzed_data_provider, const WeakEnumType (&all_types)[size]) noexcept
 {
     return fuzzed_data_provider.ConsumeBool() ?
                fuzzed_data_provider.PickValueInArray<WeakEnumType>(all_types) :
@@ -130,17 +131,11 @@ template <typename WeakEnumType, size_t size>
     return fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, MAX_MONEY);
 }
 
-[[ nodiscard ]] inline int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider) noexcept
-{
-    // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) is a no-op.
-    static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z");
-    static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z");
-    return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(time_min, time_max);
-}
+[[ nodiscard ]] int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min = std::nullopt, const std::optional<int64_t>& max = std::nullopt) noexcept;
 
 [[ nodiscard ]] CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in = 10, const int max_num_out = 10) noexcept;
 
-[[ nodiscard ]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept;
+[[ nodiscard ]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept;
 
 [[ nodiscard ]] uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept;
 

test/functional/test_framework/blockfilter.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Helper routines relevant for compact block filters (BIP158).
+"""
+from .siphash import siphash
+
+
+def bip158_basic_element_hash(script_pub_key, N, block_hash):
+    """ Calculates the ranged hash of a filter element as defined in BIP158:
+
+    'The first step in the filter construction is hashing the variable-sized
+    raw items in the set to the range [0, F), where F = N * M.'
+
+    'The items are first passed through the pseudorandom function SipHash, which takes a
+    128-bit key k and a variable-sized byte vector and produces a uniformly random 64-bit
+    output. Implementations of this BIP MUST use the SipHash parameters c = 2 and d = 4.'
+
+    'The parameter k MUST be set to the first 16 bytes of the hash (in standard
+    little-endian representation) of the block for which the filter is constructed. This
+    ensures the key is deterministic while still varying from block to block.'
+    """
+    M = 784931
+    block_hash_bytes = bytes.fromhex(block_hash)[::-1]
+    k0 = int.from_bytes(block_hash_bytes[0:8], 'little')
+    k1 = int.from_bytes(block_hash_bytes[8:16], 'little')
+    return (siphash(k0, k1, script_pub_key) * (N * M)) >> 64
+
+
+def bip158_relevant_scriptpubkeys(node, block_hash):
+    """ Determines the basic filter relvant scriptPubKeys as defined in BIP158:
+
+    'A basic filter MUST contain exactly the following items for each transaction in a block:
+       - The previous output script (the script being spent) for each input, except for
+         the coinbase transaction.
+       - The scriptPubKey of each output, aside from all OP_RETURN output scripts.'
+    """
+    spks = set()
+    for tx in node.getblock(blockhash=block_hash, verbosity=3)['tx']:
+        # gather prevout scripts
+        for i in tx['vin']:
+            if 'prevout' in i:
+                spks.add(bytes.fromhex(i['prevout']['scriptPubKey']['hex']))
+        # gather output scripts
+        for o in tx['vout']:
+            if o['scriptPubKey']['type'] != 'nulldata':
+                spks.add(bytes.fromhex(o['scriptPubKey']['hex']))
+    return spks

test/functional/test_framework/siphash.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016 The Bitcoin Core developers
+# Copyright (c) 2016-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Specialized SipHash-2-4 implementations.
+"""SipHash-2-4 implementation.
 
-This implements SipHash-2-4 for 256-bit integers.
+This implements SipHash-2-4. For convenience, an interface taking 256-bit
+integers is provided in addition to the one accepting generic data.
 """
 
 def rotl64(n, b):
     return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
 
+
 def siphash_round(v0, v1, v2, v3):
     v0 = (v0 + v1) & ((1 << 64) - 1)
     v1 = rotl64(v1, 13)
@@ -27,37 +29,37 @@ def siphash_round(v0, v1, v2, v3):
     v2 = rotl64(v2, 32)
     return (v0, v1, v2, v3)
 
-def siphash256(k0, k1, h):
-    n0 = h & ((1 << 64) - 1)
-    n1 = (h >> 64) & ((1 << 64) - 1)
-    n2 = (h >> 128) & ((1 << 64) - 1)
-    n3 = (h >> 192) & ((1 << 64) - 1)
+
+def siphash(k0, k1, data):
+    assert(type(data) == bytes)
     v0 = 0x736f6d6570736575 ^ k0
     v1 = 0x646f72616e646f6d ^ k1
     v2 = 0x6c7967656e657261 ^ k0
-    v3 = 0x7465646279746573 ^ k1 ^ n0
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n0
-    v3 ^= n1
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n1
-    v3 ^= n2
+    v3 = 0x7465646279746573 ^ k1
+    c = 0
+    t = 0
+    for d in data:
+        t |= d << (8 * (c % 8))
+        c = (c + 1) & 0xff
+        if (c & 7) == 0:
+            v3 ^= t
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0 ^= t
+            t = 0
+    t = t | (c << 56)
+    v3 ^= t
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n2
-    v3 ^= n3
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n3
-    v3 ^= 0x2000000000000000
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= 0x2000000000000000
-    v2 ^= 0xFF
+    v0 ^= t
+    v2 ^= 0xff
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     return v0 ^ v1 ^ v2 ^ v3
+
+
+def siphash256(k0, k1, num):
+    assert(type(num) == int)
+    return siphash(k0, k1, num.to_bytes(32, 'little'))