Merge #12254: BIP 158: Compact Block Filters for Light Clients

254c85b bench: Benchmark GCS filter creation and matching. (Jim Posen)
f33b717 blockfilter: Optimization on compilers with int128 support. (Jim Posen)
97b64d6 blockfilter: Unit test against BIP 158 test vectors. (Jim Posen)
a4afb9c blockfilter: Additional helper methods to compute hash and header. (Jim Posen)
cd09c79 blockfilter: Serialization methods on BlockFilter. (Jim Posen)
c1855f6 blockfilter: Construction of basic block filters. (Jim Posen)
53e7874 blockfilter: Simple test for GCSFilter construction and Match. (Jim Posen)
558c536 blockfilter: Implement GCSFilter Match methods. (Jim Posen)
cf70b55 blockfilter: Implement GCSFilter constructors. (Jim Posen)
c454f0a blockfilter: Declare GCSFilter class for BIP 158 impl. (Jim Posen)
9b622dc streams: Unit tests for BitStreamReader and BitStreamWriter. (Jim Posen)
fe943f9 streams: Implement BitStreamReader/Writer classes. (Jim Posen)
87f2d9e streams: Unit test for VectorReader class. (Jim Posen)
947133d streams: Create VectorReader stream interface for vectors. (Jim Posen)

Pull request description:

  This implements the compact block filter construction in [BIP 158](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki). The code is not used anywhere in the Bitcoin Core code base yet. The next step towards [BIP 157](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) support would be to create an indexing module similar to `TxIndex` that constructs the basic and extended filters for each validated block.

  ### Filter Sizes

  [Here](https://gateway.ipfs.io/ipfs/QmRqaAAQZ5ZX5eqxP7J2R1MzFrc2WDdKSWJEKtQzyawqog) is a CSV of filter sizes for blocks in the main chain.

  As you can see below, the ratio of filter size to block size drops after the first ~150,000 blocks:

  ![filter_sizes](https://user-images.githubusercontent.com/881253/42900589-299772d4-8a7e-11e8-886d-0d4f3f4fbe44.png)

  The reason for the relatively large filter sizes is that Golomb-coded sets only achieve good compression with a sufficient number of elements. Empirically, the average element size with 100 elements is 14% larger than with 10,000 elements.

  The ratio of filter size to block size is computed without witness data for basic filters. Here is a summary table of filter size ratios *for blocks after height 150,000*:

  | Stat | Filter Type |
  |-------|--------------|
  | Weighted Size Ratio Mean | 0.0198 |
  | Size Ratio Mean | 0.0224 |
  | Size Ratio Std Deviation | 0.0202 |
  | Mean Element Size (bits) | 21.145 |
  | Approx Theoretical Min Element Size (bits) | 21.025 |

Tree-SHA512: 2d045fbfc3fc45490ecb9b08d2f7e4dbbe7cd8c1c939f06bbdb8e8aacfe4c495cdb67c820e52520baebbf8a8305a0efd8e59d3fa8e367574a4b830509a39223f

Author: laanwj

src/Makefile.am

@@ -96,6 +96,7 @@ BITCOIN_CORE_H = \
   bech32.h \
   bloom.h \
   blockencodings.h \
+  blockfilter.h \
   chain.h \
   chainparams.h \
   chainparamsbase.h \
@@ -219,6 +220,7 @@ libbitcoin_server_a_SOURCES = \
   addrman.cpp \
   bloom.cpp \
   blockencodings.cpp \
+  blockfilter.cpp \
   chain.cpp \
   checkpoints.cpp \
   consensus/tx_verify.cpp \

src/Makefile.bench.include

@@ -22,6 +22,7 @@ bench_bench_bitcoin_SOURCES = \
   bench/rollingbloom.cpp \
   bench/crypto_hash.cpp \
   bench/ccoins_caching.cpp \
+  bench/gcs_filter.cpp \
   bench/merkle_root.cpp \
   bench/mempool_eviction.cpp \
   bench/verify_script.cpp \

src/Makefile.test.include

@@ -9,6 +9,7 @@ TEST_BINARY=test/test_bitcoin$(EXEEXT)
 
 JSON_TEST_FILES = \
   test/data/base58_encode_decode.json \
+  test/data/blockfilters.json \
   test/data/key_io_valid.json \
   test/data/key_io_invalid.json \
   test/data/script_tests.json \
@@ -39,6 +40,7 @@ BITCOIN_TESTS =\
   test/bip32_tests.cpp \
   test/blockchain_tests.cpp \
   test/blockencodings_tests.cpp \
+  test/blockfilter_tests.cpp \
   test/bloom_tests.cpp \
   test/bswap_tests.cpp \
   test/checkqueue_tests.cpp \

src/bench/gcs_filter.cpp

@@ -0,0 +1,43 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <blockfilter.h>
+
+static void ConstructGCSFilter(benchmark::State& state)
+{
+    GCSFilter::ElementSet elements;
+    for (int i = 0; i < 10000; ++i) {
+        GCSFilter::Element element(32);
+        element[0] = static_cast<unsigned char>(i);
+        element[1] = static_cast<unsigned char>(i >> 8);
+        elements.insert(std::move(element));
+    }
+
+    uint64_t siphash_k0 = 0;
+    while (state.KeepRunning()) {
+        GCSFilter filter(siphash_k0, 0, 20, 1 << 20, elements);
+
+        siphash_k0++;
+    }
+}
+
+static void MatchGCSFilter(benchmark::State& state)
+{
+    GCSFilter::ElementSet elements;
+    for (int i = 0; i < 10000; ++i) {
+        GCSFilter::Element element(32);
+        element[0] = static_cast<unsigned char>(i);
+        element[1] = static_cast<unsigned char>(i >> 8);
+        elements.insert(std::move(element));
+    }
+    GCSFilter filter(0, 0, 20, 1 << 20, elements);
+
+    while (state.KeepRunning()) {
+        filter.Match(GCSFilter::Element());
+    }
+}
+
+BENCHMARK(ConstructGCSFilter, 1000);
+BENCHMARK(MatchGCSFilter, 50 * 1000);

src/blockfilter.cpp

@@ -0,0 +1,260 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <blockfilter.h>
+#include <hash.h>
+#include <primitives/transaction.h>
+#include <script/script.h>
+#include <streams.h>
+
+/// SerType used to serialize parameters in GCS filter encoding.
+static constexpr int GCS_SER_TYPE = SER_NETWORK;
+
+/// Protocol version used to serialize parameters in GCS filter encoding.
+static constexpr int GCS_SER_VERSION = 0;
+
+template <typename OStream>
+static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x)
+{
+    // Write quotient as unary-encoded: q 1's followed by one 0.
+    uint64_t q = x >> P;
+    while (q > 0) {
+        int nbits = q <= 64 ? static_cast<int>(q) : 64;
+        bitwriter.Write(~0ULL, nbits);
+        q -= nbits;
+    }
+    bitwriter.Write(0, 1);
+
+    // Write the remainder in P bits. Since the remainder is just the bottom
+    // P bits of x, there is no need to mask first.
+    bitwriter.Write(x, P);
+}
+
+template <typename IStream>
+static uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P)
+{
+    // Read unary-encoded quotient: q 1's followed by one 0.
+    uint64_t q = 0;
+    while (bitreader.Read(1) == 1) {
+        ++q;
+    }
+
+    uint64_t r = bitreader.Read(P);
+
+    return (q << P) + r;
+}
+
+// Map a value x that is uniformly distributed in the range [0, 2^64) to a
+// value uniformly distributed in [0, n) by returning the upper 64 bits of
+// x * n.
+//
+// See: https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
+static uint64_t MapIntoRange(uint64_t x, uint64_t n)
+{
+#ifdef __SIZEOF_INT128__
+    return (static_cast<unsigned __int128>(x) * static_cast<unsigned __int128>(n)) >> 64;
+#else
+    // To perform the calculation on 64-bit numbers without losing the
+    // result to overflow, split the numbers into the most significant and
+    // least significant 32 bits and perform multiplication piece-wise.
+    //
+    // See: https://stackoverflow.com/a/26855440
+    uint64_t x_hi = x >> 32;
+    uint64_t x_lo = x & 0xFFFFFFFF;
+    uint64_t n_hi = n >> 32;
+    uint64_t n_lo = n & 0xFFFFFFFF;
+
+    uint64_t ac = x_hi * n_hi;
+    uint64_t ad = x_hi * n_lo;
+    uint64_t bc = x_lo * n_hi;
+    uint64_t bd = x_lo * n_lo;
+
+    uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF);
+    uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32);
+    return upper64;
+#endif
+}
+
+uint64_t GCSFilter::HashToRange(const Element& element) const
+{
+    uint64_t hash = CSipHasher(m_siphash_k0, m_siphash_k1)
+        .Write(element.data(), element.size())
+        .Finalize();
+    return MapIntoRange(hash, m_F);
+}
+
+std::vector<uint64_t> GCSFilter::BuildHashedSet(const ElementSet& elements) const
+{
+    std::vector<uint64_t> hashed_elements;
+    hashed_elements.reserve(elements.size());
+    for (const Element& element : elements) {
+        hashed_elements.push_back(HashToRange(element));
+    }
+    std::sort(hashed_elements.begin(), hashed_elements.end());
+    return hashed_elements;
+}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M)
+    : m_siphash_k0(siphash_k0), m_siphash_k1(siphash_k1), m_P(P), m_M(M), m_N(0), m_F(0)
+{}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+                     std::vector<unsigned char> encoded_filter)
+    : GCSFilter(siphash_k0, siphash_k1, P, M)
+{
+    m_encoded = std::move(encoded_filter);
+
+    VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+    uint64_t N = ReadCompactSize(stream);
+    m_N = static_cast<uint32_t>(N);
+    if (m_N != N) {
+        throw std::ios_base::failure("N must be <2^32");
+    }
+    m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_M);
+
+    // Verify that the encoded filter contains exactly N elements. If it has too much or too little
+    // data, a std::ios_base::failure exception will be raised.
+    BitStreamReader<VectorReader> bitreader(stream);
+    for (uint64_t i = 0; i < m_N; ++i) {
+        GolombRiceDecode(bitreader, m_P);
+    }
+    if (!stream.empty()) {
+        throw std::ios_base::failure("encoded_filter contains excess data");
+    }
+}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+                     const ElementSet& elements)
+    : GCSFilter(siphash_k0, siphash_k1, P, M)
+{
+    size_t N = elements.size();
+    m_N = static_cast<uint32_t>(N);
+    if (m_N != N) {
+        throw std::invalid_argument("N must be <2^32");
+    }
+    m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_M);
+
+    CVectorWriter stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+    WriteCompactSize(stream, m_N);
+
+    if (elements.empty()) {
+        return;
+    }
+
+    BitStreamWriter<CVectorWriter> bitwriter(stream);
+
+    uint64_t last_value = 0;
+    for (uint64_t value : BuildHashedSet(elements)) {
+        uint64_t delta = value - last_value;
+        GolombRiceEncode(bitwriter, m_P, delta);
+        last_value = value;
+    }
+
+    bitwriter.Flush();
+}
+
+bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const
+{
+    VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+    // Seek forward by size of N
+    uint64_t N = ReadCompactSize(stream);
+    assert(N == m_N);
+
+    BitStreamReader<VectorReader> bitreader(stream);
+
+    uint64_t value = 0;
+    size_t hashes_index = 0;
+    for (uint32_t i = 0; i < m_N; ++i) {
+        uint64_t delta = GolombRiceDecode(bitreader, m_P);
+        value += delta;
+
+        while (true) {
+            if (hashes_index == size) {
+                return false;
+            } else if (element_hashes[hashes_index] == value) {
+                return true;
+            } else if (element_hashes[hashes_index] > value) {
+                break;
+            }
+
+            hashes_index++;
+        }
+    }
+
+    return false;
+}
+
+bool GCSFilter::Match(const Element& element) const
+{
+    uint64_t query = HashToRange(element);
+    return MatchInternal(&query, 1);
+}
+
+bool GCSFilter::MatchAny(const ElementSet& elements) const
+{
+    const std::vector<uint64_t> queries = BuildHashedSet(elements);
+    return MatchInternal(queries.data(), queries.size());
+}
+
+static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
+                                                 const CBlockUndo& block_undo)
+{
+    GCSFilter::ElementSet elements;
+
+    for (const CTransactionRef& tx : block.vtx) {
+        for (const CTxOut& txout : tx->vout) {
+            const CScript& script = txout.scriptPubKey;
+            if (script[0] == OP_RETURN) continue;
+            elements.emplace(script.begin(), script.end());
+        }
+    }
+
+    for (const CTxUndo& tx_undo : block_undo.vtxundo) {
+        for (const Coin& prevout : tx_undo.vprevout) {
+            const CScript& script = prevout.out.scriptPubKey;
+            elements.emplace(script.begin(), script.end());
+        }
+    }
+
+    return elements;
+}
+
+BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo)
+    : m_filter_type(filter_type), m_block_hash(block.GetHash())
+{
+    switch (m_filter_type) {
+    case BlockFilterType::BASIC:
+        m_filter = GCSFilter(m_block_hash.GetUint64(0), m_block_hash.GetUint64(1),
+                             BASIC_FILTER_P, BASIC_FILTER_M,
+                             BasicFilterElements(block, block_undo));
+        break;
+
+    default:
+        throw std::invalid_argument("unknown filter_type");
+    }
+}
+
+uint256 BlockFilter::GetHash() const
+{
+    const std::vector<unsigned char>& data = GetEncodedFilter();
+
+    uint256 result;
+    CHash256().Write(data.data(), data.size()).Finalize(result.begin());
+    return result;
+}
+
+uint256 BlockFilter::ComputeHeader(const uint256& prev_header) const
+{
+    const uint256& filter_hash = GetHash();
+
+    uint256 result;
+    CHash256()
+        .Write(filter_hash.begin(), filter_hash.size())
+        .Write(prev_header.begin(), prev_header.size())
+        .Finalize(result.begin());
+    return result;
+}