Merge #14121: Index for BIP 157 block filters

c7efb65 blockfilter: Update BIP 158 test vectors. (Jim Posen)
19308c9 rpc: Add getblockfilter RPC method. (Jim Posen)
ff35105 init: Add CLI option to enable block filter index. (Jim Posen)
accc8b8 index: Access functions for global block filter indexes. (Jim Posen)
2bc90e4 test: Unit test for block filter index reorg handling. (Jim Posen)
6bcf099 test: Unit tests for block index filter. (Jim Posen)
b5e8200 index: Implement lookup methods on block filter index. (Jim Posen)
75a76e3 index: Implement block filter index with write operations. (Jim Posen)
2ad2338 serialize: Serialization support for big-endian 32-bit ints. (Jim Posen)
ba6ff9a blockfilter: Functions to translate filter types to/from names. (Jim Posen)
62b7a4f index: Ensure block locator is not stale after chain reorg. (Jim Posen)
4368384 index: Allow atomic commits of index state to be extended. (Jim Posen)

Pull request description:

  This introduces a new BlockFilterIndex class, which is required for BIP 157 support.

  The index is uses the asynchronous BaseIndex infrastructure driven by the ValidationInterface callbacks. Filters are stored sequentially in flat files and the disk location of each filter is indexed in LevelDB along with the filter hash and header. The index is designed to ensure persistence of filters reorganized out of the main chain to simplify the BIP 157 net implementation.

  Stats (block height = 565500):
  - Syncing the index from scratch takes 45m
  - Total index size is 3.8 GiB

ACKs for commit c7efb6:
  MarcoFalke:
    utACK c7efb65
  ryanofsky:
    Slightly tested ACK c7efb65 (I just rebuilt the index with the updated PR and tested the RPC). Changes since last review: rebase, fixed compile errors in internal commits, new comments, updated error messages, tweaked cache size logic, renamed commit method, renamed constants and globals, fixed whitespace, extra BlockFilterIndex::Init error check.

Tree-SHA512: f8ed7a9b6f76df45933aa5eba92b27b3af83f6df2ccb3728a5c89eec80f654344dc14f055f6f63eb9b3a7649dd8af6553fe14969889e7e2fd2f8461574d18f28

Author: MarcoFalke

src/Makefile.am

@@ -134,6 +134,7 @@ BITCOIN_CORE_H = \
   httprpc.h \
   httpserver.h \
   index/base.h \
+  index/blockfilterindex.h \
   index/txindex.h \
   indirectmap.h \
   init.h \
@@ -263,6 +264,7 @@ libbitcoin_server_a_SOURCES = \
   httprpc.cpp \
   httpserver.cpp \
   index/base.cpp \
+  index/blockfilterindex.cpp \
   index/txindex.cpp \
   interfaces/chain.cpp \
   interfaces/node.cpp \

src/Makefile.test.include

@@ -92,6 +92,7 @@ BITCOIN_TESTS =\
   test/blockchain_tests.cpp \
   test/blockencodings_tests.cpp \
   test/blockfilter_tests.cpp \
+  test/blockfilter_index_tests.cpp \
   test/bloom_tests.cpp \
   test/bswap_tests.cpp \
   test/checkqueue_tests.cpp \

src/blockfilter.cpp

@@ -2,6 +2,9 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <mutex>
+#include <sstream>
+
 #include <blockfilter.h>
 #include <crypto/siphash.h>
 #include <hash.h>
@@ -15,6 +18,10 @@ static constexpr int GCS_SER_TYPE = SER_NETWORK;
 /// Protocol version used to serialize parameters in GCS filter encoding.
 static constexpr int GCS_SER_VERSION = 0;
 
+static const std::map<BlockFilterType, std::string> g_filter_types = {
+    {BlockFilterType::BASIC, "basic"},
+};
+
 template <typename OStream>
 static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x)
 {
@@ -197,6 +204,57 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const
     return MatchInternal(queries.data(), queries.size());
 }
 
+const std::string& BlockFilterTypeName(BlockFilterType filter_type)
+{
+    static std::string unknown_retval = "";
+    auto it = g_filter_types.find(filter_type);
+    return it != g_filter_types.end() ? it->second : unknown_retval;
+}
+
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) {
+    for (const auto& entry : g_filter_types) {
+        if (entry.second == name) {
+            filter_type = entry.first;
+            return true;
+        }
+    }
+    return false;
+}
+
+const std::vector<BlockFilterType>& AllBlockFilterTypes()
+{
+    static std::vector<BlockFilterType> types;
+
+    static std::once_flag flag;
+    std::call_once(flag, []() {
+            types.reserve(g_filter_types.size());
+            for (auto entry : g_filter_types) {
+                types.push_back(entry.first);
+            }
+        });
+
+    return types;
+}
+
+const std::string& ListBlockFilterTypes()
+{
+    static std::string type_list;
+
+    static std::once_flag flag;
+    std::call_once(flag, []() {
+            std::stringstream ret;
+            bool first = true;
+            for (auto entry : g_filter_types) {
+                if (!first) ret << ", ";
+                ret << entry.second;
+                first = false;
+            }
+            type_list = ret.str();
+        });
+
+    return type_list;
+}
+
 static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
                                                  const CBlockUndo& block_undo)
 {

src/blockfilter.h

@@ -6,6 +6,7 @@
 #define BITCOIN_BLOCKFILTER_H
 
 #include <stdint.h>
+#include <string>
 #include <unordered_set>
 #include <vector>
 
@@ -89,6 +90,18 @@ enum class BlockFilterType : uint8_t
     INVALID = 255,
 };
 
+/** Get the human-readable name for a filter type. Returns empty string for unknown types. */
+const std::string& BlockFilterTypeName(BlockFilterType filter_type);
+
+/** Find a filter type by its human-readable name. */
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type);
+
+/** Get a list of known filter types. */
+const std::vector<BlockFilterType>& AllBlockFilterTypes();
+
+/** Get a comma-separated list of known filter type names. */
+const std::string& ListBlockFilterTypes();
+
 /**
  * Complete block filter struct as defined in BIP 157. Serialization matches
  * payload of "cfilter" messages.

src/index/base.cpp

@@ -41,9 +41,9 @@ bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
     return success;
 }
 
-bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
+void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator)
 {
-    return Write(DB_BEST_BLOCK, locator);
+    batch.Write(DB_BEST_BLOCK, locator);
 }
 
 BaseIndex::~BaseIndex()
@@ -95,19 +95,29 @@ void BaseIndex::ThreadSync()
         int64_t last_locator_write_time = 0;
         while (true) {
             if (m_interrupt) {
-                WriteBestBlock(pindex);
+                m_best_block_index = pindex;
+                // No need to handle errors in Commit. If it fails, the error will be already be
+                // logged. The best way to recover is to continue, as index cannot be corrupted by
+                // a missed commit to disk for an advanced index state.
+                Commit();
                 return;
             }
 
             {
                 LOCK(cs_main);
                 const CBlockIndex* pindex_next = NextSyncBlock(pindex);
                 if (!pindex_next) {
-                    WriteBestBlock(pindex);
                     m_best_block_index = pindex;
                     m_synced = true;
+                    // No need to handle errors in Commit. See rationale above.
+                    Commit();
                     break;
                 }
+                if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) {
+                    FatalError("%s: Failed to rewind index %s to a previous chain tip",
+                               __func__, GetName());
+                    return;
+                }
                 pindex = pindex_next;
             }
 
@@ -119,8 +129,10 @@ void BaseIndex::ThreadSync()
             }
 
             if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                WriteBestBlock(pindex);
+                m_best_block_index = pindex;
                 last_locator_write_time = current_time;
+                // No need to handle errors in Commit. See rationale above.
+                Commit();
             }
 
             CBlock block;
@@ -144,12 +156,35 @@ void BaseIndex::ThreadSync()
     }
 }
 
-bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
+bool BaseIndex::Commit()
+{
+    CDBBatch batch(GetDB());
+    if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) {
+        return error("%s: Failed to commit latest %s state", __func__, GetName());
+    }
+    return true;
+}
+
+bool BaseIndex::CommitInternal(CDBBatch& batch)
 {
     LOCK(cs_main);
-    if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
-        return error("%s: Failed to write locator to disk", __func__);
+    GetDB().WriteBestBlock(batch, chainActive.GetLocator(m_best_block_index));
+    return true;
+}
+
+bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
+{
+    assert(current_tip == m_best_block_index);
+    assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
+
+    // In the case of a reorg, ensure persisted block locator is not stale.
+    m_best_block_index = new_tip;
+    if (!Commit()) {
+        // If commit fails, revert the best block index to avoid corruption.
+        m_best_block_index = current_tip;
+        return false;
     }
+
     return true;
 }
 
@@ -180,6 +215,11 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
                       best_block_index->GetBlockHash().ToString());
             return;
         }
+        if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) {
+            FatalError("%s: Failed to rewind index %s to a previous chain tip",
+                       __func__, GetName());
+            return;
+        }
     }
 
     if (WriteBlock(*block, pindex)) {
@@ -224,9 +264,10 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
         return;
     }
 
-    if (!GetDB().WriteBestBlock(locator)) {
-        error("%s: Failed to write locator to disk", __func__);
-    }
+    // No need to handle errors in Commit. If it fails, the error will be already be logged. The
+    // best way to recover is to continue, as index cannot be corrupted by a missed commit to disk
+    // for an advanced index state.
+    Commit();
 }
 
 bool BaseIndex::BlockUntilSyncedToCurrentChain()

src/index/base.h

@@ -32,7 +32,7 @@ class BaseIndex : public CValidationInterface
         bool ReadBestBlock(CBlockLocator& locator) const;
 
         /// Write block locator of the chain that the txindex is in sync with.
-        bool WriteBestBlock(const CBlockLocator& locator);
+        void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator);
     };
 
 private:
@@ -54,8 +54,15 @@ class BaseIndex : public CValidationInterface
     /// over and the sync thread exits.
     void ThreadSync();
 
-    /// Write the current chain block locator to the DB.
-    bool WriteBestBlock(const CBlockIndex* block_index);
+    /// Write the current index state (eg. chain block locator and subclass-specific items) to disk.
+    ///
+    /// Recommendations for error handling:
+    /// If called on a successor of the previous committed best block in the index, the index can
+    /// continue processing without risk of corruption, though the index state will need to catch up
+    /// from further behind on reboot. If the new state is not a successor of the previous state (due
+    /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
+    /// getting corrupted.
+    bool Commit();
 
 protected:
     void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
@@ -69,6 +76,14 @@ class BaseIndex : public CValidationInterface
     /// Write update index entries for a newly connected block.
     virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
 
+    /// Virtual method called internally by Commit that can be overridden to atomically
+    /// commit more index state.
+    virtual bool CommitInternal(CDBBatch& batch);
+
+    /// Rewind index to an earlier chain tip during a chain reorg. The tip must
+    /// be an ancestor of the current best block.
+    virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip);
+
     virtual DB& GetDB() const = 0;
 
     /// Get the name of the index for display in logs.