Merge #13243: Make reusable base class for auxiliary indices

ec3073a index: Move index DBs into index/ directory. (Jim Posen)
89eddcd index: Remove TxIndexDB from public interface of TxIndex. (Jim Posen)
2318aff MOVEONLY: Move BaseIndex to its own file. (Jim Posen)
f376a49 index: Generalize logged statements in BaseIndex. (Jim Posen)
61a1226 index: Extract logic from TxIndex into reusable base class. (Jim Posen)
e5af5fc db: Make reusable base class for index databases. (Jim Posen)
9b0ec1a db: Remove obsolete methods from CBlockTreeDB. (Jim Posen)

Pull request description:

  This refactors most of the logic in TxIndex into a reusable base class for other indices. There are two commits moving code between files, which may be be more easily reviewed using `git diff --color-moved` (https://blog.github.com/2018-04-05-git-217-released/).

  The motivation for this is to support BIP 157 by indexing block filters.

  <!-- Reviewable:start -->
  ---
  This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/bitcoin/bitcoin/13243)
  <!-- Reviewable:end -->

Tree-SHA512: 0857f04df2aa920178dab2eb8e57984d8eb4d5010deca9971190358479e05b6672ccca2a08af0a7ac9fe02afb947be84cf35a3693204d0667263c6add2959cbf

Author: laanwj

src/Makefile.am

@@ -107,6 +107,7 @@ BITCOIN_CORE_H = \
   fs.h \
   httprpc.h \
   httpserver.h \
+  index/base.h \
   index/txindex.h \
   indirectmap.h \
   init.h \
@@ -208,6 +209,7 @@ libbitcoin_server_a_SOURCES = \
   consensus/tx_verify.cpp \
   httprpc.cpp \
   httpserver.cpp \
+  index/base.cpp \
   index/txindex.cpp \
   init.cpp \
   dbwrapper.cpp \

src/index/base.cpp

@@ -0,0 +1,278 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <chainparams.h>
+#include <index/base.h>
+#include <init.h>
+#include <tinyformat.h>
+#include <ui_interface.h>
+#include <util.h>
+#include <validation.h>
+#include <warnings.h>
+
+constexpr char DB_BEST_BLOCK = 'B';
+
+constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
+constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
+
+template<typename... Args>
+static void FatalError(const char* fmt, const Args&... args)
+{
+    std::string strMessage = tfm::format(fmt, args...);
+    SetMiscWarning(strMessage);
+    LogPrintf("*** %s\n", strMessage);
+    uiInterface.ThreadSafeMessageBox(
+        "Error: A fatal internal error occurred, see debug.log for details",
+        "", CClientUIInterface::MSG_ERROR);
+    StartShutdown();
+}
+
+BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
+    CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
+{}
+
+bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
+{
+    bool success = Read(DB_BEST_BLOCK, locator);
+    if (!success) {
+        locator.SetNull();
+    }
+    return success;
+}
+
+bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
+{
+    return Write(DB_BEST_BLOCK, locator);
+}
+
+BaseIndex::~BaseIndex()
+{
+    Interrupt();
+    Stop();
+}
+
+bool BaseIndex::Init()
+{
+    CBlockLocator locator;
+    if (!GetDB().ReadBestBlock(locator)) {
+        locator.SetNull();
+    }
+
+    LOCK(cs_main);
+    m_best_block_index = FindForkInGlobalIndex(chainActive, locator);
+    m_synced = m_best_block_index.load() == chainActive.Tip();
+    return true;
+}
+
+static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev)
+{
+    AssertLockHeld(cs_main);
+
+    if (!pindex_prev) {
+        return chainActive.Genesis();
+    }
+
+    const CBlockIndex* pindex = chainActive.Next(pindex_prev);
+    if (pindex) {
+        return pindex;
+    }
+
+    return chainActive.Next(chainActive.FindFork(pindex_prev));
+}
+
+void BaseIndex::ThreadSync()
+{
+    const CBlockIndex* pindex = m_best_block_index.load();
+    if (!m_synced) {
+        auto& consensus_params = Params().GetConsensus();
+
+        int64_t last_log_time = 0;
+        int64_t last_locator_write_time = 0;
+        while (true) {
+            if (m_interrupt) {
+                WriteBestBlock(pindex);
+                return;
+            }
+
+            {
+                LOCK(cs_main);
+                const CBlockIndex* pindex_next = NextSyncBlock(pindex);
+                if (!pindex_next) {
+                    WriteBestBlock(pindex);
+                    m_best_block_index = pindex;
+                    m_synced = true;
+                    break;
+                }
+                pindex = pindex_next;
+            }
+
+            int64_t current_time = GetTime();
+            if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
+                LogPrintf("Syncing %s with block chain from height %d\n",
+                          GetName(), pindex->nHeight);
+                last_log_time = current_time;
+            }
+
+            if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
+                WriteBestBlock(pindex);
+                last_locator_write_time = current_time;
+            }
+
+            CBlock block;
+            if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
+                FatalError("%s: Failed to read block %s from disk",
+                           __func__, pindex->GetBlockHash().ToString());
+                return;
+            }
+            if (!WriteBlock(block, pindex)) {
+                FatalError("%s: Failed to write block %s to index database",
+                           __func__, pindex->GetBlockHash().ToString());
+                return;
+            }
+        }
+    }
+
+    if (pindex) {
+        LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight);
+    } else {
+        LogPrintf("%s is enabled\n", GetName());
+    }
+}
+
+bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
+{
+    LOCK(cs_main);
+    if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
+        return error("%s: Failed to write locator to disk", __func__);
+    }
+    return true;
+}
+
+void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
+                               const std::vector<CTransactionRef>& txn_conflicted)
+{
+    if (!m_synced) {
+        return;
+    }
+
+    const CBlockIndex* best_block_index = m_best_block_index.load();
+    if (!best_block_index) {
+        if (pindex->nHeight != 0) {
+            FatalError("%s: First block connected is not the genesis block (height=%d)",
+                       __func__, pindex->nHeight);
+            return;
+        }
+    } else {
+        // Ensure block connects to an ancestor of the current best block. This should be the case
+        // most of the time, but may not be immediately after the sync thread catches up and sets
+        // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
+        // in the ValidationInterface queue backlog even after the sync thread has caught up to the
+        // new chain tip. In this unlikely event, log a warning and let the queue clear.
+        if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
+            LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
+                      "known best chain (tip=%s); not updating index\n",
+                      __func__, pindex->GetBlockHash().ToString(),
+                      best_block_index->GetBlockHash().ToString());
+            return;
+        }
+    }
+
+    if (WriteBlock(*block, pindex)) {
+        m_best_block_index = pindex;
+    } else {
+        FatalError("%s: Failed to write block %s to index",
+                   __func__, pindex->GetBlockHash().ToString());
+        return;
+    }
+}
+
+void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
+{
+    if (!m_synced) {
+        return;
+    }
+
+    const uint256& locator_tip_hash = locator.vHave.front();
+    const CBlockIndex* locator_tip_index;
+    {
+        LOCK(cs_main);
+        locator_tip_index = LookupBlockIndex(locator_tip_hash);
+    }
+
+    if (!locator_tip_index) {
+        FatalError("%s: First block (hash=%s) in locator was not found",
+                   __func__, locator_tip_hash.ToString());
+        return;
+    }
+
+    // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail
+    // immediately after the sync thread catches up and sets m_synced. Consider the case where
+    // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
+    // backlog even after the sync thread has caught up to the new chain tip. In this unlikely
+    // event, log a warning and let the queue clear.
+    const CBlockIndex* best_block_index = m_best_block_index.load();
+    if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
+        LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
+                  "chain (tip=%s); not writing index locator\n",
+                  __func__, locator_tip_hash.ToString(),
+                  best_block_index->GetBlockHash().ToString());
+        return;
+    }
+
+    if (!GetDB().WriteBestBlock(locator)) {
+        error("%s: Failed to write locator to disk", __func__);
+    }
+}
+
+bool BaseIndex::BlockUntilSyncedToCurrentChain()
+{
+    AssertLockNotHeld(cs_main);
+
+    if (!m_synced) {
+        return false;
+    }
+
+    {
+        // Skip the queue-draining stuff if we know we're caught up with
+        // chainActive.Tip().
+        LOCK(cs_main);
+        const CBlockIndex* chain_tip = chainActive.Tip();
+        const CBlockIndex* best_block_index = m_best_block_index.load();
+        if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
+            return true;
+        }
+    }
+
+    LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName());
+    SyncWithValidationInterfaceQueue();
+    return true;
+}
+
+void BaseIndex::Interrupt()
+{
+    m_interrupt();
+}
+
+void BaseIndex::Start()
+{
+    // Need to register this ValidationInterface before running Init(), so that
+    // callbacks are not missed if Init sets m_synced to true.
+    RegisterValidationInterface(this);
+    if (!Init()) {
+        FatalError("%s: %s failed to initialize", __func__, GetName());
+        return;
+    }
+
+    m_thread_sync = std::thread(&TraceThread<std::function<void()>>, GetName(),
+                                std::bind(&BaseIndex::ThreadSync, this));
+}
+
+void BaseIndex::Stop()
+{
+    UnregisterValidationInterface(this);
+
+    if (m_thread_sync.joinable()) {
+        m_thread_sync.join();
+    }
+}

src/index/base.h

@@ -0,0 +1,98 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_INDEX_BASE_H
+#define BITCOIN_INDEX_BASE_H
+
+#include <dbwrapper.h>
+#include <primitives/block.h>
+#include <primitives/transaction.h>
+#include <threadinterrupt.h>
+#include <uint256.h>
+#include <validationinterface.h>
+
+class CBlockIndex;
+
+/**
+ * Base class for indices of blockchain data. This implements
+ * CValidationInterface and ensures blocks are indexed sequentially according
+ * to their position in the active chain.
+ */
+class BaseIndex : public CValidationInterface
+{
+protected:
+    class DB : public CDBWrapper
+    {
+    public:
+        DB(const fs::path& path, size_t n_cache_size,
+           bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
+
+        /// Read block locator of the chain that the txindex is in sync with.
+        bool ReadBestBlock(CBlockLocator& locator) const;
+
+        /// Write block locator of the chain that the txindex is in sync with.
+        bool WriteBestBlock(const CBlockLocator& locator);
+    };
+
+private:
+    /// Whether the index is in sync with the main chain. The flag is flipped
+    /// from false to true once, after which point this starts processing
+    /// ValidationInterface notifications to stay in sync.
+    std::atomic<bool> m_synced{false};
+
+    /// The last block in the chain that the index is in sync with.
+    std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
+
+    std::thread m_thread_sync;
+    CThreadInterrupt m_interrupt;
+
+    /// Sync the index with the block index starting from the current best block.
+    /// Intended to be run in its own thread, m_thread_sync, and can be
+    /// interrupted with m_interrupt. Once the index gets in sync, the m_synced
+    /// flag is set and the BlockConnected ValidationInterface callback takes
+    /// over and the sync thread exits.
+    void ThreadSync();
+
+    /// Write the current chain block locator to the DB.
+    bool WriteBestBlock(const CBlockIndex* block_index);
+
+protected:
+    void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
+                        const std::vector<CTransactionRef>& txn_conflicted) override;
+
+    void ChainStateFlushed(const CBlockLocator& locator) override;
+
+    /// Initialize internal state from the database and block index.
+    virtual bool Init();
+
+    /// Write update index entries for a newly connected block.
+    virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
+
+    virtual DB& GetDB() const = 0;
+
+    /// Get the name of the index for display in logs.
+    virtual const char* GetName() const = 0;
+
+public:
+    /// Destructor interrupts sync thread if running and blocks until it exits.
+    virtual ~BaseIndex();
+
+    /// Blocks the current thread until the index is caught up to the current
+    /// state of the block chain. This only blocks if the index has gotten in
+    /// sync once and only needs to process blocks in the ValidationInterface
+    /// queue. If the index is catching up from far behind, this method does
+    /// not block and immediately returns false.
+    bool BlockUntilSyncedToCurrentChain();
+
+    void Interrupt();
+
+    /// Start initializes the sync state and registers the instance as a
+    /// ValidationInterface so that it stays in sync with blockchain updates.
+    void Start();
+
+    /// Stops the instance from staying in sync with blockchain updates.
+    void Stop();
+};
+
+#endif // BITCOIN_INDEX_BASE_H