Remove utxo db upgrade code

Author: MarcoFalke

ci/test/00_setup_env_native_qt5.sh

@@ -14,6 +14,6 @@ export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude fe
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
-export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0"
+export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.14.3 v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0"
 export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \
 --enable-debug --disable-fuzz-binary  CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8"

src/init.cpp

@@ -1450,8 +1450,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
                 strLoadError = _("Error initializing block database");
                 break;
             case ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED:
-                strLoadError = _("Error upgrading chainstate database");
-                break;
+                return InitError(_("Unsupported chainstate database format found. "
+                                   "Please restart with -reindex-chainstate. This will "
+                                   "rebuild the chainstate database."));
             case ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED:
                 strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.");
                 break;

src/node/chainstate.cpp

@@ -90,9 +90,9 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset,
             chainstate->CoinsErrorCatcher().AddReadErrCallback(coins_error_cb);
         }
 
-        // If necessary, upgrade from older database format.
+        // Refuse to load unsupported database format.
         // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
-        if (!chainstate->CoinsDB().Upgrade()) {
+        if (chainstate->CoinsDB().NeedsUpgrade()) {
             return ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED;
         }
 

src/txdb.cpp

@@ -6,7 +6,6 @@
 #include <txdb.h>
 
 #include <chain.h>
-#include <node/ui_interface.h>
 #include <pow.h>
 #include <random.h>
 #include <shutdown.h>
@@ -18,7 +17,6 @@
 #include <stdint.h>
 
 static constexpr uint8_t DB_COIN{'C'};
-static constexpr uint8_t DB_COINS{'c'};
 static constexpr uint8_t DB_BLOCK_FILES{'f'};
 static constexpr uint8_t DB_BLOCK_INDEX{'b'};
 
@@ -29,6 +27,7 @@ static constexpr uint8_t DB_REINDEX_FLAG{'R'};
 static constexpr uint8_t DB_LAST_BLOCK{'l'};
 
 // Keys used in previous version that might still be found in the DB:
+static constexpr uint8_t DB_COINS{'c'};
 static constexpr uint8_t DB_TXINDEX_BLOCK{'T'};
 //               uint8_t DB_TXINDEX{'t'}
 
@@ -50,6 +49,15 @@ std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db)
     return std::nullopt;
 }
 
+bool CCoinsViewDB::NeedsUpgrade()
+{
+    std::unique_ptr<CDBIterator> cursor{m_db->NewIterator()};
+    // DB_COINS was deprecated in v0.15.0, commit
+    // 1088b02f0ccd7358d2b7076bb9e122d59d502d02
+    cursor->Seek(std::make_pair(DB_COINS, uint256{}));
+    return cursor->Valid();
+}
+
 namespace {
 
 struct CoinEntry {
@@ -60,7 +68,7 @@ struct CoinEntry {
     SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
 };
 
-}
+} // namespace
 
 CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) :
     m_db(std::make_unique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)),
@@ -337,125 +345,3 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
 
     return true;
 }
-
-namespace {
-
-//! Legacy class to deserialize pre-pertxout database entries without reindex.
-class CCoins
-{
-public:
-    //! whether transaction is a coinbase
-    bool fCoinBase;
-
-    //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped
-    std::vector<CTxOut> vout;
-
-    //! at which height this transaction was included in the active block chain
-    int nHeight;
-
-    //! empty constructor
-    CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
-
-    template<typename Stream>
-    void Unserialize(Stream &s) {
-        unsigned int nCode = 0;
-        // version
-        unsigned int nVersionDummy;
-        ::Unserialize(s, VARINT(nVersionDummy));
-        // header code
-        ::Unserialize(s, VARINT(nCode));
-        fCoinBase = nCode & 1;
-        std::vector<bool> vAvail(2, false);
-        vAvail[0] = (nCode & 2) != 0;
-        vAvail[1] = (nCode & 4) != 0;
-        unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
-        // spentness bitmask
-        while (nMaskCode > 0) {
-            unsigned char chAvail = 0;
-            ::Unserialize(s, chAvail);
-            for (unsigned int p = 0; p < 8; p++) {
-                bool f = (chAvail & (1 << p)) != 0;
-                vAvail.push_back(f);
-            }
-            if (chAvail != 0)
-                nMaskCode--;
-        }
-        // txouts themself
-        vout.assign(vAvail.size(), CTxOut());
-        for (unsigned int i = 0; i < vAvail.size(); i++) {
-            if (vAvail[i])
-                ::Unserialize(s, Using<TxOutCompression>(vout[i]));
-        }
-        // coinbase height
-        ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
-    }
-};
-
-}
-
-/** Upgrade the database from older formats.
- *
- * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout.
- */
-bool CCoinsViewDB::Upgrade() {
-    std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
-    pcursor->Seek(std::make_pair(DB_COINS, uint256()));
-    if (!pcursor->Valid()) {
-        return true;
-    }
-
-    int64_t count = 0;
-    LogPrintf("Upgrading utxo-set database...\n");
-    LogPrintf("[0%%]..."); /* Continued */
-    uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
-    size_t batch_size = 1 << 24;
-    CDBBatch batch(*m_db);
-    int reportDone = 0;
-    std::pair<unsigned char, uint256> key;
-    std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
-    while (pcursor->Valid()) {
-        if (ShutdownRequested()) {
-            break;
-        }
-        if (pcursor->GetKey(key) && key.first == DB_COINS) {
-            if (count++ % 256 == 0) {
-                uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
-                int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
-                uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
-                if (reportDone < percentageDone/10) {
-                    // report max. every 10% step
-                    LogPrintf("[%d%%]...", percentageDone); /* Continued */
-                    reportDone = percentageDone/10;
-                }
-            }
-            CCoins old_coins;
-            if (!pcursor->GetValue(old_coins)) {
-                return error("%s: cannot parse CCoins record", __func__);
-            }
-            COutPoint outpoint(key.second, 0);
-            for (size_t i = 0; i < old_coins.vout.size(); ++i) {
-                if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
-                    Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
-                    outpoint.n = i;
-                    CoinEntry entry(&outpoint);
-                    batch.Write(entry, newcoin);
-                }
-            }
-            batch.Erase(key);
-            if (batch.SizeEstimate() > batch_size) {
-                m_db->WriteBatch(batch);
-                batch.Clear();
-                m_db->CompactRange(prev_key, key);
-                prev_key = key;
-            }
-            pcursor->Next();
-        } else {
-            break;
-        }
-    }
-    m_db->WriteBatch(batch);
-    m_db->CompactRange({DB_COINS, uint256()}, key);
-    uiInterface.ShowProgress("", 100, false);
-    LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
-    return !ShutdownRequested();
-}

src/txdb.h

@@ -65,8 +65,8 @@ class CCoinsViewDB final : public CCoinsView
     bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
     std::unique_ptr<CCoinsViewCursor> Cursor() const override;
 
-    //! Attempt to update from an older database format. Returns whether an error occurred.
-    bool Upgrade();
+    //! Whether an unsupported database format is used.
+    bool NeedsUpgrade();
     size_t EstimateSize() const override;
 
     //! Dynamically alter the underlying leveldb cache size.

test/README.md

@@ -98,7 +98,7 @@ test/functional/test_runner.py --extended
 In order to run backwards compatibility tests, download the previous node binaries:
 
 ```
-test/get_previous_releases.py -b v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2
+test/get_previous_releases.py -b v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.14.3
 ```
 
 By default, up to 4 tests will be run in parallel by test_runner. To specify

test/functional/feature_unsupported_utxo_db.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test that unsupported utxo db causes an init error.
+
+Previous releases are required by this test, see test/README.md.
+"""
+
+import shutil
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+class UnsupportedUtxoDbTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_previous_releases()
+
+    def setup_network(self):
+        self.add_nodes(
+            self.num_nodes,
+            versions=[
+                140300,  # Last release with previous utxo db format
+                None,  # For MiniWallet, without migration code
+            ],
+        )
+
+    def run_test(self):
+        self.log.info("Create previous version (v0.14.3) utxo db")
+        self.start_node(0)
+        block = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[-1]
+        assert_equal(self.nodes[0].getbestblockhash(), block)
+        assert_equal(self.nodes[0].gettxoutsetinfo()["total_amount"], 50)
+        self.stop_nodes()
+
+        self.log.info("Check init error")
+        legacy_utxos_dir = self.nodes[0].chain_path / "chainstate"
+        legacy_blocks_dir = self.nodes[0].chain_path / "blocks"
+        recent_utxos_dir = self.nodes[1].chain_path / "chainstate"
+        recent_blocks_dir = self.nodes[1].chain_path / "blocks"
+        shutil.copytree(legacy_utxos_dir, recent_utxos_dir)
+        shutil.copytree(legacy_blocks_dir, recent_blocks_dir)
+        self.nodes[1].assert_start_raises_init_error(
+            expected_msg="Error: Unsupported chainstate database format found. "
+            "Please restart with -reindex-chainstate. "
+            "This will rebuild the chainstate database.",
+        )
+
+        self.log.info("Drop legacy utxo db")
+        self.start_node(1, extra_args=["-reindex-chainstate"])
+        assert_equal(self.nodes[1].getbestblockhash(), block)
+        assert_equal(self.nodes[1].gettxoutsetinfo()["total_amount"], 50)
+
+
+if __name__ == "__main__":
+    UnsupportedUtxoDbTest().main()

test/functional/test_runner.py

@@ -307,6 +307,7 @@
     'p2p_ping.py',
     'rpc_scantxoutset.py',
     'feature_txindex_compatibility.py',
+    'feature_unsupported_utxo_db.py',
     'feature_logging.py',
     'feature_anchors.py',
     'feature_coinstatsindex.py --legacy-wallet',

test/get_previous_releases.py

@@ -19,8 +19,12 @@
 import sys
 import hashlib
 
-
 SHA256_SUMS = {
+    "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz",
+    "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz",
+    "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": "bitcoin-0.14.3-osx64.tar.gz",
+    "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz",
+    #
     "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz",
     "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz",
     "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz",