Merge bitcoin#26606: wallet: Implement independent BDB parser

d51fbab wallet, test: Be able to always swap BDB endianness (Ava Chow)
0b75315 test: Test bdb_ro dump of wallet without reset LSNs (Ava Chow)
c1984f1 test: Test dumping dbs with overflow pages (Ava Chow)
fd7b16e test: Test dumps of other endian BDB files (Ava Chow)
6ace3e9 bdb: Be able to make byteswapped databases (Ava Chow)
d987890 Error if LSNs are not reset (Ava Chow)
4d7a3ae Berkeley RO Database fuzz test (TheCharlatan)
3568dce tests: Add BerkeleyRO to db prefix tests (Ava Chow)
70cfbfd wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets (Ava Chow)
dd57713 Add MakeBerkeleyRODatabase (Ava Chow)
6e50bee Implement handling of other endianness in BerkeleyRODatabase (Ava Chow)
cdd61c9 wallet: implement independent BDB deserializer in BerkeleyRODatabase (Ava Chow)
ecba230 wallet: implement BerkeleyRODatabase::Backup (Ava Chow)
0c8e728 wallet: implement BerkeleyROBatch (Ava Chow)
756ff9b wallet: add dummy BerkeleyRODatabase and BerkeleyROBatch classes (Ava Chow)
ca18aea Add AutoFile::seek and tell (Ava Chow)

Pull request description:

  Split from bitcoin#26596

  This PR adds `BerkeleyRODatabase` which is an independent implementation of a BDB file parser. It provides read only access to a BDB file, and can therefore be used as a read only database backend for wallets. This will be used for dumping legacy wallet records and migrating legacy wallets without the need for BDB itself.

  Wallettool's `dump` command is changed to use `BerkeleyRODatabase` instead of `BerkeleyDatabase` (and `CWallet` itself) to demonstrate that this parser works and to test it against the existing wallettool functional tests.

ACKs for top commit:
  josibake:
    reACK bitcoin@d51fbab
  TheCharlatan:
    Re-ACK d51fbab
  furszy:
    reACK d51fbab
  laanwj:
    re-ACK d51fbab
  theStack:
    ACK d51fbab

Tree-SHA512: 1e7b97edf223b2974eed2e9eac1179fc82bb6359e0a66b7d2a0c8b9fa515eae9ea036f1edf7c76cdab2e75ad994962b134b41056ccfbc33b8d54f0859e86657b

Author: fanquake

src/Makefile.am

@@ -348,6 +348,7 @@ BITCOIN_CORE_H = \
   wallet/feebumper.h \
   wallet/fees.h \
   wallet/load.h \
+  wallet/migrate.h \
   wallet/receive.h \
   wallet/rpc/util.h \
   wallet/rpc/wallet.h \
@@ -508,6 +509,7 @@ libbitcoin_wallet_a_SOURCES = \
   wallet/fees.cpp \
   wallet/interfaces.cpp \
   wallet/load.cpp \
+  wallet/migrate.cpp \
   wallet/receive.cpp \
   wallet/rpc/addresses.cpp \
   wallet/rpc/backup.cpp \

src/Makefile.test.include

@@ -204,7 +204,8 @@ FUZZ_WALLET_SRC = \
  wallet/test/fuzz/coincontrol.cpp \
  wallet/test/fuzz/coinselection.cpp \
  wallet/test/fuzz/fees.cpp \
- wallet/test/fuzz/parse_iso8601.cpp
+ wallet/test/fuzz/parse_iso8601.cpp \
+ wallet/test/fuzz/wallet_bdb_parser.cpp
 
 if USE_SQLITE
 FUZZ_WALLET_SRC += \

src/bitcoin-wallet.cpp

@@ -40,6 +40,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman)
     argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-withinternalbdb", "Use the internal Berkeley DB parser when dumping a Berkeley DB wallet file (default: false)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
 
     argsman.AddCommand("info", "Get wallet info");
     argsman.AddCommand("create", "Create new wallet file");

src/dummywallet.cpp

@@ -53,6 +53,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const
         "-walletrejectlongchains",
         "-walletcrosschain",
         "-unsafesqlitesync",
+        "-swapbdbendian",
     });
 }
 

src/streams.cpp

@@ -21,6 +21,28 @@ std::size_t AutoFile::detail_fread(Span<std::byte> dst)
     }
 }
 
+void AutoFile::seek(int64_t offset, int origin)
+{
+    if (IsNull()) {
+        throw std::ios_base::failure("AutoFile::seek: file handle is nullptr");
+    }
+    if (std::fseek(m_file, offset, origin) != 0) {
+        throw std::ios_base::failure(feof() ? "AutoFile::seek: end of file" : "AutoFile::seek: fseek failed");
+    }
+}
+
+int64_t AutoFile::tell()
+{
+    if (IsNull()) {
+        throw std::ios_base::failure("AutoFile::tell: file handle is nullptr");
+    }
+    int64_t r{std::ftell(m_file)};
+    if (r < 0) {
+        throw std::ios_base::failure("AutoFile::tell: ftell failed");
+    }
+    return r;
+}
+
 void AutoFile::read(Span<std::byte> dst)
 {
     if (detail_fread(dst) != dst.size()) {

src/streams.h

@@ -435,6 +435,9 @@ class AutoFile
     /** Implementation detail, only used internally. */
     std::size_t detail_fread(Span<std::byte> dst);
 
+    void seek(int64_t offset, int origin);
+    int64_t tell();
+
     //
     // Stream subset
     //

src/wallet/bdb.cpp

@@ -65,6 +65,8 @@ RecursiveMutex cs_db;
 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment.
 } // namespace
 
+static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234};
+
 bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
 {
     return memcmp(value, &rhs.value, sizeof(value)) == 0;
@@ -300,7 +302,11 @@ static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt)
 }
 
 BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) :
-    WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb)
+    WalletDatabase(),
+    env(std::move(env)),
+    m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP),
+    m_filename(std::move(filename)),
+    m_max_log_mb(options.max_log_mb)
 {
     auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this));
     assert(inserted.second);
@@ -389,6 +395,10 @@ void BerkeleyDatabase::Open()
                 }
             }
 
+            if (m_byteswap) {
+                pdb_temp->set_lorder(REVERSE_BYTE_ORDER);
+            }
+
             ret = pdb_temp->open(nullptr,                             // Txn pointer
                             fMockDb ? nullptr : strFile.c_str(),      // Filename
                             fMockDb ? strFile.c_str() : "main",       // Logical db name
@@ -521,6 +531,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip)
                     BerkeleyBatch db(*this, true);
                     std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
 
+                    if (m_byteswap) {
+                        pdbCopy->set_lorder(REVERSE_BYTE_ORDER);
+                    }
+
                     int ret = pdbCopy->open(nullptr,               // Txn pointer
                                             strFileRes.c_str(), // Filename
                                             "main",             // Logical db name

src/wallet/bdb.h

@@ -147,6 +147,9 @@ class BerkeleyDatabase : public WalletDatabase
     /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */
     std::unique_ptr<Db> m_db;
 
+    // Whether to byteswap
+    bool m_byteswap;
+
     fs::path m_filename;
     int64_t m_max_log_mb;
 

src/wallet/db.cpp

@@ -16,6 +16,9 @@
 #include <vector>
 
 namespace wallet {
+bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); }
+bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; }
+
 std::vector<fs::path> ListDatabases(const fs::path& wallet_dir)
 {
     std::vector<fs::path> paths;

src/wallet/db.h

@@ -20,6 +20,12 @@ class ArgsManager;
 struct bilingual_str;
 
 namespace wallet {
+// BytePrefix compares equality with other byte spans that begin with the same prefix.
+struct BytePrefix {
+    Span<const std::byte> prefix;
+};
+bool operator<(BytePrefix a, Span<const std::byte> b);
+bool operator<(Span<const std::byte> a, BytePrefix b);
 
 class DatabaseCursor
 {
@@ -177,6 +183,8 @@ class WalletDatabase
 enum class DatabaseFormat {
     BERKELEY,
     SQLITE,
+    BERKELEY_RO,
+    BERKELEY_SWAP,
 };
 
 struct DatabaseOptions {