Add -blocksxor boolean option

Author: MarcoFalke

doc/files.md

@@ -47,8 +47,9 @@ Subdirectory       | File(s)               | Description
 -------------------|-----------------------|------------
 `blocks/`          |                       | Blocks directory; can be specified by `-blocksdir` option (except for `blocks/index/`)
 `blocks/index/`    | LevelDB database      | Block index; `-blocksdir` option does not affect this path
-`blocks/`          | `blkNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file)
+`blocks/`          | `blkNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Actual Bitcoin blocks (dumped in network format, 128 MiB per file)
 `blocks/`          | `revNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Block undo data (custom format)
+`blocks/`          | `xor.dat`             | Rolling XOR pattern for block and undo data files
 `chainstate/`      | LevelDB database      | Blockchain state (a compact representation of all currently unspent transaction outputs (UTXOs) and metadata about the transactions they are from)
 `indexes/txindex/` | LevelDB database      | Transaction index; *optional*, used if `-txindex=1`
 `indexes/blockfilter/basic/db/` | LevelDB database      | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic`

doc/release-notes-28052.md

@@ -0,0 +1,6 @@
+Blockstorage
+============
+
+Block files are now XOR'd by default with a key stored in the blocksdir.
+Previous releases of Bitcoin Core or previous external software will not be able to read the blocksdir with a non-zero XOR-key.
+Refer to the `-blocksxor` help for more details.

src/init.cpp

@@ -469,6 +469,13 @@ void SetupServerArgs(ArgsManager& argsman)
 #endif
     argsman.AddArg("-assumevalid=<hex>", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), signetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-blocksdir=<dir>", "Specify directory to hold blocks subdirectory for *.dat files (default: <datadir>)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-blocksxor",
+                   strprintf("Whether an XOR-key applies to blocksdir *.dat files. "
+                             "The created XOR-key will be zeros for an existing blocksdir or when `-blocksxor=0` is "
+                             "set, and random for a freshly initialized blocksdir. "
+                             "(default: %u)",
+                             kernel::DEFAULT_XOR_BLOCKSDIR),
+                   ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
 #if HAVE_SYSTEM
     argsman.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -1534,7 +1541,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
         LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024));
 
-        node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts);
+        try {
+            node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts);
+        } catch (std::exception& e) {
+            return InitError(strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what()));
+        }
         ChainstateManager& chainman = *node.chainman;
 
         // This is defined and set here instead of inline in validation.h to avoid a hard

src/kernel/blockmanager_opts.h

@@ -14,12 +14,15 @@ class CChainParams;
 
 namespace kernel {
 
+static constexpr bool DEFAULT_XOR_BLOCKSDIR{true};
+
 /**
  * An options struct for `BlockManager`, more ergonomically referred to as
  * `BlockManager::Options` due to the using-declaration in `BlockManager`.
  */
 struct BlockManagerOpts {
     const CChainParams& chainparams;
+    bool use_xor{DEFAULT_XOR_BLOCKSDIR};
     uint64_t prune_target{0};
     bool fast_prune{false};
     const fs::path blocks_dir;

src/node/blockmanager_args.cpp

@@ -16,6 +16,7 @@
 namespace node {
 util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts)
 {
+    if (auto value{args.GetBoolArg("-blocksxor")}) opts.use_xor = *value;
     // block pruning; get the amount of disk space (in MiB) to allot for block & undo files
     int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)};
     if (nPruneArg < 0) {

src/node/blockstorage.cpp

@@ -19,6 +19,7 @@
 #include <pow.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
+#include <random.h>
 #include <reverse_iterator.h>
 #include <serialize.h>
 #include <signet.h>
@@ -1144,9 +1145,43 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight)
     return blockPos;
 }
 
+static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
+{
+    // Bytes are serialized without length indicator, so this is also the exact
+    // size of the XOR-key file.
+    std::array<std::byte, 8> xor_key{};
+
+    if (opts.use_xor && fs::is_empty(opts.blocks_dir)) {
+        // Only use random fresh key when the boolean option is set and on the
+        // very first start of the program.
+        FastRandomContext{}.fillrand(xor_key);
+    }
+
+    const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
+    if (fs::exists(xor_key_path)) {
+        // A pre-existing xor key file has priority.
+        AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
+        xor_key_file >> xor_key;
+    } else {
+        // Create initial or missing xor key file
+        AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "wbx")};
+        xor_key_file << xor_key;
+    }
+    // If the user disabled the key, it must be zero.
+    if (!opts.use_xor && xor_key != decltype(xor_key){}) {
+        throw std::runtime_error{
+            strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
+                      "Stored key: '%s', stored path: '%s'.",
+                      HexStr(xor_key), fs::PathToString(xor_key_path)),
+        };
+    }
+    LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
+    return std::vector<std::byte>{xor_key.begin(), xor_key.end()};
+}
+
 BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
     : m_prune_mode{opts.prune_target > 0},
-      m_xor_key{},
+      m_xor_key{InitBlocksdirXorKey(opts)},
       m_opts{std::move(opts)},
       m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
       m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},

test/functional/feature_loadblock.py

@@ -26,6 +26,10 @@ def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 2
         self.supports_cli = False
+        self.extra_args = [
+            ["-blocksxor=0"],  # TODO: The linearize scripts should be adjusted to apply any XOR
+            [],
+        ]
 
     def run_test(self):
         self.nodes[1].setnetworkactive(state=False)

test/functional/feature_reindex.py

@@ -39,9 +39,19 @@ def out_of_order(self):
         # we're generating them rather than getting them from peers), so to
         # test out-of-order handling, swap blocks 1 and 2 on disk.
         blk0 = self.nodes[0].blocks_path / "blk00000.dat"
+        with open(self.nodes[0].blocks_path / "xor.dat", "rb") as xor_f:
+            NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size()
+            xor_dat = xor_f.read(NUM_XOR_BYTES)
+
+        def util_xor(data, key, *, offset):
+            data = bytearray(data)
+            for i in range(len(data)):
+                data[i] ^= key[(i + offset) % len(key)]
+            return bytes(data)
+
         with open(blk0, 'r+b') as bf:
             # Read at least the first few blocks (including genesis)
-            b = bf.read(2000)
+            b = util_xor(bf.read(2000), xor_dat, offset=0)
 
             # Find the offsets of blocks 2, 3, and 4 (the first 3 blocks beyond genesis)
             # by searching for the regtest marker bytes (see pchMessageStart).
@@ -55,12 +65,12 @@ def find_block(b, start):
             b4_start = find_block(b, b3_start)
 
             # Blocks 2 and 3 should be the same size.
-            assert_equal(b3_start-b2_start, b4_start-b3_start)
+            assert_equal(b3_start - b2_start, b4_start - b3_start)
 
             # Swap the second and third blocks (don't disturb the genesis block).
             bf.seek(b2_start)
-            bf.write(b[b3_start:b4_start])
-            bf.write(b[b2_start:b3_start])
+            bf.write(util_xor(b[b3_start:b4_start], xor_dat, offset=b2_start))
+            bf.write(util_xor(b[b2_start:b3_start], xor_dat, offset=b3_start))
 
         # The reindexing code should detect and accommodate out of order blocks.
         with self.nodes[0].assert_debug_log([