Merge #7871: Manual block file pruning.

afffeea fixup! Add pruneblockchain RPC to enable manual block file pruning. (Russell Yanofsky)
1fc4ec7 Add pruneblockchain RPC to enable manual block file pruning. (mrbandrews)

Author: laanwj

doc/man/bitcoin-qt.1

@@ -75,11 +75,13 @@ Specify pid file (default: bitcoind.pid)
 .HP
 \fB\-prune=\fR<n>
 .IP
-Reduce storage requirements by pruning (deleting) old blocks. This mode
-is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
-this setting requires re\-downloading the entire blockchain.
-(default: 0 = disable pruning blocks, >550 = target size in MiB
-to use for block files)
+Reduce storage requirements by enabling pruning (deleting) of old blocks.
+This allows the pruneblockchain RPC to be called to delete specific blocks,
+and enables automatic pruning of old blocks if a target size in MiB is
+provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
+Warning: Reverting this setting requires re\-downloading the entire blockchain.
+(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
+automatically prune block files to stay under the specified target size in MiB)
 .HP
 \fB\-reindex\-chainstate\fR
 .IP

doc/man/bitcoind.1

@@ -80,11 +80,13 @@ Specify pid file (default: bitcoind.pid)
 .HP
 \fB\-prune=\fR<n>
 .IP
-Reduce storage requirements by pruning (deleting) old blocks. This mode
-is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
-this setting requires re\-downloading the entire blockchain.
-(default: 0 = disable pruning blocks, >550 = target size in MiB
-to use for block files)
+Reduce storage requirements by enabling pruning (deleting) of old blocks.
+This allows the pruneblockchain RPC to be called to delete specific blocks,
+and enables automatic pruning of old blocks if a target size in MiB is
+provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
+Warning: Reverting this setting requires re\-downloading the entire blockchain.
+(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
+automatically prune block files to stay under the specified target size in MiB)
 .HP
 \fB\-reindex\-chainstate\fR
 .IP

qa/rpc-tests/pruning.py

@@ -25,7 +25,7 @@ class PruneTest(BitcoinTestFramework):
     def __init__(self):
         super().__init__()
         self.setup_clean_chain = True
-        self.num_nodes = 3
+        self.num_nodes = 6
 
         # Cache for utxos, as the listunspent may take a long time later in the test
         self.utxo_cache_0 = []
@@ -43,10 +43,22 @@ def setup_network(self):
         self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900))
         self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
 
+        # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
+        self.nodes.append(start_node(3, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
+        self.nodes.append(start_node(4, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
+
+        # Create nodes 5 to test wallet in prune mode, but do not connect
+        self.nodes.append(start_node(5, self.options.tmpdir, ["-debug=0", "-prune=550"]))
+
+        # Determine default relay fee
+        self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
+
         connect_nodes(self.nodes[0], 1)
         connect_nodes(self.nodes[1], 2)
         connect_nodes(self.nodes[2], 0)
-        sync_blocks(self.nodes[0:3])
+        connect_nodes(self.nodes[0], 3)
+        connect_nodes(self.nodes[0], 4)
+        sync_blocks(self.nodes[0:5])
 
     def create_big_chain(self):
         # Start by creating some coinbases we can spend later
@@ -57,7 +69,7 @@ def create_big_chain(self):
         for i in range(645):
             mine_large_block(self.nodes[0], self.utxo_cache_0)
 
-        sync_blocks(self.nodes[0:3])
+        sync_blocks(self.nodes[0:5])
 
     def test_height_min(self):
         if not os.path.isfile(self.prunedir+"blk00000.dat"):
@@ -212,6 +224,103 @@ def reorg_back(self):
         # Verify we can now have the data for a block previously pruned
         assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
 
+    def manual_test(self, node_number, use_timestamp):
+        # at this point, node has 995 blocks and has not yet run in prune mode
+        node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0"], timewait=900)
+        assert_equal(node.getblockcount(), 995)
+        assert_raises_message(JSONRPCException, "not in prune mode", node.pruneblockchain, 500)
+        stop_node(node, node_number)
+
+        # now re-start in manual pruning mode
+        node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=1"], timewait=900)
+        assert_equal(node.getblockcount(), 995)
+
+        def height(index):
+            if use_timestamp:
+                return node.getblockheader(node.getblockhash(index))["time"]
+            else:
+                return index
+
+        def has_block(index):
+            return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index))
+
+        # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
+        assert_raises_message(JSONRPCException, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
+
+        # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
+        node.generate(6)
+
+        # negative and zero inputs should raise an exception
+        try:
+            node.pruneblockchain(-10)
+            raise AssertionError("pruneblockchain(-10) should have failed.")
+        except:
+            pass
+
+        try:
+            node.pruneblockchain(0)
+            raise AssertionError("pruneblockchain(0) should have failed.")
+        except:
+            pass
+
+        # height=100 too low to prune first block file so this is a no-op
+        node.pruneblockchain(height(100))
+        if not has_block(0):
+            raise AssertionError("blk00000.dat is missing when should still be there")
+
+        # height=500 should prune first file
+        node.pruneblockchain(height(500))
+        if has_block(0):
+            raise AssertionError("blk00000.dat is still there, should be pruned by now")
+        if not has_block(1):
+            raise AssertionError("blk00001.dat is missing when should still be there")
+
+        # height=650 should prune second file
+        node.pruneblockchain(height(650))
+        if has_block(1):
+            raise AssertionError("blk00001.dat is still there, should be pruned by now")
+
+        # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
+        node.pruneblockchain(height(1000))
+        if not has_block(2):
+            raise AssertionError("blk00002.dat is still there, should be pruned by now")
+
+        # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
+        node.generate(288)
+        node.pruneblockchain(height(1000))
+        if has_block(2):
+            raise AssertionError("blk00002.dat is still there, should be pruned by now")
+        if has_block(3):
+            raise AssertionError("blk00003.dat is still there, should be pruned by now")
+
+        # stop node, start back up with auto-prune at 550MB, make sure still runs
+        stop_node(node, node_number)
+        self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=550"], timewait=900)
+
+        print("Success")
+
+    def wallet_test(self):
+        # check that the pruning node's wallet is still in good shape
+        print("Stop and start pruning node to trigger wallet rescan")
+        try:
+            stop_node(self.nodes[2], 2)
+            start_node(2, self.options.tmpdir, ["-debug=1","-prune=550"])
+            print("Success")
+        except Exception as detail:
+            raise AssertionError("Wallet test: unable to re-start the pruning node")
+
+        # check that wallet loads loads successfully when restarting a pruned node after IBD.
+        # this was reported to fail in #7494.
+        print ("Syncing node 5 to test wallet")
+        connect_nodes(self.nodes[0], 5)
+        nds = [self.nodes[0], self.nodes[5]]
+        sync_blocks(nds)
+        try:
+            stop_node(self.nodes[5],5) #stop and start to trigger rescan
+            start_node(5, self.options.tmpdir, ["-debug=1","-prune=550"])
+            print ("Success")
+        except Exception as detail:
+            raise AssertionError("Wallet test: unable to re-start node5")
 
     def run_test(self):
         print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
@@ -226,6 +335,10 @@ def run_test(self):
         # Start by mining a simple chain that all nodes have
         # N0=N1=N2 **...*(995)
 
+        # stop manual-pruning node with 995 blocks
+        stop_node(self.nodes[3],3)
+        stop_node(self.nodes[4],4)
+
         print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
         self.test_height_min()
         # Extend this chain past the PruneAfterHeight
@@ -308,6 +421,15 @@ def run_test(self):
         #
         # N1 doesn't change because 1033 on main chain (*) is invalid
 
+        print("Test manual pruning with block indices")
+        self.manual_test(3, use_timestamp=False)
+
+        print("Test manual pruning with timestamps")
+        self.manual_test(4, use_timestamp=True)
+
+        print("Test wallet re-scan")
+        self.wallet_test()
+
         print("Done")
 
 if __name__ == '__main__':

src/init.cpp

@@ -351,9 +351,9 @@ std::string HelpMessage(HelpMessageMode mode)
 #ifndef WIN32
     strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
 #endif
-    strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. "
+    strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "
-            "(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
+            "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
     strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
     strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk"));
 #ifndef WIN32
@@ -936,12 +936,16 @@ bool AppInitParameterInteraction()
         nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;
 
     // block pruning; get the amount of disk space (in MiB) to allot for block & undo files
-    int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
-    if (nSignedPruneTarget < 0) {
+    int64_t nPruneArg = GetArg("-prune", 0);
+    if (nPruneArg < 0) {
         return InitError(_("Prune cannot be configured with a negative value."));
     }
-    nPruneTarget = (uint64_t) nSignedPruneTarget;
-    if (nPruneTarget) {
+    nPruneTarget = (uint64_t) nPruneArg * 1024 * 1024;
+    if (nPruneArg == 1) {  // manual pruning: -prune=1
+        LogPrintf("Block pruning enabled.  Use RPC call pruneblockchain(height) to manually prune block and undo files.\n");
+        nPruneTarget = std::numeric_limits<uint64_t>::max();
+        fPruneMode = true;
+    } else if (nPruneTarget) {
         if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
             return InitError(strprintf(_("Prune configured below the minimum of %d MiB.  Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
         }

src/rpc/blockchain.cpp

@@ -814,6 +814,46 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
     return true;
 }
 
+UniValue pruneblockchain(const JSONRPCRequest& request)
+{
+    if (request.fHelp || request.params.size() != 1)
+        throw runtime_error(
+            "pruneblockchain\n"
+            "\nArguments:\n"
+            "1. \"height\"       (numeric, required) The block height to prune up to. May be set to a discrete height, or to a unix timestamp to prune based on block time.\n");
+
+    if (!fPruneMode)
+        throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Cannot prune blocks because node is not in prune mode.");
+
+    LOCK(cs_main);
+
+    int heightParam = request.params[0].get_int();
+    if (heightParam < 0)
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative block height.");
+
+    // Height value more than a billion is too high to be a block height, and
+    // too low to be a block time (corresponds to timestamp from Sep 2001).
+    if (heightParam > 1000000000) {
+        CBlockIndex* pindex = chainActive.FindLatestBefore(heightParam);
+        if (!pindex) {
+            throw JSONRPCError(RPC_INTERNAL_ERROR, "Could not find block before specified timestamp.");
+        }
+        heightParam = pindex->nHeight;
+    }
+
+    unsigned int height = (unsigned int) heightParam;
+    unsigned int chainHeight = (unsigned int) chainActive.Height();
+    if (chainHeight < Params().PruneAfterHeight())
+        throw JSONRPCError(RPC_INTERNAL_ERROR, "Blockchain is too short for pruning.");
+    else if (height > chainHeight)
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height.");
+    else if (height > chainHeight - MIN_BLOCKS_TO_KEEP)
+        LogPrint("rpc", "Attempt to prune blocks close to the tip.  Retaining the minimum number of blocks.");
+
+    PruneBlockFilesManual(height);
+    return NullUniValue;
+}
+
 UniValue gettxoutsetinfo(const JSONRPCRequest& request)
 {
     if (request.fHelp || request.params.size() != 0)
@@ -1384,6 +1424,7 @@ static const CRPCCommand commands[] =
     { "blockchain",         "getrawmempool",          &getrawmempool,          true,  {"verbose"} },
     { "blockchain",         "gettxout",               &gettxout,               true,  {"txid","n","include_mempool"} },
     { "blockchain",         "gettxoutsetinfo",        &gettxoutsetinfo,        true,  {} },
+    { "blockchain",         "pruneblockchain",        &pruneblockchain,        true,  {"height"} },
     { "blockchain",         "verifychain",            &verifychain,            true,  {"checklevel","nblocks"} },
 
     { "blockchain",         "preciousblock",          &preciousblock,          true,  {"blockhash"} },

src/rpc/client.cpp

@@ -103,6 +103,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "importmulti", 1, "options" },
     { "verifychain", 0, "checklevel" },
     { "verifychain", 1, "nblocks" },
+    { "pruneblockchain", 0, "height" },
     { "keypoolrefill", 0, "newsize" },
     { "getrawmempool", 0, "verbose" },
     { "estimatefee", 0, "nblocks" },

src/validation.cpp

@@ -185,7 +185,8 @@ enum FlushStateMode {
 };
 
 // See definition for documentation
-bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode);
+bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0);
+void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
 
 bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime)
 {
@@ -1934,7 +1935,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
  * if they're too large, if it's been a while since the last write,
  * or always and in all cases if we're in prune mode and are deleting files.
  */
-bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
+bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight) {
     int64_t nMempoolUsage = mempool.DynamicMemoryUsage();
     const CChainParams& chainparams = Params();
     LOCK2(cs_main, cs_LastBlockFile);
@@ -1944,9 +1945,13 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
     std::set<int> setFilesToPrune;
     bool fFlushForPrune = false;
     try {
-    if (fPruneMode && fCheckForPruning && !fReindex) {
-        FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
-        fCheckForPruning = false;
+    if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) {
+        if (nManualPruneHeight > 0) {
+            FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight);
+        } else {
+            FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
+            fCheckForPruning = false;
+        }
         if (!setFilesToPrune.empty()) {
             fFlushForPrune = true;
             if (!fHavePruned) {
@@ -3247,6 +3252,35 @@ void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
     }
 }
 
+/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
+void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight)
+{
+    assert(fPruneMode && nManualPruneHeight > 0);
+
+    LOCK2(cs_main, cs_LastBlockFile);
+    if (chainActive.Tip() == NULL)
+        return;
+
+    // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
+    unsigned int nLastBlockWeCanPrune = min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP);
+    int count=0;
+    for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
+        if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
+            continue;
+        PruneOneBlockFile(fileNumber);
+        setFilesToPrune.insert(fileNumber);
+        count++;
+    }
+    LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count);
+}
+
+/* This function is called from the RPC code for pruneblockchain */
+void PruneBlockFilesManual(int nManualPruneHeight)
+{
+    CValidationState state;
+    FlushStateToDisk(state, FLUSH_STATE_NONE, nManualPruneHeight);
+}
+
 /* Calculate the block/rev files that should be deleted to remain under target*/
 void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight)
 {

src/validation.h

@@ -309,6 +309,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash);
 void FlushStateToDisk();
 /** Prune block files and flush state to disk. */
 void PruneAndFlush();
+/** Prune block files up to a given height */
+void PruneBlockFilesManual(int nPruneUpToHeight);
 
 /** (try to) add transaction to memory pool **/
 bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,