Skip to content

Commit 66c08e7

Browse files
committed
Merge bitcoin/bitcoin#24865: rpc: Enable wallet import on pruned nodes and add test
564b580 test: Introduce MIN_BLOCKS_TO_KEEP constant (Aurèle Oulès) 71d9a7c test: Wallet imports on pruned nodes (Aurèle Oulès) e6906fc rpc: Enable wallet import on pruned nodes (Aurèle Oulès) Pull request description: Reopens #16037 I have rebased the PR, addressed the comments of the original PR and added a functional test. > Before this change importwallet fails if any block is pruned. This PR makes it possible to importwallet if all required blocks aren't pruned. This is possible because the dump format includes key timestamps. For reviewers: `python test/functional/wallet_pruning.py --nocleanup` will generate a large blockchain (~700MB) that can be used to manually test wallet imports on a pruned node. Node0 is not pruned, while node1 is. ACKs for top commit: kouloumos: ACK 564b580 achow101: reACK 564b580 furszy: ACK 564b580 w0xlt: ACK bitcoin/bitcoin@564b580 Tree-SHA512: b345a6c455fcb6581cdaa5f7a55d79e763a55cb08c81d66be5b12794985d79cd51b9b39bdcd0f7ba0a2a2643e9b2ddc49310ff03d16b430df2f74e990800eabf
2 parents 7386da7 + 564b580 commit 66c08e7

File tree

5 files changed

+189
-31
lines changed

5 files changed

+189
-31
lines changed

src/wallet/rpc/backup.cpp

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,22 @@ static void RescanWallet(CWallet& wallet, const WalletRescanReserver& reserver,
9393
}
9494
}
9595

96+
static void EnsureBlockDataFromTime(const CWallet& wallet, int64_t timestamp)
97+
{
98+
auto& chain{wallet.chain()};
99+
if (!chain.havePruned()) {
100+
return;
101+
}
102+
103+
int height{0};
104+
const bool found{chain.findFirstBlockWithTimeAndHeight(timestamp - TIMESTAMP_WINDOW, 0, FoundBlock().height(height))};
105+
106+
uint256 tip_hash{WITH_LOCK(wallet.cs_wallet, return wallet.GetLastBlockHash())};
107+
if (found && !chain.hasBlocks(tip_hash, height)) {
108+
throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Pruned blocks from height %d required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", height));
109+
}
110+
}
111+
96112
RPCHelpMan importprivkey()
97113
{
98114
return RPCHelpMan{"importprivkey",
@@ -504,13 +520,6 @@ RPCHelpMan importwallet()
504520

505521
EnsureLegacyScriptPubKeyMan(*pwallet, true);
506522

507-
if (pwallet->chain().havePruned()) {
508-
// Exit early and print an error.
509-
// If a block is pruned after this check, we will import the key(s),
510-
// but fail the rescan with a generic error.
511-
throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned");
512-
}
513-
514523
WalletRescanReserver reserver(*pwallet);
515524
if (!reserver.reserve()) {
516525
throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait.");
@@ -565,15 +574,18 @@ RPCHelpMan importwallet()
565574
fLabel = true;
566575
}
567576
}
577+
nTimeBegin = std::min(nTimeBegin, nTime);
568578
keys.push_back(std::make_tuple(key, nTime, fLabel, strLabel));
569579
} else if(IsHex(vstr[0])) {
570580
std::vector<unsigned char> vData(ParseHex(vstr[0]));
571581
CScript script = CScript(vData.begin(), vData.end());
572582
int64_t birth_time = ParseISO8601DateTime(vstr[1]);
583+
if (birth_time > 0) nTimeBegin = std::min(nTimeBegin, birth_time);
573584
scripts.push_back(std::pair<CScript, int64_t>(script, birth_time));
574585
}
575586
}
576587
file.close();
588+
EnsureBlockDataFromTime(*pwallet, nTimeBegin);
577589
// We now know whether we are importing private keys, so we can error if private keys are disabled
578590
if (keys.size() > 0 && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) {
579591
pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI
@@ -602,8 +614,6 @@ RPCHelpMan importwallet()
602614

603615
if (has_label)
604616
pwallet->SetAddressBook(PKHash(keyid), label, "receive");
605-
606-
nTimeBegin = std::min(nTimeBegin, time);
607617
progress++;
608618
}
609619
for (const auto& script_pair : scripts) {
@@ -616,9 +626,6 @@ RPCHelpMan importwallet()
616626
fGood = false;
617627
continue;
618628
}
619-
if (time > 0) {
620-
nTimeBegin = std::min(nTimeBegin, time);
621-
}
622629

623630
progress++;
624631
}

test/functional/feature_pruning.py

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@
1010
"""
1111
import os
1212

13-
from test_framework.blocktools import create_coinbase
14-
from test_framework.messages import CBlock
13+
from test_framework.blocktools import (
14+
MIN_BLOCKS_TO_KEEP,
15+
create_block,
16+
create_coinbase,
17+
)
1518
from test_framework.script import (
1619
CScript,
1720
OP_NOP,
@@ -48,21 +51,7 @@ def mine_large_blocks(node, n):
4851
previousblockhash = int(best_block["hash"], 16)
4952

5053
for _ in range(n):
51-
# Build the coinbase transaction (with large scriptPubKey)
52-
coinbase_tx = create_coinbase(height)
53-
coinbase_tx.vin[0].nSequence = 2 ** 32 - 1
54-
coinbase_tx.vout[0].scriptPubKey = big_script
55-
coinbase_tx.rehash()
56-
57-
# Build the block
58-
block = CBlock()
59-
block.nVersion = best_block["version"]
60-
block.hashPrevBlock = previousblockhash
61-
block.nTime = mine_large_blocks.nTime
62-
block.nBits = int('207fffff', 16)
63-
block.nNonce = 0
64-
block.vtx = [coinbase_tx]
65-
block.hashMerkleRoot = block.calc_merkle_root()
54+
block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
6655
block.solve()
6756

6857
# Submit to the node
@@ -345,7 +334,7 @@ def has_block(index):
345334
assert has_block(2), "blk00002.dat is still there, should be pruned by now"
346335

347336
# advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
348-
self.generate(node, 288, sync_fun=self.no_op)
337+
self.generate(node, MIN_BLOCKS_TO_KEEP, sync_fun=self.no_op)
349338
prune(1000)
350339
assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
351340
assert not has_block(3), "blk00003.dat is still there, should be pruned by now"

test/functional/test_framework/blocktools.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161

6262
NORMAL_GBT_REQUEST_PARAMS = {"rules": ["segwit"]}
6363
VERSIONBITS_LAST_OLD_BLOCK_VERSION = 4
64+
MIN_BLOCKS_TO_KEEP = 288
6465

6566

6667
def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl=None, txlist=None):
@@ -120,7 +121,7 @@ def script_BIP34_coinbase_height(height):
120121
return CScript([CScriptNum(height)])
121122

122123

123-
def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50):
124+
def create_coinbase(height, pubkey=None, *, script_pubkey=None, extra_output_script=None, fees=0, nValue=50):
124125
"""Create a coinbase transaction.
125126
126127
If pubkey is passed in, the coinbase output will be a P2PK output;
@@ -138,6 +139,8 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu
138139
coinbaseoutput.nValue += fees
139140
if pubkey is not None:
140141
coinbaseoutput.scriptPubKey = key_to_p2pk_script(pubkey)
142+
elif script_pubkey is not None:
143+
coinbaseoutput.scriptPubKey = script_pubkey
141144
else:
142145
coinbaseoutput.scriptPubKey = CScript([OP_TRUE])
143146
coinbase.vout = [coinbaseoutput]

test/functional/test_runner.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
'feature_pruning.py',
8686
'feature_dbcrash.py',
8787
'feature_index_prune.py',
88+
'wallet_pruning.py --legacy-wallet',
8889
]
8990

9091
BASE_SCRIPTS = [

test/functional/wallet_pruning.py

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) 2022 The Bitcoin Core developers
3+
# Distributed under the MIT software license, see the accompanying
4+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5+
6+
"""Test wallet import on pruned node."""
7+
import os
8+
9+
from test_framework.util import assert_equal, assert_raises_rpc_error
10+
from test_framework.blocktools import (
11+
COINBASE_MATURITY,
12+
create_block
13+
)
14+
from test_framework.blocktools import create_coinbase
15+
from test_framework.test_framework import BitcoinTestFramework
16+
17+
from test_framework.script import (
18+
CScript,
19+
OP_RETURN,
20+
OP_TRUE,
21+
)
22+
23+
class WalletPruningTest(BitcoinTestFramework):
24+
def add_options(self, parser):
25+
self.add_wallet_options(parser, descriptors=False)
26+
27+
def set_test_params(self):
28+
self.setup_clean_chain = True
29+
self.num_nodes = 2
30+
self.wallet_names = []
31+
self.extra_args = [
32+
[], # node dedicated to mining
33+
['-prune=550'], # node dedicated to testing pruning
34+
]
35+
36+
def skip_test_if_missing_module(self):
37+
self.skip_if_no_wallet()
38+
self.skip_if_no_bdb()
39+
40+
def mine_large_blocks(self, node, n):
41+
# Get the block parameters for the first block
42+
best_block = node.getblock(node.getbestblockhash())
43+
height = int(best_block["height"]) + 1
44+
self.nTime = max(self.nTime, int(best_block["time"])) + 1
45+
previousblockhash = int(best_block["hash"], 16)
46+
big_script = CScript([OP_RETURN] + [OP_TRUE] * 950000)
47+
for _ in range(n):
48+
block = create_block(hashprev=previousblockhash, ntime=self.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
49+
block.solve()
50+
51+
# Submit to the node
52+
node.submitblock(block.serialize().hex())
53+
54+
previousblockhash = block.sha256
55+
height += 1
56+
57+
# Simulate 10 minutes of work time per block
58+
# Important for matching a timestamp with a block +- some window
59+
self.nTime += 600
60+
for n in self.nodes:
61+
if n.running:
62+
n.setmocktime(self.nTime) # Update node's time to accept future blocks
63+
self.sync_all()
64+
65+
def test_wallet_import_pruned(self, wallet_name):
66+
self.log.info("Make sure we can import wallet when pruned and required blocks are still available")
67+
68+
wallet_file = wallet_name + ".dat"
69+
wallet_birthheight = self.get_birthheight(wallet_file)
70+
71+
# Verify that the block at wallet's birthheight is available at the pruned node
72+
self.nodes[1].getblock(self.nodes[1].getblockhash(wallet_birthheight))
73+
74+
# Import wallet into pruned node
75+
self.nodes[1].createwallet(wallet_name="wallet_pruned", descriptors=False, load_on_startup=True)
76+
self.nodes[1].importwallet(os.path.join(self.nodes[0].datadir, wallet_file))
77+
78+
# Make sure that prune node's wallet correctly accounts for balances
79+
assert_equal(self.nodes[1].getbalance(), self.nodes[0].getbalance())
80+
81+
self.log.info("- Done")
82+
83+
def test_wallet_import_pruned_with_missing_blocks(self, wallet_name):
84+
self.log.info("Make sure we cannot import wallet when pruned and required blocks are not available")
85+
86+
wallet_file = wallet_name + ".dat"
87+
wallet_birthheight = self.get_birthheight(wallet_file)
88+
89+
# Verify that the block at wallet's birthheight is not available at the pruned node
90+
assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[1].getblock, self.nodes[1].getblockhash(wallet_birthheight))
91+
92+
# Make sure wallet cannot be imported because of missing blocks
93+
# This will try to rescan blocks `TIMESTAMP_WINDOW` (2h) before the wallet birthheight.
94+
# There are 6 blocks an hour, so 11 blocks (excluding birthheight).
95+
assert_raises_rpc_error(-4, f"Pruned blocks from height {wallet_birthheight - 11} required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[1].importwallet, os.path.join(self.nodes[0].datadir, wallet_file))
96+
self.log.info("- Done")
97+
98+
def get_birthheight(self, wallet_file):
99+
"""Gets birthheight of a wallet on node0"""
100+
with open(os.path.join(self.nodes[0].datadir, wallet_file), 'r', encoding="utf8") as f:
101+
for line in f:
102+
if line.startswith('# * Best block at time of backup'):
103+
wallet_birthheight = int(line.split(' ')[9])
104+
return wallet_birthheight
105+
106+
def has_block(self, block_index):
107+
"""Checks if the pruned node has the specific blk0000*.dat file"""
108+
return os.path.isfile(os.path.join(self.nodes[1].datadir, self.chain, "blocks", f"blk{block_index:05}.dat"))
109+
110+
def create_wallet(self, wallet_name, *, unload=False):
111+
"""Creates and dumps a wallet on the non-pruned node0 to be later import by the pruned node"""
112+
self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=False, load_on_startup=True)
113+
self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, wallet_name + ".dat"))
114+
if (unload):
115+
self.nodes[0].unloadwallet(wallet_name)
116+
117+
def run_test(self):
118+
self.nTime = 0
119+
self.log.info("Warning! This test requires ~1.3GB of disk space")
120+
121+
self.log.info("Generating a long chain of blocks...")
122+
123+
# A blk*.dat file is 128MB
124+
# Generate 250 light blocks
125+
self.generate(self.nodes[0], 250, sync_fun=self.no_op)
126+
# Generate 50MB worth of large blocks in the blk00000.dat file
127+
self.mine_large_blocks(self.nodes[0], 50)
128+
129+
# Create a wallet which birth's block is in the blk00000.dat file
130+
wallet_birthheight_1 = "wallet_birthheight_1"
131+
assert_equal(self.has_block(1), False)
132+
self.create_wallet(wallet_birthheight_1, unload=True)
133+
134+
# Generate enough large blocks to reach pruning disk limit
135+
# Not pruning yet because we are still below PruneAfterHeight
136+
self.mine_large_blocks(self.nodes[0], 600)
137+
self.log.info("- Long chain created")
138+
139+
# Create a wallet with birth height > wallet_birthheight_1
140+
wallet_birthheight_2 = "wallet_birthheight_2"
141+
self.create_wallet(wallet_birthheight_2)
142+
143+
# Fund wallet to later verify that importwallet correctly accounts for balances
144+
self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress(), sync_fun=self.no_op)
145+
146+
# We've reached pruning storage & height limit but
147+
# pruning doesn't run until another chunk (blk*.dat file) is allocated.
148+
# That's why we are generating another 5 large blocks
149+
self.mine_large_blocks(self.nodes[0], 5)
150+
151+
# blk00000.dat file is now pruned from node1
152+
assert_equal(self.has_block(0), False)
153+
154+
self.test_wallet_import_pruned(wallet_birthheight_2)
155+
self.test_wallet_import_pruned_with_missing_blocks(wallet_birthheight_1)
156+
157+
if __name__ == '__main__':
158+
WalletPruningTest().main()

0 commit comments

Comments
 (0)