Skip to content

Commit 143bd10

Browse files
author
MarcoFalke
committed
Merge #19137: wallettool: Add dump and createfromdump commands
23cac24 tests: Test bitcoin-wallet dump and createfromdump (Andrew Chow) a88c320 wallettool: Add createfromdump command (Andrew Chow) e1e7a90 wallettool: Add dump command (Andrew Chow) Pull request description: Adds two commands to the `bitcoin-wallet` tool: `dump` and `createfromdump`. These commands will be useful for a wallet storage migration in the future. It is also generally useful to have a storage agnostic dump like this. These commands are similar to BDB's `db_dump` and `db_load` tools. This can also be useful for manual construction of a wallet file for tests. `dump` outputs every key-value pair from the wallet as comma separated hex. Each key-value pair is on its own line with the key and value in hex separated by a comma. This is output to the file specified by the new `-dumpfile` option. `createfromdump` takes a file produced by `dump` and creates a new wallet file with exactly the records specified in that file. A new option `-dumpfile` is added to the wallet tool. When used with `dump`, the records will be written to the specified file. When used with `createfromdump`, the file is read and the key-value pairs constructed from it. `createfromdump` requires `-dumpfile`. A simple round-trip test is added to the `tool_wallet.py`. This PR is based on #19334, ACKs for top commit: Sjors: re-utACK 23cac24 MarcoFalke: re review ACK 23cac24 only change is rebase and removing useless shared_ptr wrapper 🎼 ryanofsky: Code review ACK 23cac24. Only changes since last review rebase and changing a pointer to a reference Tree-SHA512: 2d63cf62baca3d16495aa698dc02f7d889c81b41015e9c92c23c275bb4a690fc176d351c3fd7f310bd6b17f5a936cc9be694cbecd702af741b96c0f530e72fa2
2 parents af4ce67 + 23cac24 commit 143bd10

File tree

6 files changed

+522
-22
lines changed

6 files changed

+522
-22
lines changed

src/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ BITCOIN_CORE_H = \
249249
wallet/context.h \
250250
wallet/crypter.h \
251251
wallet/db.h \
252+
wallet/dump.h \
252253
wallet/feebumper.h \
253254
wallet/fees.h \
254255
wallet/ismine.h \
@@ -361,6 +362,7 @@ libbitcoin_wallet_a_SOURCES = \
361362
wallet/context.cpp \
362363
wallet/crypter.cpp \
363364
wallet/db.cpp \
365+
wallet/dump.cpp \
364366
wallet/feebumper.cpp \
365367
wallet/fees.cpp \
366368
wallet/interfaces.cpp \

src/bitcoin-wallet.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,17 @@ static void SetupWalletToolArgs(ArgsManager& argsman)
2727
argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
2828
argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
2929
argsman.AddArg("-wallet=<wallet-name>", "Specify wallet name", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::OPTIONS);
30+
argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS);
3031
argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
3132
argsman.AddArg("-descriptors", "Create descriptors wallet. Only for create", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS);
33+
argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
3234
argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
3335

3436
argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
3537
argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
3638
argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
39+
argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
40+
argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
3741
}
3842

3943
static bool WalletAppInit(int argc, char* argv[])

src/wallet/dump.cpp

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
// Copyright (c) 2020 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#include <wallet/dump.h>
6+
7+
#include <util/translation.h>
8+
#include <wallet/wallet.h>
9+
10+
static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP";
11+
uint32_t DUMP_VERSION = 1;
12+
13+
bool DumpWallet(CWallet& wallet, bilingual_str& error)
14+
{
15+
// Get the dumpfile
16+
std::string dump_filename = gArgs.GetArg("-dumpfile", "");
17+
if (dump_filename.empty()) {
18+
error = _("No dump file provided. To use dump, -dumpfile=<filename> must be provided.");
19+
return false;
20+
}
21+
22+
fs::path path = dump_filename;
23+
path = fs::absolute(path);
24+
if (fs::exists(path)) {
25+
error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), path.string());
26+
return false;
27+
}
28+
fsbridge::ofstream dump_file;
29+
dump_file.open(path);
30+
if (dump_file.fail()) {
31+
error = strprintf(_("Unable to open %s for writing"), path.string());
32+
return false;
33+
}
34+
35+
CHashWriter hasher(0, 0);
36+
37+
WalletDatabase& db = wallet.GetDatabase();
38+
std::unique_ptr<DatabaseBatch> batch = db.MakeBatch();
39+
40+
bool ret = true;
41+
if (!batch->StartCursor()) {
42+
error = _("Error: Couldn't create cursor into database");
43+
ret = false;
44+
}
45+
46+
// Write out a magic string with version
47+
std::string line = strprintf("%s,%u\n", DUMP_MAGIC, DUMP_VERSION);
48+
dump_file.write(line.data(), line.size());
49+
hasher.write(line.data(), line.size());
50+
51+
// Write out the file format
52+
line = strprintf("%s,%s\n", "format", db.Format());
53+
dump_file.write(line.data(), line.size());
54+
hasher.write(line.data(), line.size());
55+
56+
if (ret) {
57+
58+
// Read the records
59+
while (true) {
60+
CDataStream ss_key(SER_DISK, CLIENT_VERSION);
61+
CDataStream ss_value(SER_DISK, CLIENT_VERSION);
62+
bool complete;
63+
ret = batch->ReadAtCursor(ss_key, ss_value, complete);
64+
if (complete) {
65+
ret = true;
66+
break;
67+
} else if (!ret) {
68+
error = _("Error reading next record from wallet database");
69+
break;
70+
}
71+
std::string key_str = HexStr(ss_key);
72+
std::string value_str = HexStr(ss_value);
73+
line = strprintf("%s,%s\n", key_str, value_str);
74+
dump_file.write(line.data(), line.size());
75+
hasher.write(line.data(), line.size());
76+
}
77+
}
78+
79+
batch->CloseCursor();
80+
batch.reset();
81+
82+
// Close the wallet after we're done with it. The caller won't be doing this
83+
wallet.Close();
84+
85+
if (ret) {
86+
// Write the hash
87+
tfm::format(dump_file, "checksum,%s\n", HexStr(hasher.GetHash()));
88+
dump_file.close();
89+
} else {
90+
// Remove the dumpfile on failure
91+
dump_file.close();
92+
fs::remove(path);
93+
}
94+
95+
return ret;
96+
}
97+
98+
// The standard wallet deleter function blocks on the validation interface
99+
// queue, which doesn't exist for the bitcoin-wallet. Define our own
100+
// deleter here.
101+
static void WalletToolReleaseWallet(CWallet* wallet)
102+
{
103+
wallet->WalletLogPrintf("Releasing wallet\n");
104+
wallet->Close();
105+
delete wallet;
106+
}
107+
108+
bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings)
109+
{
110+
// Get the dumpfile
111+
std::string dump_filename = gArgs.GetArg("-dumpfile", "");
112+
if (dump_filename.empty()) {
113+
error = _("No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided.");
114+
return false;
115+
}
116+
117+
fs::path dump_path = dump_filename;
118+
dump_path = fs::absolute(dump_path);
119+
if (!fs::exists(dump_path)) {
120+
error = strprintf(_("Dump file %s does not exist."), dump_path.string());
121+
return false;
122+
}
123+
fsbridge::ifstream dump_file(dump_path);
124+
125+
// Compute the checksum
126+
CHashWriter hasher(0, 0);
127+
uint256 checksum;
128+
129+
// Check the magic and version
130+
std::string magic_key;
131+
std::getline(dump_file, magic_key, ',');
132+
std::string version_value;
133+
std::getline(dump_file, version_value, '\n');
134+
if (magic_key != DUMP_MAGIC) {
135+
error = strprintf(_("Error: Dumpfile identifier record is incorrect. Got \"%s\", expected \"%s\"."), magic_key, DUMP_MAGIC);
136+
dump_file.close();
137+
return false;
138+
}
139+
// Check the version number (value of first record)
140+
uint32_t ver;
141+
if (!ParseUInt32(version_value, &ver)) {
142+
error =strprintf(_("Error: Unable to parse version %u as a uint32_t"), version_value);
143+
dump_file.close();
144+
return false;
145+
}
146+
if (ver != DUMP_VERSION) {
147+
error = strprintf(_("Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version %s"), version_value);
148+
dump_file.close();
149+
return false;
150+
}
151+
std::string magic_hasher_line = strprintf("%s,%s\n", magic_key, version_value);
152+
hasher.write(magic_hasher_line.data(), magic_hasher_line.size());
153+
154+
// Get the stored file format
155+
std::string format_key;
156+
std::getline(dump_file, format_key, ',');
157+
std::string format_value;
158+
std::getline(dump_file, format_value, '\n');
159+
if (format_key != "format") {
160+
error = strprintf(_("Error: Dumpfile format record is incorrect. Got \"%s\", expected \"format\"."), format_key);
161+
dump_file.close();
162+
return false;
163+
}
164+
// Get the data file format with format_value as the default
165+
std::string file_format = gArgs.GetArg("-format", format_value);
166+
if (file_format.empty()) {
167+
error = _("No wallet file format provided. To use createfromdump, -format=<format> must be provided.");
168+
return false;
169+
}
170+
DatabaseFormat data_format;
171+
if (file_format == "bdb") {
172+
data_format = DatabaseFormat::BERKELEY;
173+
} else if (file_format == "sqlite") {
174+
data_format = DatabaseFormat::SQLITE;
175+
} else {
176+
error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format);
177+
return false;
178+
}
179+
if (file_format != format_value) {
180+
warnings.push_back(strprintf(_("Warning: Dumpfile wallet format \"%s\" does not match command line specified format \"%s\"."), format_value, file_format));
181+
}
182+
std::string format_hasher_line = strprintf("%s,%s\n", format_key, format_value);
183+
hasher.write(format_hasher_line.data(), format_hasher_line.size());
184+
185+
DatabaseOptions options;
186+
DatabaseStatus status;
187+
options.require_create = true;
188+
options.require_format = data_format;
189+
std::unique_ptr<WalletDatabase> database = MakeDatabase(wallet_path, options, status, error);
190+
if (!database) return false;
191+
192+
// dummy chain interface
193+
bool ret = true;
194+
std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet);
195+
{
196+
LOCK(wallet->cs_wallet);
197+
bool first_run = true;
198+
DBErrors load_wallet_ret = wallet->LoadWallet(first_run);
199+
if (load_wallet_ret != DBErrors::LOAD_OK) {
200+
error = strprintf(_("Error creating %s"), name);
201+
return false;
202+
}
203+
204+
// Get the database handle
205+
WalletDatabase& db = wallet->GetDatabase();
206+
std::unique_ptr<DatabaseBatch> batch = db.MakeBatch();
207+
batch->TxnBegin();
208+
209+
// Read the records from the dump file and write them to the database
210+
while (dump_file.good()) {
211+
std::string key;
212+
std::getline(dump_file, key, ',');
213+
std::string value;
214+
std::getline(dump_file, value, '\n');
215+
216+
if (key == "checksum") {
217+
std::vector<unsigned char> parsed_checksum = ParseHex(value);
218+
std::copy(parsed_checksum.begin(), parsed_checksum.end(), checksum.begin());
219+
break;
220+
}
221+
222+
std::string line = strprintf("%s,%s\n", key, value);
223+
hasher.write(line.data(), line.size());
224+
225+
if (key.empty() || value.empty()) {
226+
continue;
227+
}
228+
229+
if (!IsHex(key)) {
230+
error = strprintf(_("Error: Got key that was not hex: %s"), key);
231+
ret = false;
232+
break;
233+
}
234+
if (!IsHex(value)) {
235+
error = strprintf(_("Error: Got value that was not hex: %s"), value);
236+
ret = false;
237+
break;
238+
}
239+
240+
std::vector<unsigned char> k = ParseHex(key);
241+
std::vector<unsigned char> v = ParseHex(value);
242+
243+
CDataStream ss_key(k, SER_DISK, CLIENT_VERSION);
244+
CDataStream ss_value(v, SER_DISK, CLIENT_VERSION);
245+
246+
if (!batch->Write(ss_key, ss_value)) {
247+
error = strprintf(_("Error: Unable to write record to new wallet"));
248+
ret = false;
249+
break;
250+
}
251+
}
252+
253+
if (ret) {
254+
uint256 comp_checksum = hasher.GetHash();
255+
if (checksum.IsNull()) {
256+
error = _("Error: Missing checksum");
257+
ret = false;
258+
} else if (checksum != comp_checksum) {
259+
error = strprintf(_("Error: Dumpfile checksum does not match. Computed %s, expected %s"), HexStr(comp_checksum), HexStr(checksum));
260+
ret = false;
261+
}
262+
}
263+
264+
if (ret) {
265+
batch->TxnCommit();
266+
} else {
267+
batch->TxnAbort();
268+
}
269+
270+
batch.reset();
271+
272+
dump_file.close();
273+
}
274+
wallet.reset(); // The pointer deleter will close the wallet for us.
275+
276+
// Remove the wallet dir if we have a failure
277+
if (!ret) {
278+
fs::remove_all(wallet_path);
279+
}
280+
281+
return ret;
282+
}

src/wallet/dump.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) 2020 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#ifndef BITCOIN_WALLET_DUMP_H
6+
#define BITCOIN_WALLET_DUMP_H
7+
8+
#include <fs.h>
9+
10+
class CWallet;
11+
12+
struct bilingual_str;
13+
14+
bool DumpWallet(CWallet& wallet, bilingual_str& error);
15+
bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings);
16+
17+
#endif // BITCOIN_WALLET_DUMP_H

0 commit comments

Comments
 (0)