Skip to content

Commit 239d419

Browse files
laanwjlaanwj
committed
Merge #7917: Optimize reindex
b4d24e1 Report reindexing progress in GUI (Pieter Wuille) d3d7547 Add -reindex-chainstate that does not rebuild block index (Pieter Wuille) fb8fad1 Optimize ActivateBestChain for long chains (Pieter Wuille) 316623f Switch reindexing to AcceptBlock in-loop and ActivateBestChain afterwards (Pieter Wuille) d253ec4 Make ProcessNewBlock dbp const and update comment (Pieter Wuille)
2 parents 8e8bebc + b4d24e1 commit 239d419

File tree

12 files changed

+130
-49
lines changed

12 files changed

+130
-49
lines changed

‎qa/rpc-tests/reindex.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
55

66
#
7-
# Test -reindex with CheckBlockIndex
7+
# Test -reindex and -reindex-chainstate with CheckBlockIndex
88
#
99
from test_framework.test_framework import BitcoinTestFramework
1010
from test_framework.util import *
11+
import time
1112

1213
class ReindexTest(BitcoinTestFramework):
1314

@@ -20,13 +21,22 @@ def setup_network(self):
2021
self.is_network_split = False
2122
self.nodes.append(start_node(0, self.options.tmpdir))
2223

23-
def run_test(self):
24+
def reindex(self, justchainstate=False):
2425
self.nodes[0].generate(3)
26+
blockcount = self.nodes[0].getblockcount()
2527
stop_node(self.nodes[0], 0)
2628
wait_bitcoinds()
27-
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex", "-checkblockindex=1"])
28-
assert_equal(self.nodes[0].getblockcount(), 3)
29+
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"])
30+
while self.nodes[0].getblockcount() < blockcount:
31+
time.sleep(0.1)
32+
assert_equal(self.nodes[0].getblockcount(), blockcount)
2933
print("Success")
3034

35+
def run_test(self):
36+
self.reindex(False)
37+
self.reindex(True)
38+
self.reindex(False)
39+
self.reindex(True)
40+
3141
if __name__ == '__main__':
3242
ReindexTest().main()

‎src/init.cpp

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,8 @@ std::string HelpMessage(HelpMessageMode mode)
327327
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. "
328328
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
329329
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
330-
strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup"));
330+
strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
331+
strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk"));
331332
#ifndef WIN32
332333
strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)"));
333334
#endif
@@ -404,7 +405,7 @@ std::string HelpMessage(HelpMessageMode mode)
404405
strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
405406
strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
406407
}
407-
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http, libevent, tor, zmq"; // Don't translate these and qt below
408+
string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below
408409
if (mode == HMM_BITCOIN_QT)
409410
debugCategories += ", qt";
410411
strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " +
@@ -554,9 +555,10 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
554555
{
555556
const CChainParams& chainparams = Params();
556557
RenameThread("bitcoin-loadblk");
558+
CImportingNow imp;
559+
557560
// -reindex
558561
if (fReindex) {
559-
CImportingNow imp;
560562
int nFile = 0;
561563
while (true) {
562564
CDiskBlockPos pos(nFile, 0);
@@ -581,7 +583,6 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
581583
if (boost::filesystem::exists(pathBootstrap)) {
582584
FILE *file = fopen(pathBootstrap.string().c_str(), "rb");
583585
if (file) {
584-
CImportingNow imp;
585586
boost::filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old";
586587
LogPrintf("Importing bootstrap.dat...\n");
587588
LoadExternalBlockFile(chainparams, file);
@@ -595,14 +596,20 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
595596
BOOST_FOREACH(const boost::filesystem::path& path, vImportFiles) {
596597
FILE *file = fopen(path.string().c_str(), "rb");
597598
if (file) {
598-
CImportingNow imp;
599599
LogPrintf("Importing blocks file %s...\n", path.string());
600600
LoadExternalBlockFile(chainparams, file);
601601
} else {
602602
LogPrintf("Warning: Could not open blocks file %s\n", path.string());
603603
}
604604
}
605605

606+
// scan for better chains in the block chain database, that are not yet connected in the active best chain
607+
CValidationState state;
608+
if (!ActivateBestChain(state, chainparams)) {
609+
LogPrintf("Failed to connect best block");
610+
StartShutdown();
611+
}
612+
606613
if (GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) {
607614
LogPrintf("Stopping after block import\n");
608615
StartShutdown();
@@ -1158,6 +1165,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
11581165
// ********************************************************* Step 7: load block chain
11591166

11601167
fReindex = GetBoolArg("-reindex", false);
1168+
bool fReindexChainState = GetBoolArg("-reindex-chainstate", false);
11611169

11621170
// Upgrading to 0.8; hard-link the old blknnnn.dat files into /blocks/
11631171
boost::filesystem::path blocksDir = GetDataDir() / "blocks";
@@ -1219,7 +1227,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
12191227
delete pblocktree;
12201228

12211229
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
1222-
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
1230+
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex || fReindexChainState);
12231231
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
12241232
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
12251233

@@ -1248,7 +1256,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
12481256

12491257
// Check for changed -txindex state
12501258
if (fTxIndex != GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
1251-
strLoadError = _("You need to rebuild the database using -reindex to change -txindex");
1259+
strLoadError = _("You need to rebuild the database using -reindex-chainstate to change -txindex");
12521260
break;
12531261
}
12541262

@@ -1358,12 +1366,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
13581366
if (mapArgs.count("-blocknotify"))
13591367
uiInterface.NotifyBlockTip.connect(BlockNotifyCallback);
13601368

1361-
uiInterface.InitMessage(_("Activating best chain..."));
1362-
// scan for better chains in the block chain database, that are not yet connected in the active best chain
1363-
CValidationState state;
1364-
if (!ActivateBestChain(state, chainparams))
1365-
strErrors << "Failed to connect best block";
1366-
13671369
std::vector<boost::filesystem::path> vImportFiles;
13681370
if (mapArgs.count("-loadblock"))
13691371
{

‎src/main.cpp

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2811,10 +2811,9 @@ static void PruneBlockIndexCandidates() {
28112811
* Try to make some progress towards making pindexMostWork the active block.
28122812
* pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork.
28132813
*/
2814-
static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock)
2814+
static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound)
28152815
{
28162816
AssertLockHeld(cs_main);
2817-
bool fInvalidFound = false;
28182817
const CBlockIndex *pindexOldTip = chainActive.Tip();
28192818
const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork);
28202819

@@ -2884,6 +2883,28 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c
28842883
return true;
28852884
}
28862885

2886+
static void NotifyHeaderTip() {
2887+
bool fNotify = false;
2888+
bool fInitialBlockDownload = false;
2889+
static CBlockIndex* pindexHeaderOld = NULL;
2890+
CBlockIndex* pindexHeader = NULL;
2891+
{
2892+
LOCK(cs_main);
2893+
if (!setBlockIndexCandidates.empty()) {
2894+
pindexHeader = *setBlockIndexCandidates.rbegin();
2895+
}
2896+
if (pindexHeader != pindexHeaderOld) {
2897+
fNotify = true;
2898+
fInitialBlockDownload = IsInitialBlockDownload();
2899+
pindexHeaderOld = pindexHeader;
2900+
}
2901+
}
2902+
// Send block tip changed notifications without cs_main
2903+
if (fNotify) {
2904+
uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader);
2905+
}
2906+
}
2907+
28872908
/**
28882909
* Make the best chain active, in multiple steps. The result is either failure
28892910
* or an activated best chain. pblock is either NULL or a pointer to a block
@@ -2902,15 +2923,22 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams,
29022923
{
29032924
LOCK(cs_main);
29042925
CBlockIndex *pindexOldTip = chainActive.Tip();
2905-
pindexMostWork = FindMostWorkChain();
2926+
if (pindexMostWork == NULL) {
2927+
pindexMostWork = FindMostWorkChain();
2928+
}
29062929

29072930
// Whether we have anything to do at all.
29082931
if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip())
29092932
return true;
29102933

2911-
if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL))
2934+
bool fInvalidFound = false;
2935+
if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound))
29122936
return false;
29132937

2938+
if (fInvalidFound) {
2939+
// Wipe cache, we may need another branch now.
2940+
pindexMostWork = NULL;
2941+
}
29142942
pindexNewTip = chainActive.Tip();
29152943
pindexFork = chainActive.FindFork(pindexOldTip);
29162944
fInitialDownload = IsInitialBlockDownload();
@@ -3398,11 +3426,12 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state
33983426
}
33993427

34003428
/** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */
3401-
static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, CDiskBlockPos* dbp)
3429+
static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp)
34023430
{
34033431
AssertLockHeld(cs_main);
34043432

3405-
CBlockIndex *&pindex = *ppindex;
3433+
CBlockIndex *pindexDummy = NULL;
3434+
CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy;
34063435

34073436
if (!AcceptBlockHeader(block, state, chainparams, &pindex))
34083437
return false;
@@ -3474,7 +3503,7 @@ static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned
34743503
}
34753504

34763505

3477-
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp)
3506+
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp)
34783507
{
34793508
{
34803509
LOCK(cs_main);
@@ -3492,6 +3521,8 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c
34923521
return error("%s: AcceptBlock FAILED", __func__);
34933522
}
34943523

3524+
NotifyHeaderTip();
3525+
34953526
if (!ActivateBestChain(state, chainparams, pblock))
34963527
return error("%s: ActivateBestChain failed", __func__);
34973528

@@ -4037,15 +4068,26 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
40374068

40384069
// process in case the block isn't known yet
40394070
if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) {
4071+
LOCK(cs_main);
40404072
CValidationState state;
4041-
if (ProcessNewBlock(state, chainparams, NULL, &block, true, dbp))
4073+
if (AcceptBlock(block, state, chainparams, NULL, true, dbp))
40424074
nLoaded++;
40434075
if (state.IsError())
40444076
break;
40454077
} else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) {
4046-
LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight);
4078+
LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight);
4079+
}
4080+
4081+
// Activate the genesis block so normal node progress can continue
4082+
if (hash == chainparams.GetConsensus().hashGenesisBlock) {
4083+
CValidationState state;
4084+
if (!ActivateBestChain(state, chainparams)) {
4085+
break;
4086+
}
40474087
}
40484088

4089+
NotifyHeaderTip();
4090+
40494091
// Recursively process earlier encountered successors of this block
40504092
deque<uint256> queue;
40514093
queue.push_back(hash);
@@ -4057,17 +4099,19 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
40574099
std::multimap<uint256, CDiskBlockPos>::iterator it = range.first;
40584100
if (ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()))
40594101
{
4060-
LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(),
4102+
LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(),
40614103
head.ToString());
4104+
LOCK(cs_main);
40624105
CValidationState dummy;
4063-
if (ProcessNewBlock(dummy, chainparams, NULL, &block, true, &it->second))
4106+
if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second))
40644107
{
40654108
nLoaded++;
40664109
queue.push_back(block.GetHash());
40674110
}
40684111
}
40694112
range.first++;
40704113
mapBlocksUnknownParent.erase(it);
4114+
NotifyHeaderTip();
40714115
}
40724116
}
40734117
} catch (const std::exception& e) {
@@ -5077,6 +5121,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
50775121
ReadCompactSize(vRecv); // ignore tx count; assume it is 0.
50785122
}
50795123

5124+
{
50805125
LOCK(cs_main);
50815126

50825127
if (nCount == 0) {
@@ -5167,6 +5212,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
51675212
}
51685213

51695214
CheckBlockIndex(chainparams.GetConsensus());
5215+
}
5216+
5217+
NotifyHeaderTip();
51705218
}
51715219

51725220
else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing

‎src/main.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,10 +212,10 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals);
212212
* @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be penalised if the block is invalid.
213213
* @param[in] pblock The block we want to process.
214214
* @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers.
215-
* @param[out] dbp If pblock is stored to disk (or already there), this will be set to its location.
215+
* @param[out] dbp The already known disk position of pblock, or NULL if not yet stored.
216216
* @return True if state.IsValid()
217217
*/
218-
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp);
218+
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp);
219219
/** Check whether enough disk space is available for an incoming block */
220220
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0);
221221
/** Open a block file (blk?????.dat) */

‎src/qt/bitcoingui.cpp

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -457,8 +457,8 @@ void BitcoinGUI::setClientModel(ClientModel *clientModel)
457457
setNumConnections(clientModel->getNumConnections());
458458
connect(clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int)));
459459

460-
setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL));
461-
connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double)), this, SLOT(setNumBlocks(int,QDateTime,double)));
460+
setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL), false);
461+
connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool)));
462462

463463
// Receive and report messages from client model
464464
connect(clientModel, SIGNAL(message(QString,QString,unsigned int)), this, SLOT(message(QString,QString,unsigned int)));
@@ -696,7 +696,7 @@ void BitcoinGUI::setNumConnections(int count)
696696
labelConnectionsIcon->setToolTip(tr("%n active connection(s) to Bitcoin network", "", count));
697697
}
698698

699-
void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress)
699+
void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header)
700700
{
701701
if(!clientModel)
702702
return;
@@ -708,15 +708,25 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer
708708
enum BlockSource blockSource = clientModel->getBlockSource();
709709
switch (blockSource) {
710710
case BLOCK_SOURCE_NETWORK:
711+
if (header) {
712+
return;
713+
}
711714
progressBarLabel->setText(tr("Synchronizing with network..."));
712715
break;
713716
case BLOCK_SOURCE_DISK:
714-
progressBarLabel->setText(tr("Importing blocks from disk..."));
717+
if (header) {
718+
progressBarLabel->setText(tr("Indexing blocks on disk..."));
719+
} else {
720+
progressBarLabel->setText(tr("Processing blocks on disk..."));
721+
}
715722
break;
716723
case BLOCK_SOURCE_REINDEX:
717724
progressBarLabel->setText(tr("Reindexing blocks on disk..."));
718725
break;
719726
case BLOCK_SOURCE_NONE:
727+
if (header) {
728+
return;
729+
}
720730
// Case: not Importing, not Reindexing and no network connection
721731
progressBarLabel->setText(tr("No block source available..."));
722732
break;

‎src/qt/bitcoingui.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public Q_SLOTS:
150150
/** Set number of connections shown in the UI */
151151
void setNumConnections(int count);
152152
/** Set number of blocks and last block date shown in the UI */
153-
void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress);
153+
void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers);
154154

155155
/** Notify the user of an event from the core network or transaction handling code.
156156
@param[in] title the message box / notification title

0 commit comments

Comments
 (0)