Skip to content

Commit ec3073a

Browse files
jimpojimpo
committed
index: Move index DBs into index/ directory.
1 parent 89eddcd commit ec3073a

File tree

6 files changed

+264
-245
lines changed

6 files changed

+264
-245
lines changed

‎src/index/base.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include <validation.h>
1212
#include <warnings.h>
1313

14+
constexpr char DB_BEST_BLOCK = 'B';
15+
1416
constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
1517
constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
1618

@@ -26,6 +28,24 @@ static void FatalError(const char* fmt, const Args&... args)
2628
StartShutdown();
2729
}
2830

31+
BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
32+
CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
33+
{}
34+
35+
bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
36+
{
37+
bool success = Read(DB_BEST_BLOCK, locator);
38+
if (!success) {
39+
locator.SetNull();
40+
}
41+
return success;
42+
}
43+
44+
bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
45+
{
46+
return Write(DB_BEST_BLOCK, locator);
47+
}
48+
2949
BaseIndex::~BaseIndex()
3050
{
3151
Interrupt();

‎src/index/base.h

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
#ifndef BITCOIN_INDEX_BASE_H
66
#define BITCOIN_INDEX_BASE_H
77

8+
#include <dbwrapper.h>
89
#include <primitives/block.h>
910
#include <primitives/transaction.h>
1011
#include <threadinterrupt.h>
11-
#include <txdb.h>
1212
#include <uint256.h>
1313
#include <validationinterface.h>
1414

@@ -21,6 +21,20 @@ class CBlockIndex;
2121
*/
2222
class BaseIndex : public CValidationInterface
2323
{
24+
protected:
25+
class DB : public CDBWrapper
26+
{
27+
public:
28+
DB(const fs::path& path, size_t n_cache_size,
29+
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
30+
31+
/// Read block locator of the chain that the txindex is in sync with.
32+
bool ReadBestBlock(CBlockLocator& locator) const;
33+
34+
/// Write block locator of the chain that the txindex is in sync with.
35+
bool WriteBestBlock(const CBlockLocator& locator);
36+
};
37+
2438
private:
2539
/// Whether the index is in sync with the main chain. The flag is flipped
2640
/// from false to true once, after which point this starts processing
@@ -55,7 +69,7 @@ class BaseIndex : public CValidationInterface
5569
/// Write update index entries for a newly connected block.
5670
virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
5771

58-
virtual BaseIndexDB& GetDB() const = 0;
72+
virtual DB& GetDB() const = 0;
5973

6074
/// Get the name of the index for display in logs.
6175
virtual const char* GetName() const = 0;

‎src/index/txindex.cpp

Lines changed: 218 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,232 @@
33
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
44

55
#include <index/txindex.h>
6+
#include <init.h>
7+
#include <ui_interface.h>
68
#include <util.h>
79
#include <validation.h>
810

11+
#include <boost/thread.hpp>
12+
13+
constexpr char DB_BEST_BLOCK = 'B';
14+
constexpr char DB_TXINDEX = 't';
15+
constexpr char DB_TXINDEX_BLOCK = 'T';
16+
917
std::unique_ptr<TxIndex> g_txindex;
1018

19+
struct CDiskTxPos : public CDiskBlockPos
20+
{
21+
unsigned int nTxOffset; // after header
22+
23+
ADD_SERIALIZE_METHODS;
24+
25+
template <typename Stream, typename Operation>
26+
inline void SerializationOp(Stream& s, Operation ser_action) {
27+
READWRITEAS(CDiskBlockPos, *this);
28+
READWRITE(VARINT(nTxOffset));
29+
}
30+
31+
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
32+
}
33+
34+
CDiskTxPos() {
35+
SetNull();
36+
}
37+
38+
void SetNull() {
39+
CDiskBlockPos::SetNull();
40+
nTxOffset = 0;
41+
}
42+
};
43+
44+
/**
45+
* Access to the txindex database (indexes/txindex/)
46+
*
47+
* The database stores a block locator of the chain the database is synced to
48+
* so that the TxIndex can efficiently determine the point it last stopped at.
49+
* A locator is used instead of a simple hash of the chain tip because blocks
50+
* and block index entries may not be flushed to disk until after this database
51+
* is updated.
52+
*/
53+
class TxIndex::DB : public BaseIndex::DB
54+
{
55+
public:
56+
explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
57+
58+
/// Read the disk location of the transaction data with the given hash. Returns false if the
59+
/// transaction hash is not indexed.
60+
bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
61+
62+
/// Write a batch of transaction positions to the DB.
63+
bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
64+
65+
/// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
66+
/// been upgraded yet to the new database.
67+
bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
68+
};
69+
70+
TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
71+
BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
72+
{}
73+
74+
bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
75+
{
76+
return Read(std::make_pair(DB_TXINDEX, txid), pos);
77+
}
78+
79+
bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
80+
{
81+
CDBBatch batch(*this);
82+
for (const auto& tuple : v_pos) {
83+
batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
84+
}
85+
return WriteBatch(batch);
86+
}
87+
88+
/*
89+
* Safely persist a transfer of data from the old txindex database to the new one, and compact the
90+
* range of keys updated. This is used internally by MigrateData.
91+
*/
92+
static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
93+
CDBBatch& batch_newdb, CDBBatch& batch_olddb,
94+
const std::pair<unsigned char, uint256>& begin_key,
95+
const std::pair<unsigned char, uint256>& end_key)
96+
{
97+
// Sync new DB changes to disk before deleting from old DB.
98+
newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
99+
olddb.WriteBatch(batch_olddb);
100+
olddb.CompactRange(begin_key, end_key);
101+
102+
batch_newdb.Clear();
103+
batch_olddb.Clear();
104+
}
105+
106+
bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
107+
{
108+
// The prior implementation of txindex was always in sync with block index
109+
// and presence was indicated with a boolean DB flag. If the flag is set,
110+
// this means the txindex from a previous version is valid and in sync with
111+
// the chain tip. The first step of the migration is to unset the flag and
112+
// write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
113+
// index entries are copied over in batches to the new database. Finally,
114+
// DB_TXINDEX_BLOCK is erased from the old database and the block hash is
115+
// written to the new database.
116+
//
117+
// Unsetting the boolean flag ensures that if the node is downgraded to a
118+
// previous version, it will not see a corrupted, partially migrated index
119+
// -- it will see that the txindex is disabled. When the node is upgraded
120+
// again, the migration will pick up where it left off and sync to the block
121+
// with hash DB_TXINDEX_BLOCK.
122+
bool f_legacy_flag = false;
123+
block_tree_db.ReadFlag("txindex", f_legacy_flag);
124+
if (f_legacy_flag) {
125+
if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
126+
return error("%s: cannot write block indicator", __func__);
127+
}
128+
if (!block_tree_db.WriteFlag("txindex", false)) {
129+
return error("%s: cannot write block index db flag", __func__);
130+
}
131+
}
132+
133+
CBlockLocator locator;
134+
if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
135+
return true;
136+
}
137+
138+
int64_t count = 0;
139+
LogPrintf("Upgrading txindex database... [0%%]\n");
140+
uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
141+
int report_done = 0;
142+
const size_t batch_size = 1 << 24; // 16 MiB
143+
144+
CDBBatch batch_newdb(*this);
145+
CDBBatch batch_olddb(block_tree_db);
146+
147+
std::pair<unsigned char, uint256> key;
148+
std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
149+
std::pair<unsigned char, uint256> prev_key = begin_key;
150+
151+
bool interrupted = false;
152+
std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
153+
for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
154+
boost::this_thread::interruption_point();
155+
if (ShutdownRequested()) {
156+
interrupted = true;
157+
break;
158+
}
159+
160+
if (!cursor->GetKey(key)) {
161+
return error("%s: cannot get key from valid cursor", __func__);
162+
}
163+
if (key.first != DB_TXINDEX) {
164+
break;
165+
}
166+
167+
// Log progress every 10%.
168+
if (++count % 256 == 0) {
169+
// Since txids are uniformly random and traversed in increasing order, the high 16 bits
170+
// of the hash can be used to estimate the current progress.
171+
const uint256& txid = key.second;
172+
uint32_t high_nibble =
173+
(static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
174+
(static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
175+
int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
176+
177+
uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
178+
if (report_done < percentage_done/10) {
179+
LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
180+
report_done = percentage_done/10;
181+
}
182+
}
183+
184+
CDiskTxPos value;
185+
if (!cursor->GetValue(value)) {
186+
return error("%s: cannot parse txindex record", __func__);
187+
}
188+
batch_newdb.Write(key, value);
189+
batch_olddb.Erase(key);
190+
191+
if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
192+
// NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
193+
// because LevelDB iterators are guaranteed to provide a consistent view of the
194+
// underlying data, like a lightweight snapshot.
195+
WriteTxIndexMigrationBatches(*this, block_tree_db,
196+
batch_newdb, batch_olddb,
197+
prev_key, key);
198+
prev_key = key;
199+
}
200+
}
201+
202+
// If these final DB batches complete the migration, write the best block
203+
// hash marker to the new database and delete from the old one. This signals
204+
// that the former is fully caught up to that point in the blockchain and
205+
// that all txindex entries have been removed from the latter.
206+
if (!interrupted) {
207+
batch_olddb.Erase(DB_TXINDEX_BLOCK);
208+
batch_newdb.Write(DB_BEST_BLOCK, locator);
209+
}
210+
211+
WriteTxIndexMigrationBatches(*this, block_tree_db,
212+
batch_newdb, batch_olddb,
213+
begin_key, key);
214+
215+
if (interrupted) {
216+
LogPrintf("[CANCELLED].\n");
217+
return false;
218+
}
219+
220+
uiInterface.ShowProgress("", 100, false);
221+
222+
LogPrintf("[DONE].\n");
223+
return true;
224+
}
225+
11226
TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
12227
: m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
13228
{}
14229

230+
TxIndex::~TxIndex() {}
231+
15232
bool TxIndex::Init()
16233
{
17234
LOCK(cs_main);
@@ -38,7 +255,7 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
38255
return m_db->WriteTxs(vPos);
39256
}
40257

41-
BaseIndexDB& TxIndex::GetDB() const { return *m_db; }
258+
BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
42259

43260
bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
44261
{

‎src/index/txindex.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
#ifndef BITCOIN_INDEX_TXINDEX_H
66
#define BITCOIN_INDEX_TXINDEX_H
77

8+
#include <chain.h>
89
#include <index/base.h>
10+
#include <txdb.h>
911

1012
/**
1113
* TxIndex is used to look up transactions included in the blockchain by hash.
@@ -14,23 +16,29 @@
1416
*/
1517
class TxIndex final : public BaseIndex
1618
{
19+
protected:
20+
class DB;
21+
1722
private:
18-
const std::unique_ptr<TxIndexDB> m_db;
23+
const std::unique_ptr<DB> m_db;
1924

2025
protected:
2126
/// Override base class init to migrate from old database.
2227
bool Init() override;
2328

2429
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
2530

26-
BaseIndexDB& GetDB() const override;
31+
BaseIndex::DB& GetDB() const override;
2732

2833
const char* GetName() const override { return "txindex"; }
2934

3035
public:
3136
/// Constructs the index, which becomes available to be queried.
3237
explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
3338

39+
// Destructor is declared because this class contains a unique_ptr to an incomplete type.
40+
virtual ~TxIndex() override;
41+
3442
/// Look up a transaction by hash.
3543
///
3644
/// @param[in] tx_hash The hash of the transaction to be returned.

0 commit comments

Comments
 (0)