Skip to content

Commit 528f79f

Browse files
achow101achow101
committed
Merge bitcoin/bitcoin#32835: test: fix feature_init.py intermittencies
4207d9b test: feature_init, ensure indexes are synced prior to perturbing files (furszy) abd07cf test: feature_init, only init what's needed per perturbation/deletion round (furszy) Pull request description: Aims to solve #32600. Found it while working on #26966 (this was really annoying there). This ensures the node is index-synced before perturbing files. If the index sync gets interrupted before it starts, the database could be empty, making any following perturbation ineffective (which explains why the node does not abort during startup in the #32600 logs). Also, the first commit avoids initializing components not under test. This reduces log flooding, which helped in understanding the issue. Patch to reproduce the issue on master using `feature_init.py` (this simulates a node shutting down before the index starts syncing): ``` diff --git a/src/index/base.cpp b/src/index/base.cpp --- a/src/index/base.cpp(revision 1e03052c3fefb188f047e72548f2c6b0cc019e50) +++ b/src/index/base.cpp(date 1751293306725) @@ -185,6 +185,7 @@ void BaseIndex::Sync() { const CBlockIndex* pindex = m_best_block_index.load(); + m_interrupt(); if (!m_synced) { std::chrono::steady_clock::time_point last_log_time{0s}; std::chrono::steady_clock::time_point last_locator_write_time{0s}; ``` ACKs for top commit: maflcko: lgtm ACK 4207d9b 🍄 achow101: ACK 4207d9b hodlinator: ACK 4207d9b Tree-SHA512: c8c89c7af9d473a12756b6a59b97f8fb473500181620eb96ecc10da954fe185d13fbb1d00a4ecb181e8daf149ec93cc547e292da0877522a4d23425fa7fd646b
2 parents fc543f9 + 4207d9b commit 528f79f

File tree

1 file changed

+82
-27
lines changed

1 file changed

+82
-27
lines changed

‎test/functional/feature_init.py

Lines changed: 82 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,20 @@ def sigterm_node():
4747
node.process.terminate()
4848
node.process.wait()
4949

50-
def start_expecting_error(err_fragment):
50+
def start_expecting_error(err_fragment, args):
5151
node.assert_start_raises_init_error(
52-
extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1', '-checkblocks=200', '-checklevel=4'],
52+
extra_args=args,
5353
expected_msg=err_fragment,
5454
match=ErrorMatch.PARTIAL_REGEX,
5555
)
5656

57-
def check_clean_start():
57+
def check_clean_start(extra_args):
5858
"""Ensure that node restarts successfully after various interrupts."""
59-
node.start()
59+
node.start(extra_args)
6060
node.wait_for_rpc_connection()
61-
assert_equal(200, node.getblockcount())
61+
height = node.getblockcount()
62+
assert_equal(200, height)
63+
self.wait_until(lambda: all(i["synced"] and i["best_block_height"] == height for i in node.getindexinfo().values()))
6264

6365
lines_to_terminate_after = [
6466
b'Validating signatures for all blocks',
@@ -97,54 +99,107 @@ def check_clean_start():
9799
self.log.debug("Terminating node after terminate line was found")
98100
sigterm_node()
99101

100-
check_clean_start()
102+
# Prior to deleting/perturbing index files, start node with all indexes enabled.
103+
# 'check_clean_start' will ensure indexes are synchronized (i.e., data exists to modify)
104+
check_clean_start(args)
101105
self.stop_node(0)
102106

103107
self.log.info("Test startup errors after removing certain essential files")
104108

105-
files_to_delete = {
106-
'blocks/index/*.ldb': 'Error opening block database.',
107-
'chainstate/*.ldb': 'Error opening coins database.',
108-
'blocks/blk*.dat': 'Error loading block database.',
109-
'indexes/txindex/MANIFEST*': 'LevelDB error: Corruption: CURRENT points to a non-existent file',
109+
deletion_rounds = [
110+
{
111+
'filepath_glob': 'blocks/index/*.ldb',
112+
'error_message': 'Error opening block database.',
113+
'startup_args': [],
114+
},
115+
{
116+
'filepath_glob': 'chainstate/*.ldb',
117+
'error_message': 'Error opening coins database.',
118+
'startup_args': ['-checklevel=4'],
119+
},
120+
{
121+
'filepath_glob': 'blocks/blk*.dat',
122+
'error_message': 'Error loading block database.',
123+
'startup_args': ['-checkblocks=200', '-checklevel=4'],
124+
},
125+
{
126+
'filepath_glob': 'indexes/txindex/MANIFEST*',
127+
'error_message': 'LevelDB error: Corruption: CURRENT points to a non-existent file',
128+
'startup_args': ['-txindex=1'],
129+
},
110130
# Removing these files does not result in a startup error:
111131
# 'indexes/blockfilter/basic/*.dat', 'indexes/blockfilter/basic/db/*.*', 'indexes/coinstats/db/*.*',
112132
# 'indexes/txindex/*.log', 'indexes/txindex/CURRENT', 'indexes/txindex/LOCK'
113-
}
114-
115-
files_to_perturb = {
116-
'blocks/index/*.ldb': 'Error loading block database.',
117-
'chainstate/*.ldb': 'Error opening coins database.',
118-
'blocks/blk*.dat': 'Corrupted block database detected.',
119-
'indexes/blockfilter/basic/db/*.*': 'LevelDB error: Corruption',
120-
'indexes/coinstats/db/*.*': 'LevelDB error: Corruption',
121-
'indexes/txindex/*.log': 'LevelDB error: Corruption',
122-
'indexes/txindex/CURRENT': 'LevelDB error: Corruption',
133+
]
134+
135+
perturbation_rounds = [
136+
{
137+
'filepath_glob': 'blocks/index/*.ldb',
138+
'error_message': 'Error loading block database.',
139+
'startup_args': [],
140+
},
141+
{
142+
'filepath_glob': 'chainstate/*.ldb',
143+
'error_message': 'Error opening coins database.',
144+
'startup_args': [],
145+
},
146+
{
147+
'filepath_glob': 'blocks/blk*.dat',
148+
'error_message': 'Corrupted block database detected.',
149+
'startup_args': ['-checkblocks=200', '-checklevel=4'],
150+
},
151+
{
152+
'filepath_glob': 'indexes/blockfilter/basic/db/*.*',
153+
'error_message': 'LevelDB error: Corruption',
154+
'startup_args': ['-blockfilterindex=1'],
155+
},
156+
{
157+
'filepath_glob': 'indexes/coinstats/db/*.*',
158+
'error_message': 'LevelDB error: Corruption',
159+
'startup_args': ['-coinstatsindex=1'],
160+
},
161+
{
162+
'filepath_glob': 'indexes/txindex/*.log',
163+
'error_message': 'LevelDB error: Corruption',
164+
'startup_args': ['-txindex=1'],
165+
},
166+
{
167+
'filepath_glob': 'indexes/txindex/CURRENT',
168+
'error_message': 'LevelDB error: Corruption',
169+
'startup_args': ['-txindex=1'],
170+
},
123171
# Perturbing these files does not result in a startup error:
124172
# 'indexes/blockfilter/basic/*.dat', 'indexes/txindex/MANIFEST*', 'indexes/txindex/LOCK'
125-
}
173+
]
126174

127-
for file_patt, err_fragment in files_to_delete.items():
175+
for round_info in deletion_rounds:
176+
file_patt = round_info['filepath_glob']
177+
err_fragment = round_info['error_message']
178+
startup_args = round_info['startup_args']
128179
target_files = list(node.chain_path.glob(file_patt))
129180

130181
for target_file in target_files:
131182
self.log.info(f"Deleting file to ensure failure {target_file}")
132183
bak_path = str(target_file) + ".bak"
133184
target_file.rename(bak_path)
134185

135-
start_expecting_error(err_fragment)
186+
start_expecting_error(err_fragment, startup_args)
136187

137188
for target_file in target_files:
138189
bak_path = str(target_file) + ".bak"
139190
self.log.debug(f"Restoring file from {bak_path} and restarting")
140191
Path(bak_path).rename(target_file)
141192

142-
check_clean_start()
193+
check_clean_start(args)
143194
self.stop_node(0)
144195

145196
self.log.info("Test startup errors after perturbing certain essential files")
146197
dirs = ["blocks", "chainstate", "indexes"]
147-
for file_patt, err_fragment in files_to_perturb.items():
198+
for round_info in perturbation_rounds:
199+
file_patt = round_info['filepath_glob']
200+
err_fragment = round_info['error_message']
201+
startup_args = round_info['startup_args']
202+
148203
for dir in dirs:
149204
shutil.copytree(node.chain_path / dir, node.chain_path / f"{dir}_bak")
150205
target_files = list(node.chain_path.glob(file_patt))
@@ -158,7 +213,7 @@ def check_clean_start():
158213
tf.seek(150)
159214
tf.write(b"1" * 200)
160215

161-
start_expecting_error(err_fragment)
216+
start_expecting_error(err_fragment, startup_args)
162217

163218
for dir in dirs:
164219
shutil.rmtree(node.chain_path / dir)

0 commit comments

Comments
 (0)