Skip to content

Commit ea2718a

Browse files
EmelyanenkoKEmelyanenkoK
authored
Merge pull request #1863 from ton-blockchain/apply-block-patch
Debug logs for applying blocks, optimize BlockArchiver
2 parents 85d8b01 + a1ddbcc commit ea2718a

17 files changed

+183
-91
lines changed

‎adnl/adnl-packet.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ td::Result<AdnlPacket> AdnlPacket::create(tl_object_ptr<ton_api::adnl_packetCont
5656
R.addr_ = std::move(addr_list);
5757
}
5858
if (R.flags_ & Flags::f_priority_address) {
59-
TRY_RESULT(addr_list, AdnlAddressList::create(std::move(packet->address_)));
59+
TRY_RESULT(addr_list, AdnlAddressList::create(std::move(packet->priority_address_)));
6060
R.priority_addr_ = std::move(addr_list);
6161
}
6262
if (R.flags_ & Flags::f_seqno) {

‎validator/apply-block.cpp‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ namespace validator {
2929

3030
void ApplyBlock::abort_query(td::Status reason) {
3131
if (promise_) {
32-
VLOG(VALIDATOR_WARNING) << "aborting apply block query for " << id_ << ": " << reason;
32+
VLOG(VALIDATOR_WARNING) << "aborting apply block query for " << id_.to_str() << ": " << reason;
3333
promise_.set_error(std::move(reason));
3434
}
3535
stop();
3636
}
3737

3838
void ApplyBlock::finish_query() {
39-
VLOG(VALIDATOR_DEBUG) << "successfully finishing apply block query";
39+
VLOG(VALIDATOR_DEBUG) << "successfully finishing apply block query in " << perf_timer_.elapsed() << " s";
4040
handle_->set_processed();
4141
ValidatorInvariants::check_post_apply(handle_);
4242

@@ -51,7 +51,7 @@ void ApplyBlock::alarm() {
5151
}
5252

5353
void ApplyBlock::start_up() {
54-
VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_;
54+
VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_.to_str() << ", mc_seqno=" << masterchain_block_id_.seqno();
5555

5656
if (id_.is_masterchain()) {
5757
masterchain_block_id_ = id_;
@@ -71,6 +71,7 @@ void ApplyBlock::start_up() {
7171
}
7272

7373
void ApplyBlock::got_block_handle(BlockHandle handle) {
74+
VLOG(VALIDATOR_DEBUG) << "got_block_handle";
7475
handle_ = std::move(handle);
7576

7677
if (handle_->is_applied() && (!handle_->id().is_masterchain() || handle_->processed())) {
@@ -79,6 +80,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
7980
}
8081

8182
if (handle_->is_applied()) {
83+
VLOG(VALIDATOR_DEBUG) << "already applied";
8284
auto P =
8385
td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = handle_->id().id.seqno](td::Result<BlockIdExt> R) {
8486
R.ensure();
@@ -94,16 +96,19 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
9496
}
9597

9698
if (handle_->id().id.seqno == 0) {
99+
VLOG(VALIDATOR_DEBUG) << "seqno == 0";
97100
written_block_data();
98101
return;
99102
}
100103

101104
if (handle_->is_archived()) {
105+
VLOG(VALIDATOR_DEBUG) << "already archived";
102106
finish_query();
103107
return;
104108
}
105109

106110
if (handle_->received()) {
111+
VLOG(VALIDATOR_DEBUG) << "already received";
107112
written_block_data();
108113
return;
109114
}
@@ -117,6 +122,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
117122
}
118123
});
119124

125+
VLOG(VALIDATOR_DEBUG) << "storing block data";
120126
td::actor::send_closure(manager_, &ValidatorManager::set_block_data, handle_, block_, std::move(P));
121127
} else {
122128
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::Ref<BlockData>> R) {
@@ -128,13 +134,14 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
128134
}
129135
});
130136

137+
VLOG(VALIDATOR_DEBUG) << "wait for block data";
131138
td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, apply_block_priority(), timeout_,
132139
std::move(P));
133140
}
134141
}
135142

136143
void ApplyBlock::written_block_data() {
137-
VLOG(VALIDATOR_DEBUG) << "apply block: written block data for " << id_;
144+
VLOG(VALIDATOR_DEBUG) << "written_block_data";
138145
if (!handle_->id().seqno()) {
139146
CHECK(handle_->inited_split_after());
140147
CHECK(handle_->inited_state_root_hash());
@@ -165,24 +172,26 @@ void ApplyBlock::written_block_data() {
165172
}
166173
});
167174

175+
VLOG(VALIDATOR_DEBUG) << "wait_block_state";
168176
td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_,
169177
true, std::move(P));
170178
}
171179
}
172180

173181
void ApplyBlock::got_cur_state(td::Ref<ShardState> state) {
174-
VLOG(VALIDATOR_DEBUG) << "apply block: received state for " << id_;
182+
VLOG(VALIDATOR_DEBUG) << "got_cur_state";
175183
state_ = std::move(state);
176184
CHECK(handle_->received_state());
177185
written_state();
178186
}
179187

180188
void ApplyBlock::written_state() {
189+
VLOG(VALIDATOR_DEBUG) << "written_state";
181190
if (handle_->is_applied() && handle_->processed()) {
182191
finish_query();
183192
return;
184193
}
185-
VLOG(VALIDATOR_DEBUG) << "apply block: setting next for parents of " << id_;
194+
VLOG(VALIDATOR_DEBUG) << "setting next for parents";
186195

187196
if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
188197
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -208,12 +217,13 @@ void ApplyBlock::written_state() {
208217
}
209218

210219
void ApplyBlock::written_next() {
220+
VLOG(VALIDATOR_DEBUG) << "written_next";
211221
if (handle_->is_applied() && handle_->processed()) {
212222
finish_query();
213223
return;
214224
}
215225

216-
VLOG(VALIDATOR_DEBUG) << "apply block: applying parents of " << id_;
226+
VLOG(VALIDATOR_DEBUG) << "applying parents";
217227

218228
if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
219229
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -241,7 +251,8 @@ void ApplyBlock::written_next() {
241251
}
242252

243253
void ApplyBlock::applied_prev() {
244-
VLOG(VALIDATOR_DEBUG) << "apply block: waiting manager's confirm for " << id_;
254+
VLOG(VALIDATOR_DEBUG) << "applying parents";
255+
VLOG(VALIDATOR_DEBUG) << "applied_prev, waiting manager's confirm";
245256
if (!id_.is_masterchain()) {
246257
handle_->set_masterchain_ref_block(masterchain_block_id_.seqno());
247258
}
@@ -253,10 +264,11 @@ void ApplyBlock::applied_prev() {
253264
}
254265
});
255266
td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, state_, std::move(P));
267+
256268
}
257269

258270
void ApplyBlock::applied_set() {
259-
VLOG(VALIDATOR_DEBUG) << "apply block: setting apply bit for " << id_;
271+
VLOG(VALIDATOR_DEBUG) << "applied_set";
260272
handle_->set_applied();
261273
if (handle_->id().seqno() > 0) {
262274
CHECK(handle_->handle_moved_to_archive());
@@ -270,6 +282,7 @@ void ApplyBlock::applied_set() {
270282
td::actor::send_closure(SelfId, &ApplyBlock::finish_query);
271283
}
272284
});
285+
VLOG(VALIDATOR_DEBUG) << "flush handle";
273286
handle_->flush(manager_, handle_, std::move(P));
274287
} else {
275288
finish_query();

‎validator/db/archive-manager.cpp‎

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,9 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promis
7070
handle->unix_time(), handle->logical_time(),
7171
handle->inited_is_key_block() && handle->is_key_block())
7272
: get_package_id(handle->masterchain_ref_block());
73-
auto f = get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(),
74-
handle->logical_time(), true);
73+
TRY_RESULT_PROMISE(promise, f,
74+
get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(),
75+
handle->logical_time(), true));
7576
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_handle, std::move(handle), std::move(promise));
7677
}
7778

@@ -83,16 +84,17 @@ void ArchiveManager::update_handle(BlockHandle handle, td::Promise<td::Unit> pro
8384
promise.set_value(td::Unit());
8485
return;
8586
}
86-
f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(),
87-
handle->unix_time(), handle->logical_time(), true);
88-
if (!f) {
87+
auto F = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()),
88+
handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
89+
if (F.is_error()) {
8990
handle->flushed_upto(handle->version());
9091
promise.set_value(td::Unit());
9192
return;
9293
}
94+
f = F.move_as_ok();
9395
} else {
94-
f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
95-
CHECK(f);
96+
TRY_RESULT_PROMISE_ASSIGN(promise, f,
97+
get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true));
9698
}
9799
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise));
98100
}
@@ -110,13 +112,17 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff
110112
auto ig = mp.init_guard();
111113
ig.add_promise(std::move(promise));
112114
auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
113-
td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), data.clone(),
114-
ig.get_promise());
115+
if (f1.is_ok()) {
116+
td::actor::send_closure(f1.ok()->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id),
117+
data.clone(), ig.get_promise());
118+
}
115119
if (copy_to_key) {
116120
auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
117121
handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
118-
td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, nullptr, ref_id, std::move(data),
119-
ig.get_promise());
122+
if (f2.is_ok()) {
123+
td::actor::send_closure(f2.ok()->file_actor_id(), &ArchiveSlice::add_file, nullptr, ref_id, std::move(data),
124+
ig.get_promise());
125+
}
120126
}
121127
return;
122128
}
@@ -128,24 +134,30 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff
128134
ig.add_promise(std::move(promise));
129135
auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()),
130136
handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
131-
td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, data.clone(), ig.get_promise());
137+
if (f1.is_ok()) {
138+
td::actor::send_closure(f1.ok()->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, data.clone(),
139+
ig.get_promise());
140+
}
132141
if (copy_to_key) {
133142
auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
134143
handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
135-
td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, std::move(data),
136-
ig.get_promise());
144+
if (f2.is_ok()) {
145+
td::actor::send_closure(f2.ok()->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, std::move(data),
146+
ig.get_promise());
147+
}
137148
}
138149
}
139150

140151
void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id,
141152
td::BufferSlice data, td::Promise<td::Unit> promise) {
142-
auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true);
153+
TRY_RESULT_PROMISE(promise, f,
154+
get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true));
143155
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data),
144156
std::move(promise));
145157
}
146158

147159
void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
148-
auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true);
160+
TRY_RESULT_PROMISE(promise, f, get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true));
149161
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data),
150162
std::move(promise));
151163
}
@@ -262,21 +274,23 @@ void ArchiveManager::get_key_block_proof(FileReference ref_id, td::Promise<td::B
262274
}
263275

264276
void ArchiveManager::get_temp_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
265-
get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
277+
get_temp_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
266278
}
267279

268-
void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise) {
280+
void ArchiveManager::get_temp_file_short_cont(FileReference ref_id, PackageId idx,
281+
td::Promise<td::BufferSlice> promise) {
269282
auto f = get_temp_file_desc_by_idx(idx);
270283
if (!f) {
271-
promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db"));
284+
promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "file not in db: " << ref_id.filename()));
272285
return;
273286
}
274287
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_prev_temp_file_desc_idx(idx),
275288
promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
276289
if (R.is_ok()) {
277290
promise.set_value(R.move_as_ok());
278291
} else {
279-
td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, std::move(ref_id), idx, std::move(promise));
292+
td::actor::send_closure(SelfId, &ArchiveManager::get_temp_file_short_cont, std::move(ref_id), idx,
293+
std::move(promise));
280294
}
281295
});
282296
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, nullptr, std::move(ref_id), std::move(P));
@@ -285,29 +299,27 @@ void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td
285299
void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td::Promise<td::BufferSlice> promise) {
286300
if (handle->moved_to_archive()) {
287301
auto f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), 0, 0, 0, false);
288-
if (f) {
289-
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id),
302+
if (f.is_ok()) {
303+
td::actor::send_closure(f.ok()->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id),
290304
std::move(promise));
291305
return;
292306
}
293307
}
294308
if (handle->handle_moved_to_archive()) {
295309
auto f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), 0, 0, 0, false);
296-
if (f) {
297-
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_max_temp_file_desc_idx(),
298-
promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
310+
if (f.ok()) {
311+
promise = [=, promise = std::move(promise),
312+
file_actor = f.ok()->file_actor_id()](td::Result<td::BufferSlice> R) mutable {
299313
if (R.is_ok()) {
300314
promise.set_value(R.move_as_ok());
301-
} else {
302-
td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, ref_id, idx, std::move(promise));
315+
return;
303316
}
304-
});
305-
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id),
306-
std::move(P));
307-
return;
317+
td::actor::send_closure(file_actor, &ArchiveSlice::get_file, std::move(handle), std::move(ref_id),
318+
std::move(promise));
319+
};
308320
}
309321
}
310-
get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
322+
get_temp_file_short(std::move(ref_id), std::move(promise));
311323
}
312324

313325
void ArchiveManager::register_perm_state(FileReferenceShort id) {
@@ -656,21 +668,22 @@ void ArchiveManager::load_package(PackageId id) {
656668
update_permanent_slices();
657669
}
658670

659-
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
660-
UnixTime ts, LogicalTime lt, bool force) {
671+
td::Result<const ArchiveManager::FileDescription *> ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id,
672+
BlockSeqno seqno, UnixTime ts,
673+
LogicalTime lt, bool force) {
661674
auto &f = get_file_map(id);
662675
auto it = f.find(id);
663676
if (it != f.end()) {
664677
if (it->second.deleted) {
665-
return nullptr;
678+
return td::Status::Error("file is deleted");
666679
}
667680
if (force && !id.temp) {
668681
update_desc(f, it->second, shard, seqno, ts, lt);
669682
}
670683
return &it->second;
671684
}
672685
if (!force) {
673-
return nullptr;
686+
return td::Status::Error("file not found");
674687
}
675688

676689
return add_file_desc(shard, id, seqno, ts, lt);
@@ -1186,13 +1199,9 @@ void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull sh
11861199
void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
11871200
td::Promise<td::BufferSlice> promise) {
11881201
auto arch = static_cast<BlockSeqno>(archive_id);
1189-
auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{arch, false, false}, 0, 0, 0, false);
1190-
if (!F) {
1191-
promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
1192-
return;
1193-
}
1194-
1195-
td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, archive_id, offset, limit, std::move(promise));
1202+
TRY_RESULT_PROMISE(promise, f,
1203+
get_file_desc(ShardIdFull{masterchainId}, PackageId{arch, false, false}, 0, 0, 0, false));
1204+
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_slice, archive_id, offset, limit, std::move(promise));
11961205
}
11971206

11981207
void ArchiveManager::commit_transaction() {

‎validator/db/archive-manager.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,10 +204,10 @@ class ArchiveManager : public td::actor::Actor {
204204
void deleted_package(PackageId seqno, td::Promise<td::Unit> promise);
205205
void get_handle_cont(BlockIdExt block_id, PackageId id, td::Promise<BlockHandle> promise);
206206
void get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise);
207-
void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise);
207+
void get_temp_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise);
208208

209-
const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt,
210-
bool force);
209+
td::Result<const FileDescription *> get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts,
210+
LogicalTime lt, bool force);
211211
const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
212212
void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
213213
LogicalTime lt);

‎validator/db/archive-slice.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P
385385
auto R = kv_->get(ref_id.hash().to_hex(), value);
386386
R.ensure();
387387
if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
388-
promise.set_error(td::Status::Error(ErrorCode::notready, "file not in archive slice"));
388+
promise.set_error(td::Status::Error(
389+
ErrorCode::notready, PSTRING() << "file " << ref_id.filename() << " not in archive slice " << get_name()));
389390
return;
390391
}
391392
auto offset = td::to_integer<td::uint64>(value);

0 commit comments

Comments
 (0)