Skip to content

Commit a7c0010

Browse files
SpyCheeseSpyCheese
authored
Ban collator node when it returns invalid block (#1797)
1 parent 3c10903 commit a7c0010

File tree

7 files changed

+95
-43
lines changed

7 files changed

+95
-43
lines changed

‎tl/generate/scheme/ton_api.tl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,7 @@ engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSiz
786786
engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys;
787787

788788
engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard;
789-
engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator;
789+
engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string banned_for:double = engine.validator.collationManagerStats.Collator;
790790
engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard)
791791
collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId;
792792
engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats;

‎tl/generate/scheme/ton_api.tlo‎

36 Bytes
Binary file not shown.

‎validator-engine-console/validator-engine-console-query.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1856,6 +1856,9 @@ td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) {
18561856
}
18571857
sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status;
18581858
}
1859+
if (collator->banned_for_ > 0.0) {
1860+
sb << " banned_for=" << td::StringBuilder::FixedDouble(std::max(collator->banned_for_, 0.0), 3);
1861+
}
18591862
td::TerminalIO::out() << sb.as_cslice() << "\n";
18601863
}
18611864
}

‎validator/collation-manager.cpp‎

Lines changed: 65 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -137,53 +137,61 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
137137

138138
adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero();
139139
size_t selected_idx = 0;
140-
auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool {
141-
auto& collator = collators_[id];
142-
if (!collator.alive) {
143-
return false;
144-
}
145-
if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) {
146-
return false;
147-
}
148-
return true;
149-
};
150-
switch (s->select_mode) {
151-
case CollatorsList::mode_random: {
152-
int cnt = 0;
153-
for (size_t i = 0; i < s->collators.size(); ++i) {
154-
adnl::AdnlNodeIdShort collator = s->collators[i];
155-
if (check_collator(collator)) {
156-
++cnt;
157-
if (td::Random::fast(1, cnt) == 1) {
140+
for (int allow_banned = 0; allow_banned < 2; ++allow_banned) {
141+
auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool {
142+
auto& collator = collators_[id];
143+
if (!collator.alive) {
144+
return false;
145+
}
146+
if (collator.banned_until && !allow_banned) {
147+
return false;
148+
}
149+
if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) {
150+
return false;
151+
}
152+
return true;
153+
};
154+
switch (s->select_mode) {
155+
case CollatorsList::mode_random: {
156+
int cnt = 0;
157+
for (size_t i = 0; i < s->collators.size(); ++i) {
158+
adnl::AdnlNodeIdShort collator = s->collators[i];
159+
if (check_collator(collator)) {
160+
++cnt;
161+
if (td::Random::fast(1, cnt) == 1) {
162+
selected_collator = collator;
163+
selected_idx = i;
164+
}
165+
}
166+
}
167+
break;
168+
}
169+
case CollatorsList::mode_ordered: {
170+
for (size_t i = 0; i < s->collators.size(); ++i) {
171+
adnl::AdnlNodeIdShort collator = s->collators[i];
172+
if (check_collator(collator)) {
158173
selected_collator = collator;
159174
selected_idx = i;
175+
break;
160176
}
161177
}
178+
break;
162179
}
163-
break;
164-
}
165-
case CollatorsList::mode_ordered: {
166-
for (size_t i = 0; i < s->collators.size(); ++i) {
167-
adnl::AdnlNodeIdShort collator = s->collators[i];
168-
if (check_collator(collator)) {
169-
selected_collator = collator;
170-
selected_idx = i;
171-
break;
180+
case CollatorsList::mode_round_robin: {
181+
size_t iters = 0;
182+
for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) {
183+
adnl::AdnlNodeIdShort& collator = s->collators[i];
184+
if (check_collator(collator)) {
185+
selected_collator = collator;
186+
selected_idx = i;
187+
s->cur_idx = (i + 1) % s->collators.size();
188+
break;
189+
}
172190
}
191+
break;
173192
}
174-
break;
175193
}
176-
case CollatorsList::mode_round_robin: {
177-
size_t iters = 0;
178-
for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) {
179-
adnl::AdnlNodeIdShort& collator = s->collators[i];
180-
if (check_collator(collator)) {
181-
selected_collator = collator;
182-
selected_idx = i;
183-
s->cur_idx = (i + 1) % s->collators.size();
184-
break;
185-
}
186-
}
194+
if (!selected_collator.is_zero() || s->self_collate) {
187195
break;
188196
}
189197
}
@@ -372,11 +380,21 @@ void CollationManager::get_stats(
372380
}
373381
obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0;
374382
obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str();
383+
obj->banned_for_ = collator.banned_until ? collator.banned_until.in() : -1.0;
375384
stats->collators_.push_back(std::move(obj));
376385
}
377386
promise.set_value(std::move(stats));
378387
}
379388

389+
void CollationManager::ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason) {
390+
auto it = collators_.find(collator_id);
391+
if (it == collators_.end()) {
392+
return;
393+
}
394+
alarm_timestamp().relax(it->second.banned_until = td::Timestamp::in(BAN_DURATION));
395+
LOG(ERROR) << "Ban collator " << collator_id << " for " << BAN_DURATION << "s: " << reason;
396+
}
397+
380398
void CollationManager::update_collators_list(const CollatorsList& collators_list) {
381399
shards_.clear();
382400
for (auto& [_, collator] : collators_) {
@@ -427,6 +445,14 @@ CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull sha
427445
void CollationManager::alarm() {
428446
alarm_timestamp() = td::Timestamp::never();
429447
for (auto& [id, collator] : collators_) {
448+
if (collator.banned_until) {
449+
if (collator.banned_until.is_in_past()) {
450+
collator.banned_until = {};
451+
LOG(ERROR) << "Unban collator " << id;
452+
} else {
453+
alarm_timestamp().relax(collator.banned_until);
454+
}
455+
}
430456
if (collator.active_cnt == 0 || collator.sent_ping) {
431457
continue;
432458
}

‎validator/collation-manager.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ class CollationManager : public td::actor::Actor {
5454

5555
void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_collationManagerStats_localId>> promise);
5656

57+
void ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason);
58+
5759
private:
5860
adnl::AdnlNodeIdShort local_id_;
5961
td::Ref<ValidatorManagerOptions> opts_;
@@ -77,6 +79,8 @@ class CollationManager : public td::actor::Actor {
7779
td::Timestamp last_ping_at = td::Timestamp::never();
7880
td::Status last_ping_status = td::Status::Error("not pinged");
7981
int version = -1;
82+
// Collator is banned when in returns invalid block
83+
td::Timestamp banned_until = td::Timestamp::never();
8084
};
8185
std::map<adnl::AdnlNodeIdShort, CollatorInfo> collators_;
8286

@@ -104,6 +108,8 @@ class CollationManager : public td::actor::Actor {
104108
size_t refcnt = 0;
105109
};
106110
std::map<BlockIdExt, OptimisticPrevCache> optimistic_prev_cache_;
111+
112+
static constexpr double BAN_DURATION = 300.0;
107113
};
108114

109115
} // namespace ton::validator

‎validator/validator-group.cpp‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo
117117
if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) {
118118
send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone());
119119
}
120+
if (!c.self_collated) {
121+
block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id};
122+
}
120123
cache->result = std::move(c);
121124
for (auto &p : cache->promises) {
122125
p.set_value(cache->result.value().clone());
@@ -182,10 +185,15 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo
182185
return;
183186
}
184187

188+
auto it2 = block_collator_node_id_.find(block.id);
189+
adnl::AdnlNodeIdShort collator_node_id =
190+
it2 == block_collator_node_id_.end() ? adnl::AdnlNodeIdShort::zero() : it2->second;
191+
185192
auto P = td::PromiseCreator::lambda(
186-
[SelfId = actor_id(this), source_info, block = block.clone(),
193+
[=, SelfId = actor_id(this), block = block.clone(),
187194
optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional<BlockCandidate>{},
188-
promise = std::move(promise)](td::Result<ValidateCandidateResult> R) mutable {
195+
promise = std::move(promise),
196+
collation_manager = collation_manager_](td::Result<ValidateCandidateResult> R) mutable {
189197
if (R.is_error()) {
190198
auto S = R.move_as_error();
191199
if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) {
@@ -212,6 +220,10 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo
212220
promise.set_value({ts, false});
213221
},
214222
[&](CandidateReject reject) {
223+
if (!collator_node_id.is_zero()) {
224+
td::actor::send_closure(collation_manager, &CollationManager::ban_collator, collator_node_id,
225+
PSTRING() << "bad candidate " << block.id.to_str() << " : " << reject.reason);
226+
}
215227
promise.set_error(
216228
td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason));
217229
}));
@@ -396,7 +408,11 @@ void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInf
396408
optimistic_generation_ = {};
397409
return;
398410
}
399-
optimistic_generation_->result = R.move_as_ok();
411+
GeneratedCandidate c = R.move_as_ok();
412+
if (!c.self_collated) {
413+
block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id};
414+
}
415+
optimistic_generation_->result = std::move(c);
400416
for (auto &promise : optimistic_generation_->promises) {
401417
promise.set_result(optimistic_generation_->result.value().clone());
402418
}

‎validator/validator-group.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ class ValidatorGroup : public td::actor::Actor {
188188
}
189189

190190
std::set<BlockIdExt> sent_candidate_broadcasts_;
191+
std::map<BlockIdExt, adnl::AdnlNodeIdShort> block_collator_node_id_;
191192

192193
void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data);
193194

0 commit comments

Comments
 (0)