Merge pull request #1761 from ton-blockchain/testnet

Merge developer branch

Author: EmelyanenkoK

Changelog.md

@@ -1,3 +1,7 @@
+## 2025.07 Accelerator Update
+
+Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators)
+
 ## 2025.06 Update
 
 1. ADNL and candidate broadcast optimization

adnl/adnl-node-id.hpp

@@ -37,7 +37,9 @@ class AdnlNodeIdShort {
   }
   explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) {
   }
-  explicit AdnlNodeIdShort(tl_object_ptr<ton_api::adnl_id_short> obj) : hash_(obj->id_) {
+  explicit AdnlNodeIdShort(tl_object_ptr<ton_api::adnl_id_short> &&obj) : hash_(obj->id_) {
+  }
+  explicit AdnlNodeIdShort(const tl_object_ptr<ton_api::adnl_id_short> &obj) : hash_(obj->id_) {
   }
 
   const auto &pubkey_hash() const {

crypto/block/block.cpp

@@ -676,10 +676,25 @@ bool ParamLimits::deserialize(vm::CellSlice& cs) {
 }
 
 bool BlockLimits::deserialize(vm::CellSlice& cs) {
-  return cs.fetch_ulong(8) == 0x5d     // block_limits#5d
-         && bytes.deserialize(cs)      // bytes:ParamLimits
-         && gas.deserialize(cs)        // gas:ParamLimits
-         && lt_delta.deserialize(cs);  // lt_delta:ParamLimits
+  auto tag = cs.fetch_ulong(8);
+  if (tag != 0x5d && tag != 0x5e) {
+    return false;
+  }
+  // block_limits#5d
+  // block_limits_v2#5e
+  bool ok = bytes.deserialize(cs)         // bytes:ParamLimits
+            && gas.deserialize(cs)        // gas:ParamLimits
+            && lt_delta.deserialize(cs);  // lt_delta:ParamLimits
+  if (!ok) {
+    return false;
+  }
+  if (tag == 0x5e) {
+    return collated_data.deserialize(cs) &&     // collated_data:ParamLimits
+           imported_msg_queue.deserialize(cs);  // imported_msg_queue:ImportedMsgQueueLimits
+  } else {
+    collated_data = bytes;
+    return true;
+  }
 }
 
 int ParamLimits::classify(td::uint64 value) const {
@@ -711,12 +726,19 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const {
   return lt_delta.classify(lt - start_lt);
 }
 
-int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const {
-  return std::max(std::max(classify_size(size), classify_gas(gas)), classify_lt(lt));
+int BlockLimits::classify_collated_data_size(td::uint64 size) const {
+  return collated_data.classify(size);
+}
+
+int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const {
+  return std::max(
+      {classify_size(size), classify_gas(gas), classify_lt(lt), classify_collated_data_size(collated_size)});
 }
 
-bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt) const {
-  return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt);
+bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt,
+                       td::uint64 collated_size) const {
+  return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) &&
+         collated_data.fits(cls, collated_size);
 }
 
 td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const {
@@ -729,20 +751,30 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S
 }
 
 int BlockLimitStatus::classify() const {
-  return limits.classify(estimate_block_size(), gas_used, cur_lt);
+  return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_size_estimate);
 }
 
 bool BlockLimitStatus::fits(unsigned cls) const {
   return cls >= ParamLimits::limits_cnt ||
          (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) &&
-          limits.bytes.fits(cls, estimate_block_size()));
+          limits.bytes.fits(cls, estimate_block_size()) && limits.collated_data.fits(cls, collated_data_size_estimate));
 }
 
 bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas,
                                  const vm::NewCellStorageStat::Stat* extra) const {
   return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) &&
                                             limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) &&
-                                            limits.bytes.fits(cls, estimate_block_size(extra)));
+                                            limits.bytes.fits(cls, estimate_block_size(extra)) &&
+                                            limits.collated_data.fits(cls, collated_data_size_estimate));
+}
+
+double BlockLimitStatus::load_fraction(unsigned cls) const {
+  if (cls >= ParamLimits::limits_cnt) {
+    return 0.0;
+  }
+  return std::max({(double)estimate_block_size() / (double)limits.bytes.limit(cls),
+                   (double)gas_used / (double)limits.gas.limit(cls),
+                   (double)collated_data_size_estimate / (double)limits.collated_data.limit(cls)});
 }
 
 // SETS: account_dict, shard_libraries_, mc_state_extra

crypto/block/block.h

@@ -219,7 +219,7 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC
 struct ImportedMsgQueueLimits {
   // Default values
   td::uint32 max_bytes = 1 << 16;
-  td::uint32 max_msgs = 30;
+  td::uint32 max_msgs = 100;
   bool deserialize(vm::CellSlice& cs);
   ImportedMsgQueueLimits operator*(td::uint32 x) const {
     return {max_bytes * x, max_msgs * x};
@@ -242,6 +242,9 @@ struct ParamLimits {
   td::uint32 hard() const {
     return limits_[3];
   }
+  td::uint32 limit(unsigned cls) const {
+    return limits_[cls];
+  }
   bool compute_medium_limit() {
     limits_[2] = soft() + ((hard() - soft()) >> 1);
     return true;
@@ -261,15 +264,17 @@ struct ParamLimits {
 };
 
 struct BlockLimits {
-  ParamLimits bytes, gas, lt_delta;
+  ParamLimits bytes, gas, lt_delta, collated_data;
   ton::LogicalTime start_lt{0};
+  ImportedMsgQueueLimits imported_msg_queue;
   const vm::CellUsageTree* usage_tree{nullptr};
   bool deserialize(vm::CellSlice& cs);
   int classify_size(td::uint64 size) const;
   int classify_gas(td::uint64 gas) const;
   int classify_lt(ton::LogicalTime lt) const;
-  int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const;
-  bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const;
+  int classify_collated_data_size(td::uint64 size) const;
+  int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const;
+  bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const;
 };
 
 struct BlockLimitStatus {
@@ -278,6 +283,7 @@ struct BlockLimitStatus {
   td::uint64 gas_used{};
   vm::NewCellStorageStat st_stat;
   unsigned accounts{}, transactions{}, extra_out_msgs{};
+  td::uint64 collated_data_size_estimate = 0;
   unsigned public_library_diff{};
   BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0)
       : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) {
@@ -289,12 +295,14 @@ struct BlockLimitStatus {
     gas_used = 0;
     extra_out_msgs = 0;
     public_library_diff = 0;
+    collated_data_size_estimate = 0;
   }
   td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const;
   int classify() const;
   bool fits(unsigned cls) const;
   bool would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas,
                  const vm::NewCellStorageStat::Stat* extra = nullptr) const;
+  double load_fraction(unsigned cls) const;
   bool add_cell(Ref<vm::Cell> cell) {
     st_stat.add_cell(std::move(cell));
     return true;

crypto/block/block.tlb

@@ -716,9 +716,13 @@ config_gas_prices#_ GasLimitsPrices = ConfigParam 21;
 
 param_limits#c3 underload:# soft_limit:# { underload <= soft_limit }
   hard_limit:# { soft_limit <= hard_limit } = ParamLimits;
+imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits;
 block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits
   = BlockLimits;
-  
+block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits
+  collated_data:ParamLimits imported_msg_queue:ImportedMsgQueueLimits
+  = BlockLimits;
+
 config_mc_block_limits#_ BlockLimits = ConfigParam 22;
 config_block_limits#_ BlockLimits = ConfigParam 23;
 
@@ -842,6 +846,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures)
 //  COLLATED DATA
 //
 top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet;
+account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof;
 
 //
 //  VALIDATOR MISBEHAVIOR COMPLAINTS

crypto/block/mc-config.h

@@ -546,6 +546,11 @@ struct PrecompiledContractsConfig {
   td::optional<Contract> get_contract(td::Bits256 code_hash) const;
 };
 
+struct CollatorNodeDescr {
+  ton::ShardIdFull shard;
+  ton::NodeIdShort adnl_id;
+};
+
 class Config {
   enum {
     default_mc_catchain_lifetime = 200,

crypto/block/output-queue-merger.cpp

@@ -134,31 +134,22 @@ bool OutputQueueMerger::MsgKeyValue::split(MsgKeyValue& second) {
   return true;
 }
 
-bool OutputQueueMerger::add_root(int src, Ref<vm::Cell> outmsg_root) {
+void OutputQueueMerger::add_root(int src, Ref<vm::Cell> outmsg_root, td::int32 msg_limit) {
   if (outmsg_root.is_null()) {
-    return true;
+    return;
   }
   auto kv = std::make_unique<MsgKeyValue>(src, std::move(outmsg_root));
   if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) {
     heap.push_back(std::move(kv));
   }
-  return true;
-}
-
-OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector<Neighbor> _neighbors)
-    : queue_for(_queue_for), neighbors(std::move(_neighbors)), eof(false), failed(false) {
-  init();
-}
-
-OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector<block::McShardDescr> _neighbors)
-    : queue_for(_queue_for), eof(false), failed(false) {
-  for (auto& nb : _neighbors) {
-    neighbors.emplace_back(nb.top_block_id(), nb.outmsg_root, nb.is_disabled());
+  if ((int)src_remaining_msgs_.size() < src + 1) {
+    src_remaining_msgs_.resize(src + 1);
   }
-  init();
+  src_remaining_msgs_[src] = msg_limit;
 }
 
-void OutputQueueMerger::init() {
+OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull queue_for, std::vector<OutputQueueMerger::Neighbor> neighbors)
+    : eof(false), failed(false) {
   common_pfx.bits().store_int(queue_for.workchain, 32);
   int l = queue_for.pfx_len();
   td::bitstring::bits_store_long_top(common_pfx.bits() + 32, queue_for.shard, l);
@@ -168,7 +159,7 @@ void OutputQueueMerger::init() {
     if (!neighbor.disabled_) {
       LOG(DEBUG) << "adding " << (neighbor.outmsg_root_.is_null() ? "" : "non-") << "empty output queue for neighbor #"
                  << i << " (" << neighbor.block_id_.to_str() << ")";
-      add_root(i++, neighbor.outmsg_root_);
+      add_root(i++, neighbor.outmsg_root_, neighbor.msg_limit_);
     } else {
       LOG(DEBUG) << "skipping output queue for disabled neighbor #" << i;
       i++;
@@ -222,7 +213,18 @@ bool OutputQueueMerger::load() {
     heap.pop_back();
   } while (!heap.empty() && heap[0]->lt <= lt);
   std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less);
-  return true;
+  for (size_t i = orig_size; i < msg_list.size(); ++i) {
+    td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source];
+    if (remaining != -1) {
+      if (remaining == 0) {
+        limit_exceeded = true;
+      } else {
+        --remaining;
+      }
+    }
+    msg_list[i]->limit_exceeded = limit_exceeded;
+  }
+  return msg_list.size() > orig_size;
 }
 
 }  // namespace block

crypto/block/output-queue-merger.h

@@ -32,6 +32,7 @@ struct OutputQueueMerger {
     int source;
     int key_len{0};
     td::BitArray<max_key_len> key;
+    bool limit_exceeded{false};
     MsgKeyValue() = default;
     MsgKeyValue(int src, Ref<vm::Cell> node);
     MsgKeyValue(td::ConstBitPtr key_pfx, int key_pfx_len, int src, Ref<vm::Cell> node);
@@ -51,23 +52,22 @@ struct OutputQueueMerger {
     bool unpack_node(td::ConstBitPtr key_pfx, int key_pfx_len, Ref<vm::Cell> node);
     bool split(MsgKeyValue& second);
   };
+  //
+  std::vector<std::unique_ptr<MsgKeyValue>> msg_list;
+
+ public:
   struct Neighbor {
     ton::BlockIdExt block_id_;
     td::Ref<vm::Cell> outmsg_root_;
     bool disabled_;
+    td::int32 msg_limit_;  // -1 - unlimited
     Neighbor() = default;
-    Neighbor(ton::BlockIdExt block_id, td::Ref<vm::Cell> outmsg_root, bool disabled = false)
-        : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled) {
+    Neighbor(ton::BlockIdExt block_id, td::Ref<vm::Cell> outmsg_root, bool disabled = false, td::int32 msg_limit = -1)
+        : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) {
     }
   };
-  //
-  ton::ShardIdFull queue_for;
-  std::vector<std::unique_ptr<MsgKeyValue>> msg_list;
-  std::vector<Neighbor> neighbors;
 
- public:
-  OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector<Neighbor> _neighbors);
-  OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector<block::McShardDescr> _neighbors);
+  OutputQueueMerger(ton::ShardIdFull queue_for, std::vector<Neighbor> neighbors);
   bool is_eof() const {
     return eof;
   }
@@ -80,10 +80,11 @@ struct OutputQueueMerger {
   int common_pfx_len;
   std::vector<std::unique_ptr<MsgKeyValue>> heap;
   std::size_t pos{0};
+  std::vector<td::int32> src_remaining_msgs_;
   bool eof;
   bool failed;
-  void init();
-  bool add_root(int src, Ref<vm::Cell> outmsg_root);
+  bool limit_exceeded{false};
+  void add_root(int src, Ref<vm::Cell> outmsg_root, td::int32 msg_limit);
   bool load();
 };