Merge branch 'testnet' into accelerator

Author: SpyCheese

Changelog.md

@@ -1,3 +1,15 @@
+## 2025.06 Update
+
+1. ADNL and candidate broadcast optimization
+2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee`  nullification.
+3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02).
+4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic
+5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods
+6. Optimized package import for archive nodes
+
+Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements).
+
+
 ## 2025.04 Update
 
 1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters.
@@ -8,7 +20,7 @@
 6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467)
 7. Fix SDBEGINS(Q) in Asm.fif
 
-Besides the work of the core team, this update is based on the efforts of  @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif).
+Besides the work of the core team, this update is based on the efforts of  @Stanislav-Povolotsky (tonlib fixes); @ice-charon (tonlib fixes); RSquad team (due payments improvements in v10); Arayz, Robinlzw, @wy666444 @Lucian-code233 from TonBit (improvements in RUNVM); @Skydev0h and @pyAndr3w (Asm.fif).
 
 ## 2025.03 Update
 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10)

assembly/native/build-macos-portable.sh

@@ -65,19 +65,19 @@ else
   echo "Using compiled lz4"
 fi
 
-if [ ! -d "../3pp/libsodium" ]; then
+if [ ! -d "../3pp/libsodium-1.0.18" ]; then
   export LIBSODIUM_FULL_BUILD=1
-  git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium
-  cd ../3pp/libsodium
+  cd ../3pp
+  curl -LO https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz
+  tar -xzf libsodium-1.0.18.tar.gz
+  cd libsodium-1.0.18
   sodiumPath=`pwd`
-  git checkout 1.0.18
-  ./autogen.sh
   ./configure --with-pic --enable-static
   make -j4
   test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; }
   cd ../../build
 else
-  sodiumPath=$(pwd)/../3pp/libsodium
+  sodiumPath=$(pwd)/../3pp/libsodium-1.0.18
   echo "Using compiled libsodium"
 fi
 

crypto/block/account-storage-stat.h

@@ -58,6 +58,10 @@ class AccountStorageStat {
     return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()};
   }
 
+  bool is_dict_ready() const {
+    return dict_up_to_date_;
+  }
+
   void apply_child_stat(AccountStorageStat &&child);
 
   static constexpr int errorcode_limits_exceeded = 999;

recent_changelog.md

@@ -1,12 +1,11 @@
-## 2025.04 Update
+## 2025.06 Update
 
-1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters.
-2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks.
-3. [TVM version v10](./doc/GlobalVersions.md)
-4. Overlay broadcast speed up and improved network stats.
-5. Fixed some issues in tonlib
-6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467)
-7. Fix SDBEGINS(Q) in Asm.fif
+1. ADNL and candidate broadcast optimization
+2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee`  nullification.
+3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02).
+4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic
+5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods
+6. Optimized package import for archive nodes
 
-Besides the work of the core team, this update is based on the efforts of  @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif).
+Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements).
 

tdutils/td/utils/LRUCache.h

@@ -26,8 +26,7 @@ namespace td {
 template <typename K, typename V>
 class LRUCache {
  public:
-  explicit LRUCache(size_t max_size) : max_size_(max_size) {
-    CHECK(max_size_ > 0);
+  explicit LRUCache(uint64 max_size) : max_size_(max_size) {
   }
   LRUCache(const LRUCache&) = delete;
   LRUCache& operator=(const LRUCache&) = delete;
@@ -49,13 +48,14 @@ class LRUCache {
     return cache_.contains(key);
   }
 
-  bool put(const K& key, V value, bool update = true) {
+  bool put(const K& key, V value, bool update = true, uint64 weight = 1) {
     bool added = false;
     auto it = cache_.find(key);
     if (it == cache_.end()) {
       update = true;
-      it = cache_.insert(std::make_unique<Entry>(key, std::move(value))).first;
+      it = cache_.insert(std::make_unique<Entry>(key, std::move(value), weight)).first;
       added = true;
+      total_weight_ += weight;
     } else {
       (*it)->value = std::move(value);
       if (update) {
@@ -69,11 +69,12 @@ class LRUCache {
     return added;
   }
 
-  V& get(const K& key, bool update = true) {
+  V& get(const K& key, bool update = true, uint64 weight = 1) {
     auto it = cache_.find(key);
     if (it == cache_.end()) {
       update = true;
-      it = cache_.insert(std::make_unique<Entry>(key)).first;
+      it = cache_.insert(std::make_unique<Entry>(key, weight)).first;
+      total_weight_ += weight;
     } else if (update) {
       (*it)->remove();
     }
@@ -87,12 +88,13 @@ class LRUCache {
 
  private:
   struct Entry : ListNode {
-    explicit Entry(K key) : key(std::move(key)) {
+    Entry(K key, uint64 weight) : key(std::move(key)), weight(weight) {
     }
-    Entry(K key, V value) : key(std::move(key)), value(std::move(value)) {
+    Entry(K key, V value, uint64 weight) : key(std::move(key)), value(std::move(value)), weight(weight) {
     }
     K key;
     V value;
+    uint64 weight;
   };
   struct Cmp {
     using is_transparent = void;
@@ -108,13 +110,15 @@ class LRUCache {
   };
   std::set<std::unique_ptr<Entry>, Cmp> cache_;
   ListNode lru_;
-  size_t max_size_;
+  uint64 max_size_;
+  uint64 total_weight_ = 0;
 
   void cleanup() {
-    while (cache_.size() > max_size_) {
+    while (total_weight_ > max_size_ && cache_.size() > 1) {
       auto to_remove = (Entry*)lru_.get();
       CHECK(to_remove);
       to_remove->remove();
+      total_weight_ -= to_remove->weight;
       cache_.erase(cache_.find(to_remove->key));
     }
   }

validator/CMakeLists.txt

@@ -59,6 +59,7 @@ set(VALIDATOR_HEADERS
   import-db-slice-local.hpp
   queue-size-counter.hpp
   validator-telemetry.hpp
+  storage-stat-cache.hpp
   shard-block-verifier.hpp
   shard-block-retainer.hpp
 
@@ -93,6 +94,7 @@ set(VALIDATOR_SOURCE
   validator-options.cpp
   queue-size-counter.cpp
   validator-telemetry.cpp
+  storage-stat-cache.cpp
   shard-block-verifier.cpp
   shard-block-retainer.cpp
 

validator/impl/collator-impl.h

@@ -250,6 +250,9 @@ class Collator final : public td::actor::Actor {
   bool deferring_messages_enabled_ = false;
   bool store_out_msg_queue_size_ = false;
 
+  std::function<td::Ref<vm::Cell>(const td::Bits256&)> storage_stat_cache_;
+  std::vector<std::pair<td::Ref<vm::Cell>, td::uint32>> storage_stat_cache_update_;
+
   td::PerfWarningTimer perf_timer_;
   td::PerfLog perf_log_;
   //
@@ -272,6 +275,7 @@ class Collator final : public td::actor::Actor {
   void after_get_shard_state(int idx, td::Result<Ref<ShardState>> res, td::PerfLogAction token);
   void after_get_block_data(int idx, td::Result<Ref<BlockData>> res, td::PerfLogAction token);
   void after_get_shard_blocks(td::Result<std::vector<Ref<ShardTopBlockDescription>>> res, td::PerfLogAction token);
+  void after_get_storage_stat_cache(td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res);
   bool preprocess_prev_mc_state();
   bool register_mc_state(Ref<MasterchainStateQ> other_mc_state);
   bool request_aux_mc_state(BlockSeqno seqno, Ref<MasterchainStateQ>& state);

validator/impl/collator.cpp

@@ -295,7 +295,16 @@ void Collator::start_up() {
                                                                   std::move(res), std::move(token));
                                   });
   }
-  // 6. set timeout
+  // 6. get storage stat cache
+  ++pending;
+  LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager";
+  td::actor::send_closure_later(
+      manager, &ValidatorManager::get_storage_stat_cache,
+      [self = get_self()](td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res) {
+        LOG(DEBUG) << "got answer to get_storage_stat_cache() query";
+        td::actor::send_closure_later(std::move(self), &Collator::after_get_storage_stat_cache, std::move(res));
+      });
+  // 7. set timeout
   alarm_timestamp() = timeout;
   CHECK(pending);
 }
@@ -727,6 +736,22 @@ void Collator::after_get_shard_blocks(td::Result<std::vector<Ref<ShardTopBlockDe
   check_pending();
 }
 
+/**
+ * Callback function called after retrieving storage stat cache.
+ *
+ * @param res The retrieved storage stat cache.
+ */
+void Collator::after_get_storage_stat_cache(td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res) {
+  --pending;
+  if (res.is_error()) {
+    LOG(INFO) << "after_get_storage_stat_cache : " << res.error();
+  } else {
+    LOG(DEBUG) << "after_get_storage_stat_cache : OK";
+    storage_stat_cache_ = res.move_as_ok();
+  }
+  check_pending();
+}
+
 /**
  * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration.
  *
@@ -2630,6 +2655,20 @@ std::unique_ptr<block::Account> Collator::make_account_from(td::ConstBitPtr addr
   if (!init_account_storage_dict(*ptr)) {
     return nullptr;
   }
+  if (storage_stat_cache_ && ptr->storage_dict_hash) {
+    auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value());
+    if (dict_root.not_null()) {
+      auto S = ptr->init_account_storage_stat(dict_root);
+      if (S.is_error()) {
+        fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account "
+                                                     << addr.to_hex(256) << ": "));
+        return nullptr;
+      }
+      LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells
+                 << " cells)";
+      storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells);
+    }
+  }
   return ptr;
 }
 
@@ -2904,6 +2943,10 @@ bool Collator::combine_account_transactions() {
       if (!process_account_storage_dict(acc)) {
         return false;
       }
+      if (acc.storage_dict_hash && acc.account_storage_stat && acc.account_storage_stat.value().is_dict_ready()) {
+        storage_stat_cache_update_.emplace_back(acc.account_storage_stat.value().get_dict_root().move_as_ok(),
+                                                acc.storage_used.cells);
+      }
     } else {
       if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) {
         return fatal_error(std::string{"total state of account "} + z.first.to_hex() +
@@ -6227,6 +6270,7 @@ bool Collator::create_block_candidate() {
     td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_),
                                   std::move(bad_ext_msgs_));
   }
+  td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_));
   return true;
 }
 

validator/impl/validate-query.cpp

@@ -394,6 +394,15 @@ void ValidateQuery::start_up() {
       return;
     }
   }
+  // 5. get storage stat cache
+  ++pending;
+  LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager";
+  td::actor::send_closure_later(
+      manager, &ValidatorManager::get_storage_stat_cache,
+      [self = get_self()](td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res) {
+        LOG(DEBUG) << "got answer to get_storage_stat_cache() query";
+        td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res));
+      });
   // ...
   CHECK(pending);
 }
@@ -783,6 +792,26 @@ void ValidateQuery::got_mc_handle(td::Result<BlockHandle> res) {
       });
 }
 
+/**
+ * Callback function called after retrieving storage stat cache.
+ *
+ * @param res The retrieved storage stat cache.
+ */
+void ValidateQuery::after_get_storage_stat_cache(td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res) {
+  --pending;
+  if (res.is_error()) {
+    LOG(INFO) << "after_get_storage_stat_cache : " << res.error();
+  } else {
+    LOG(DEBUG) << "after_get_storage_stat_cache : OK";
+    storage_stat_cache_ = res.move_as_ok();
+  }
+  if (!pending) {
+    if (!try_validate()) {
+      fatal_error("cannot validate new block");
+    }
+  }
+}
+
 /**
  * Callback function called after retrieving the shard state for a previous block.
  *
@@ -5287,6 +5316,20 @@ std::unique_ptr<block::Account> ValidateQuery::make_account_from(td::ConstBitPtr
     return nullptr;
   }
   ptr->block_lt = start_lt_;
+  if (storage_stat_cache_ && ptr->storage_dict_hash) {
+    auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value());
+    if (dict_root.not_null()) {
+      auto S = ptr->init_account_storage_stat(dict_root);
+      if (S.is_error()) {
+        fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account "
+                                                     << addr.to_hex(256) << ": "));
+        return nullptr;
+      }
+      LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells
+                 << " cells)";
+      storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells);
+    }
+  }
   return ptr;
 }
 
@@ -5912,6 +5955,11 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re
       })) {
     return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid");
   }
+  if (account.storage_dict_hash && account.account_storage_stat &&
+      account.account_storage_stat.value().is_dict_ready()) {
+    storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(),
+                                            account.storage_used.cells);
+  }
   if (is_masterchain() && account.libraries_changed()) {
     return scan_account_libraries(account.orig_library, account.library, acc_addr);
   } else {
@@ -7108,6 +7156,7 @@ bool ValidateQuery::save_candidate() {
 
   td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(),
                           validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P));
+  td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_));
   return true;
 }
 

validator/impl/validate-query.hpp

@@ -242,6 +242,9 @@ class ValidateQuery : public td::actor::Actor {
   bool out_msg_queue_size_known_ = false;
   bool have_out_msg_queue_size_in_state_ = false;
 
+  std::function<td::Ref<vm::Cell>(const td::Bits256&)> storage_stat_cache_;
+  std::vector<std::pair<td::Ref<vm::Cell>, td::uint32>> storage_stat_cache_update_;
+
   bool msg_metadata_enabled_ = false;
   bool deferring_messages_enabled_ = false;
   bool store_out_msg_queue_size_ = false;
@@ -295,6 +298,7 @@ class ValidateQuery : public td::actor::Actor {
   void after_get_latest_mc_state(td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res);
   void after_get_mc_state(td::Result<Ref<ShardState>> res);
   void got_mc_handle(td::Result<BlockHandle> res);
+  void after_get_storage_stat_cache(td::Result<std::function<td::Ref<vm::Cell>(const td::Bits256&)>> res);
   void after_get_shard_state(int idx, td::Result<Ref<ShardState>> res);
   bool process_mc_state(Ref<MasterchainState> mc_state);
   bool try_unpack_mc_state();