Merge branch 'testnet' into tvm-v9

SpyCheese · SpyCheese · commit 77e5a2f4a235 · 2025-01-17T12:32:09.000+03:00
diff --git a/Changelog.md b/Changelog.md
@@ -31,7 +31,7 @@ Besides the work of the core team, this update is based on the efforts of @krigg
 ## 2024.08 Update
 
 1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. 
-2. A number of changes to transcation executor which will activated for `Config8.version >= 8`:
+2. A number of changes to transaction executor which will activated for `Config8.version >= 8`:
     - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set.
     - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`.
     - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent.
@@ -103,7 +103,7 @@ Besides the work of the core team, this update is based on the efforts of @akifo
    * Fix error in proof generation for blocks after merge
    * Fix most of `block is not applied` issues related to sending too recent block in Proofs
    * LS now check external messages till `accept_message` (`set_gas`).
-3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator.
+3. Improvements in DHT work and storage, CellDb, config.json amendment, peer misbehavior detection, validator session stats collection, emulator.
 4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`:
    * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
    * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
@@ -114,7 +114,7 @@ Besides the work of the Core team, this update is based on the efforts of @XaBbl
 ## 2023.12 Update
 
 1. Optimized message queue handling, now queue cleaning speed doesn't depend on total queue size
-     * Cleaning delivered messages using lt augmentation instead of random search / consequtive walk
+     * Cleaning delivered messages using lt augmentation instead of random search / consecutive walk
      * Keeping root cell of queue message in memory until outdated (caching)
 2. Changes to block collation/validation limits
 3. Stop accepting new external message if message queue is overloaded
@@ -206,7 +206,7 @@ Besides the work of the core team, this update is based on the efforts of @vtama
 Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds)  and third-party security auditors.
 
 ## 2022.08 Update
-* Blockchain state serialization now works via separate db-handler which simplfies memory clearing after serialization
+* Blockchain state serialization now works via separate db-handler which simplifies memory clearing after serialization
 * CellDB now works asynchronously which substantially increase database access throughput
 * Abseil-cpp and crc32 updated: solve issues with compilation on recent OS distributives
 * Fixed a series of UBs and issues for exotic endianness hosts
diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp
@@ -119,6 +119,7 @@ void AdnlPeerPairImpl::discover() {
 void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) {
   last_received_packet_ = td::Timestamp::now();
   try_reinit_at_ = td::Timestamp::never();
+  drop_addr_list_at_ = td::Timestamp::never();
   request_reverse_ping_after_ = td::Timestamp::in(15.0);
   auto d = Adnl::adnl_start_time();
   if (packet.dst_reinit_date() > d) {
@@ -415,6 +416,9 @@ void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorI
   if (!try_reinit_at_ && last_received_packet_ < td::Timestamp::in(-5.0)) {
     try_reinit_at_ = td::Timestamp::in(10.0);
   }
+  if (!drop_addr_list_at_ && last_received_packet_ < td::Timestamp::in(-60.0 * 9.0)) {
+    drop_addr_list_at_ = td::Timestamp::in(60.0);
+  }
   packet.run_basic_checks().ensure();
   auto B = serialize_tl_object(packet.tl(), true);
   if (via_channel) {
@@ -692,6 +696,16 @@ void AdnlPeerPairImpl::reinit(td::int32 date) {
 }
 
 td::Result<std::pair<td::actor::ActorId<AdnlNetworkConnection>, bool>> AdnlPeerPairImpl::get_conn() {
+  if (drop_addr_list_at_ && drop_addr_list_at_.is_in_past()) {
+    drop_addr_list_at_ = td::Timestamp::never();
+    priority_addr_list_ = AdnlAddressList{};
+    priority_conns_.clear();
+    addr_list_ = AdnlAddressList{};
+    conns_.clear();
+    has_reverse_addr_ = false;
+    return td::Status::Error(ErrorCode::notready, "no active connections");
+  }
+
   if (!priority_addr_list_.empty() && priority_addr_list_.expire_at() < td::Clocks::system()) {
     priority_addr_list_ = AdnlAddressList{};
     priority_conns_.clear();
diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp
@@ -266,6 +266,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
 
   td::Timestamp last_received_packet_ = td::Timestamp::never();
   td::Timestamp try_reinit_at_ = td::Timestamp::never();
+  td::Timestamp drop_addr_list_at_ = td::Timestamp::never();
 
   bool has_reverse_addr_ = false;
   td::Timestamp request_reverse_ping_after_ = td::Timestamp::now();
diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp
@@ -1319,6 +1319,36 @@ CurrencyCollection CurrencyCollection::operator-(td::RefInt256 other_grams) cons
   }
 }
 
+bool CurrencyCollection::clamp(const CurrencyCollection& other) {
+  if (!is_valid() || !other.is_valid()) {
+    return invalidate();
+  }
+  grams = std::min(grams, other.grams);
+  vm::Dictionary dict1{extra, 32}, dict2(other.extra, 32);
+  bool ok = dict1.check_for_each([&](td::Ref<vm::CellSlice> cs1, td::ConstBitPtr key, int n) {
+    CHECK(n == 32);
+    td::Ref<vm::CellSlice> cs2 = dict2.lookup(key, 32);
+    td::RefInt256 val1 = tlb::t_VarUIntegerPos_32.as_integer(cs1);
+    if (val1.is_null()) {
+      return false;
+    }
+    td::RefInt256 val2 = cs2.is_null() ? td::zero_refint() : tlb::t_VarUIntegerPos_32.as_integer(cs2);
+    if (val2.is_null()) {
+      return false;
+    }
+    if (val1 > val2) {
+      if (val2->sgn() == 0) {
+        dict1.lookup_delete(key, 32);
+      } else {
+        dict1.set(key, 32, cs2);
+      }
+    }
+    return true;
+  });
+  extra = dict1.get_root_cell();
+  return ok || invalidate();
+}
+
 bool CurrencyCollection::operator==(const CurrencyCollection& other) const {
   return is_valid() && other.is_valid() && !td::cmp(grams, other.grams) &&
          (extra.not_null() == other.extra.not_null()) &&
diff --git a/crypto/block/block.h b/crypto/block/block.h
@@ -390,6 +390,7 @@ struct CurrencyCollection {
   CurrencyCollection operator-(const CurrencyCollection& other) const;
   CurrencyCollection operator-(CurrencyCollection&& other) const;
   CurrencyCollection operator-(td::RefInt256 other_grams) const;
+  bool clamp(const CurrencyCollection& other);
   bool store(vm::CellBuilder& cb) const;
   bool store_or_zero(vm::CellBuilder& cb) const;
   bool fetch(vm::CellSlice& cs);
diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb
@@ -296,7 +296,7 @@ transaction$0111 account_addr:bits256 lt:uint64
   total_fees:CurrencyCollection state_update:^(HASH_UPDATE Account)
   description:^TransactionDescr = Transaction;
 
-!merkle_update#02 {X:Type} old_hash:bits256 new_hash:bits256
+!merkle_update#04 {X:Type} old_hash:bits256 new_hash:bits256 old_depth:uint16 new_depth:uint16
   old:^X new:^X = MERKLE_UPDATE X;
 update_hashes#72 {X:Type} old_hash:bits256 new_hash:bits256
   = HASH_UPDATE X;
diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp
@@ -2795,22 +2795,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap,
     LOG(DEBUG) << "cannot reserve a negative amount: " << reserve.to_str();
     return -1;
   }
-  if (reserve.grams > ap.remaining_balance.grams) {
-    if (mode & 2) {
-      reserve.grams = ap.remaining_balance.grams;
+  if (mode & 2) {
+    if (cfg.reserve_extra_enabled) {
+      if (!reserve.clamp(ap.remaining_balance)) {
+        LOG(DEBUG) << "failed to clamp reserve amount" << mode;
+        return -1;
+      }
     } else {
-      LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams
-                 << " available";
-      return 37;  // not enough grams
+      reserve.grams = std::min(reserve.grams, ap.remaining_balance.grams);
     }
   }
+  if (reserve.grams > ap.remaining_balance.grams) {
+    LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams
+               << " available";
+    return 37;  // not enough grams
+  }
   if (!block::sub_extra_currency(ap.remaining_balance.extra, reserve.extra, newc.extra)) {
     LOG(DEBUG) << "not enough extra currency to reserve: " << block::CurrencyCollection{0, reserve.extra}.to_str()
                << " required, only " << block::CurrencyCollection{0, ap.remaining_balance.extra}.to_str()
                << " available";
-    if (mode & 2) {
-      // TODO: process (mode & 2) correctly by setting res_extra := inf (reserve.extra, ap.remaining_balance.extra)
-    }
     return 38;  // not enough (extra) funds
   }
   newc.grams = ap.remaining_balance.grams - reserve.grams;
@@ -3813,6 +3816,7 @@ td::Status FetchConfigParams::fetch_config_params(
     action_phase_cfg->bounce_on_fail_enabled = config.get_global_version() >= 4;
     action_phase_cfg->message_skip_enabled = config.get_global_version() >= 8;
     action_phase_cfg->disable_custom_fess = config.get_global_version() >= 8;
+    action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9;
     action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr;
   }
   {
diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h
@@ -169,6 +169,7 @@ struct ActionPhaseConfig {
   bool bounce_on_fail_enabled{false};
   bool message_skip_enabled{false};
   bool disable_custom_fess{false};
+  bool reserve_extra_enabled{false};
   td::optional<td::Bits256> mc_blackhole_addr;
   const MsgPrices& fetch_msg_prices(bool is_masterchain) const {
     return is_masterchain ? fwd_mc : fwd_std;
diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp
@@ -149,16 +149,17 @@ td::Ref<vm::Tuple> prepare_vm_c7(SmartContract::Args args, td::Ref<vm::Cell> cod
   }
 
   std::vector<vm::StackEntry> tuple = {
-      td::make_refint(0x076ef1ea),                            // [ magic:0x076ef1ea
-      td::make_refint(0),                                     //   actions:Integer
-      td::make_refint(0),                                     //   msgs_sent:Integer
-      td::make_refint(now),                                   //   unixtime:Integer
-      td::make_refint(0),              //TODO:                //   block_lt:Integer
-      td::make_refint(0),              //TODO:                //   trans_lt:Integer
-      std::move(rand_seed_int),                               //   rand_seed:Integer
-      block::CurrencyCollection(args.balance).as_vm_tuple(),  //   balance_remaining:[Integer (Maybe Cell)]
-      vm::load_cell_slice_ref(address),                       //   myself:MsgAddressInt
-      vm::StackEntry::maybe(config)                           //   vm::StackEntry::maybe(td::Ref<vm::Cell>())
+      td::make_refint(0x076ef1ea),  // [ magic:0x076ef1ea
+      td::make_refint(0),           //   actions:Integer
+      td::make_refint(0),           //   msgs_sent:Integer
+      td::make_refint(now),         //   unixtime:Integer
+      td::make_refint(0),           //   block_lt:Integer (TODO)
+      td::make_refint(0),           //   trans_lt:Integer (TODO)
+      std::move(rand_seed_int),     //   rand_seed:Integer
+      block::CurrencyCollection(args.balance, args.extra_currencies)
+          .as_vm_tuple(),                //   balance_remaining:[Integer (Maybe Cell)]
+      vm::load_cell_slice_ref(address),  //   myself:MsgAddressInt
+      vm::StackEntry::maybe(config)      //   vm::StackEntry::maybe(td::Ref<vm::Cell>())
   };
   if (args.config && args.config.value()->get_global_version() >= 4) {
     tuple.push_back(vm::StackEntry::maybe(code));                      // code:Cell
diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h
@@ -64,6 +64,7 @@ class SmartContract : public td::CntObject {
     bool ignore_chksig{false};
     td::uint64 amount{0};
     td::uint64 balance{0};
+    td::Ref<vm::Cell> extra_currencies;
     int vm_log_verbosity_level{0};
     bool debug_enabled{false};
 
@@ -121,6 +122,10 @@ class SmartContract : public td::CntObject {
       this->balance = balance;
       return std::move(*this);
     }
+    Args&& set_extra_currencies(td::Ref<vm::Cell> extra_currencies) {
+      this->extra_currencies = std::move(extra_currencies);
+      return std::move(*this);
+    }
     Args&& set_address(block::StdAddress address) {
       this->address = address;
       return std::move(*this);
diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp
@@ -2074,7 +2074,7 @@ void CppTypeCode::generate_skip_field(const Constructor& constr, const Field& fi
     output_cpp_expr(ss, expr, 100);
     ss << '.';
   }
-  ss << "validate_skip_ref(ops, cs, weak)" << tail;
+  ss << "validate_skip_ref(ops, cs, " << (constr.is_special ? "true" : "weak") << ")" << tail;
   actions += Action{ss.str()};
 }
 
diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp
@@ -1800,9 +1800,6 @@ void Constructor::show(std::ostream& os, int mode) const {
   }
   for (int i = 0; i < type_arity; i++) {
     os << ' ';
-    if (param_negated.at(i)) {
-      os << '~';
-    }
     params.at(i)->show(os, this, 100, mode | 1);
   }
   if (!(mode & 2)) {
diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp
@@ -133,7 +133,13 @@ bool TLB::validate_ref_internal(int* ops, Ref<vm::Cell> cell_ref, bool weak) con
   }
   bool is_special;
   auto cs = load_cell_slice_special(std::move(cell_ref), is_special);
-  return always_special() ? is_special : (is_special ? weak : (validate_skip(ops, cs) && cs.empty_ext()));
+  if (cs.special_type() == vm::Cell::SpecialType::PrunnedBranch && weak) {
+    return true;
+  }
+  if (always_special() != is_special) {
+    return false;
+  }
+  return validate_skip(ops, cs, weak) && cs.empty_ext();
 }
 
 bool TLB::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const {
diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp
@@ -100,8 +100,18 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
     return get_cell_info_lazy(level_mask, hash, depth).cell;
   }
   td::Result<Ref<DataCell>> load_cell(td::Slice hash) override {
-    TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell());
-    return std::move(loaded_cell.data_cell);
+    auto info = hash_table_.get_if_exists(hash);
+    if (info && info->sync_with_db) {
+      TRY_RESULT(loaded_cell, info->cell->load_cell());
+      return std::move(loaded_cell.data_cell);
+    }
+    TRY_RESULT(res, loader_->load(hash, true, *this));
+    if (res.status != CellLoader::LoadResult::Ok) {
+      return td::Status::Error("cell not found");
+    }
+    Ref<DataCell> cell = res.cell();
+    hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); });
+    return cell;
   }
   td::Result<Ref<DataCell>> load_root(td::Slice hash) override {
     return load_cell(hash);
@@ -145,9 +155,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
           promise->set_result(std::move(cell));
         });
   }
-  CellInfo &get_cell_info_force(td::Slice hash) {
-    return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); });
-  }
   CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) {
     return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes),
                              [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); });
diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp
@@ -1301,7 +1301,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) {
 }
 
 int exec_compute_data_size(VmState* st, int mode) {
-  VM_LOG(st) << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : "");
+  VM_LOG(st) << "execute " << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : "");
   Stack& stack = st->get_stack();
   stack.check_underflow(2);
   auto bound = stack.pop_int();
diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md
@@ -129,6 +129,7 @@ Example: if the last masterchain block seqno is `19071` then the list contains b
 - Fix `RAWRESERVE` action with flag `4` (use original balance of the account) by explicitly setting `original_balance` to `balance - msg_balance_remaining`.
   - Previously it did not work if storage fee was greater than the original balance.
 - Jumps to nested continuations of depth more than 8 consume 1 gas for eact subsequent continuation (this does not affect most of TVM code).
+- Support extra currencies in reserve action with `+2` mode.
 - Fix exception code in some TVM instructions: now `stk_und` has priority over other error codes.
   - `PFXDICTADD`, `PFXDICTSET`, `PFXDICTREPLACE`, `PFXDICTDEL`, `GETGASFEE`, `GETSTORAGEFEE`, `GETFORWARDFEE`, `GETORIGINALFWDFEE`, `GETGASFEESIMPLE`, `GETFORWARDFEESIMPLE`, `HASHEXT`
 - Now setting the contract code to a library cell does not consume additional gas on execution of the code.
diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp
@@ -615,7 +615,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc
   emulator->set_vm_verbosity_level(0);
   emulator->set_gas_limit(gas_limit);
   emulator->set_c7_raw(c7->fetch(0).as_tuple());
-  if (libs.is_empty()) {
+  if (!libs.is_empty()) {
     emulator->set_libraries(std::move(libs));
   }
   auto result = emulator->run_get_method(int(method_id), stack);
diff --git a/tl/tl/tl_jni_object.cpp b/tl/tl/tl_jni_object.cpp
@@ -115,8 +115,9 @@ static size_t get_utf8_from_utf16_length(const jchar *p, jsize len) {
   for (jsize i = 0; i < len; i++) {
     unsigned int cur = p[i];
     if ((cur & 0xF800) == 0xD800) {
+      ++i;
       if (i < len) {
-        unsigned int next = p[++i];
+        unsigned int next = p[i];
         if ((next & 0xFC00) == 0xDC00 && (cur & 0x400) == 0) {
           result += 4;
           continue;
diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp
@@ -1050,15 +1050,17 @@ class Query {
     }
 
     vm::GasLimits gas_limits = compute_gas_limits(td::make_refint(raw_.source->get_balance()), gas_limits_prices);
-    auto res = smc.write().send_external_message(raw_.message_body, ton::SmartContract::Args()
-                                                                        .set_limits(gas_limits)
-                                                                        .set_balance(raw_.source->get_balance())
-                                                                        .set_now(raw_.source->get_sync_time())
-                                                                        .set_ignore_chksig(ignore_chksig)
-                                                                        .set_address(raw_.source->get_address())
-                                                                        .set_config(cfg)
-                                                                        .set_prev_blocks_info(state.prev_blocks_info)
-                                                                        .set_libraries(libraries));
+    auto res = smc.write().send_external_message(raw_.message_body,
+                                                 ton::SmartContract::Args()
+                                                     .set_limits(gas_limits)
+                                                     .set_balance(raw_.source->get_balance())
+                                                     .set_extra_currencies(raw_.source->get_extra_currencies())
+                                                     .set_now(raw_.source->get_sync_time())
+                                                     .set_ignore_chksig(ignore_chksig)
+                                                     .set_address(raw_.source->get_address())
+                                                     .set_config(cfg)
+                                                     .set_prev_blocks_info(state.prev_blocks_info)
+                                                     .set_libraries(libraries));
     td::int64 fwd_fee = 0;
     if (res.success) {
       LOG(DEBUG) << "output actions:\n"
@@ -4790,6 +4792,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
   }
   args.set_stack(std::move(stack));
   args.set_balance(it->second->get_balance());
+  args.set_extra_currencies(it->second->get_extra_currencies());
   args.set_now(it->second->get_sync_time());
   args.set_address(it->second->get_address());
 
diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp
diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp
diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp
diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp
diff --git a/validator/manager.cpp b/validator/manager.cpp
