Merge branch 'testnet' into archive-sync

Author: SpyCheese

.github/workflows/docker-ubuntu-image.yml

@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - 'master'
-
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}

CMakeLists.txt

@@ -77,9 +77,12 @@ endif()
 
 if (WIN32)
   message("Add wingetopt")
-  add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL)
+  function(wingetopt_scope)
+    set(CMAKE_POLICY_VERSION_MINIMUM "3.10")
+    add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL)
+  endfunction()
+  wingetopt_scope()
   set(WINGETOPT_FOUND 1)
-  message(STATUS "Use wingetopt")
 endif()
 
 set(CRC32C_BUILD_TESTS OFF CACHE BOOL "Build CRC32C's unit tests")

adnl/adnl-ext-server.cpp

@@ -31,7 +31,7 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) {
       td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = f->query_id_](td::Result<td::BufferSlice> R) {
         if (R.is_error()) {
           auto S = R.move_as_error();
-          LOG(WARNING) << "failed ext query: " << S;
+          LOG(INFO) << "failed ext query: " << S;
         } else {
           auto B = create_tl_object<ton_api::adnl_message_answer>(query_id, R.move_as_ok());
           td::actor::send_closure(SelfId, &AdnlInboundConnection::send, serialize_tl_object(B, true));

adnl/adnl-peer.cpp

@@ -598,11 +598,11 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa
   respond_with_nop();
   auto size = message.total_size();
   if (size > huge_packet_max_size()) {
-    VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size;
+    VLOG(ADNL_INFO) << this << ": dropping too big huge message: size=" << size;
     return;
   }
   if (message.hash().is_zero()) {
-    VLOG(ADNL_WARNING) << this << ": dropping huge message with zero hash";
+    VLOG(ADNL_INFO) << this << ": dropping huge message with zero hash";
     return;
   }
   if (message.hash() != huge_message_hash_) {

assembly/native/build-macos-portable.sh

@@ -55,7 +55,7 @@ git clone https://github.com/lz4/lz4.git ../3pp/lz4
 cd ../3pp/lz4
 lz4Path=`pwd`
 git checkout v1.9.4
-make -j$(nproc)
+make -j4
 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
 cd ../../build
 # ./lib/liblz4.a
@@ -73,7 +73,7 @@ if [ ! -d "../3pp/libsodium" ]; then
   git checkout 1.0.18
   ./autogen.sh
   ./configure --with-pic --enable-static
-  make -j$(nproc)
+  make -j4
   test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; }
   cd ../../build
 else
@@ -87,7 +87,7 @@ if [ ! -d "../3pp/openssl_3" ]; then
   opensslPath=`pwd`
   git checkout openssl-3.1.4
   ./config
-  make build_libs -j$(nproc)
+  make build_libs -j4
   test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; }
   cd ../../build
 else
@@ -100,7 +100,7 @@ if [ ! -d "../3pp/zlib" ]; then
   cd ../3pp/zlib
   zlibPath=`pwd`
   ./configure --static
-  make -j$(nproc)
+  make -j4
   test $? -eq 0 || { echo "Can't compile zlib"; exit 1; }
   cd ../../build
 else
@@ -116,7 +116,7 @@ if [ ! -d "../3pp/libmicrohttpd" ]; then
   cd libmicrohttpd-1.0.1
   libmicrohttpdPath=`pwd`
   ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic
-  make -j$(nproc)
+  make -j4
   test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; }
   cd ../../../build
 else

assembly/native/build-macos-shared.sh

@@ -53,7 +53,7 @@ if [ ! -d "lz4" ]; then
   cd lz4
   lz4Path=`pwd`
   git checkout v1.9.4
-  make -j$(nproc)
+  make -j4
   test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
   cd ..
 else

crypto/test/test-db.cpp

@@ -2991,10 +2991,13 @@ TEST(TonDb, LargeBocSerializer) {
   td::unlink(path).ignore();
   fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write)
            .move_as_ok();
-  std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31);
+  boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31);
   fd.close();
   auto b = td::read_file_str(path).move_as_ok();
-  CHECK(a == b);
+
+  auto a_cell = vm::deserialize_boc(td::BufferSlice(a));
+  auto b_cell = vm::deserialize_boc(td::BufferSlice(b));
+  ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash());
 }
 
 TEST(TonDb, DoNotMakeListsPrunned) {

crypto/vm/boc.cpp

@@ -209,7 +209,7 @@ td::Result<int> BagOfCells::import_cell(td::Ref<vm::Cell> cell, int depth) {
     return td::Status::Error("error while importing a cell into a bag of cells: cell is null");
   }
   if (logger_ptr_) {
-    TRY_STATUS(logger_ptr_->on_cell_processed());
+    TRY_STATUS(logger_ptr_->on_cells_processed(1));
   }
   auto it = cells.find(cell->get_hash());
   if (it != cells.end()) {
@@ -555,7 +555,7 @@ td::Result<std::size_t> BagOfCells::serialize_to_impl(WriterT& writer, int mode)
       }
       store_offset(fixed_offset);
       if (logger_ptr_) {
-        TRY_STATUS(logger_ptr_->on_cell_processed());
+        TRY_STATUS(logger_ptr_->on_cells_processed(1));
       }
     }
     if (logger_ptr_) {
@@ -588,7 +588,7 @@ td::Result<std::size_t> BagOfCells::serialize_to_impl(WriterT& writer, int mode)
     }
     // std::cerr << std::endl;
     if (logger_ptr_) {
-      TRY_STATUS(logger_ptr_->on_cell_processed());
+      TRY_STATUS(logger_ptr_->on_cells_processed(1));
     }
   }
   writer.chk();

crypto/vm/boc.h

@@ -210,22 +210,27 @@ class BagOfCellsLogger {
 
   void start_stage(std::string stage) {
     log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD);
+    last_speed_log_ = td::Timestamp::now();
     processed_cells_ = 0;
     timer_ = {};
     stage_ = std::move(stage);
   }
   void finish_stage(td::Slice desc) {
     LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc;
   }
-  td::Status on_cell_processed() {
-    ++processed_cells_;
-    if (processed_cells_ % 1000 == 0) {
+  td::Status on_cells_processed(size_t count) {
+    processed_cells_ += count;
+    if (processed_cells_ / 1000 > last_token_check_) {
       TRY_STATUS(cancellation_token_.check());
+      last_token_check_ = processed_cells_ / 1000;
     }
     if (log_speed_at_.is_in_past()) {
-      log_speed_at_ += LOG_SPEED_PERIOD;
-      LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
+      double period = td::Timestamp::now().at() - last_speed_log_.at();
+
+      LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / period << " cells/s";
       processed_cells_ = 0;
+      last_speed_log_ = td::Timestamp::now();
+      log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD);
     }
     return td::Status::OK();
   }
@@ -236,6 +241,8 @@ class BagOfCellsLogger {
   td::CancellationToken cancellation_token_;
   td::Timestamp log_speed_at_;
   size_t processed_cells_ = 0;
+  size_t last_token_check_ = 0;
+  td::Timestamp last_speed_log_;
   static constexpr double LOG_SPEED_PERIOD = 120.0;
 };
 class BagOfCells {
@@ -390,7 +397,7 @@ td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root,
 
 td::Status std_boc_serialize_to_file(Ref<Cell> root, td::FileFd& fd, int mode = 0,
                                      td::CancellationToken cancellation_token = {});
-td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
+td::Status boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
                                            int mode = 0, td::CancellationToken cancellation_token = {});
 
 }  // namespace vm

crypto/vm/db/CellStorage.cpp

@@ -194,6 +194,28 @@ td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, bool need_da
   return res;
 }
 
+td::Result<std::vector<CellLoader::LoadResult>> CellLoader::load_bulk(td::Span<td::Slice> hashes, bool need_data, 
+                                                                ExtCellCreator &ext_cell_creator) {
+  std::vector<std::string> values;
+  TRY_RESULT(get_statuses, reader_->get_multi(hashes, &values));
+  std::vector<LoadResult> res;
+  res.reserve(hashes.size());
+  for (size_t i = 0; i < hashes.size(); i++) {
+    auto get_status = get_statuses[i];
+    if (get_status != KeyValue::GetStatus::Ok) {
+      DCHECK(get_status == KeyValue::GetStatus::NotFound);
+      res.push_back(LoadResult{});
+      continue;
+    }
+    TRY_RESULT(load_res, load(hashes[i], values[i], need_data, ext_cell_creator));
+    if (on_load_callback_) {
+      on_load_callback_(load_res);
+    }
+    res.push_back(std::move(load_res));
+  }
+  return res;
+}
+
 td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, td::Slice value, bool need_data,
                                                     ExtCellCreator &ext_cell_creator) {
   LoadResult res;