Replaced hashmap by b-tree map in large boc serializer (#1686)

Author: Sumrachek

crypto/test/test-db.cpp

@@ -1691,6 +1691,31 @@ TEST(TonDb, BocDeserializerSimpleThreads) {
   test_boc_deserializer_threads<StaticBagOfCellsDbLazy>();
 }
 
+class RandomTree {
+  public:
+    RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) {
+      root_ = create(size);
+    }
+    Ref<Cell> root() const {
+      return root_;
+    }
+  private:
+    Ref<Cell> root_;
+    td::Random::Xorshift128plus rnd_;
+    Ref<DataCell> create(size_t size) {
+      CellBuilder cb;
+      cb.store_long(rnd_(), rnd_() % 63 + 1);
+      if (size > 0) {
+        td::uint64 rc = (rnd_() % 4) + 1;
+        for (td::uint64 i = 0; i < rc; i++) {
+            auto ref = create(size / rc);
+            cb.store_ref(std::move(ref));
+        }
+      }
+      return cb.finalize();
+    }
+};
+
 class CompactArray {
  public:
   CompactArray(size_t size) {
@@ -2987,11 +3012,8 @@ TEST(TonDb, DynamicBocRespectsUsageCell) {
 
 TEST(TonDb, LargeBocSerializer) {
   td::Random::Xorshift128plus rnd{123};
-  size_t n = 1000000;
-  std::vector<td::uint64> data(n);
-  std::iota(data.begin(), data.end(), 0);
-  vm::CompactArray arr(data);
-  auto root = arr.root();
+  vm::RandomTree tree(100000, rnd);
+  auto root = tree.root();
   std::string path = "serialization";
   td::unlink(path).ignore();
   auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write)
@@ -3009,15 +3031,20 @@ TEST(TonDb, LargeBocSerializer) {
   dboc->commit(cell_storer);
   dboc->set_loader(std::make_unique<vm::CellLoader>(kv));
   td::unlink(path).ignore();
-  fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write)
-           .move_as_ok();
-  boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31);
-  fd.close();
-  auto b = td::read_file_str(path).move_as_ok();
-
+  td::string prev_b("");
   auto a_cell = vm::deserialize_boc(td::BufferSlice(a));
-  auto b_cell = vm::deserialize_boc(td::BufferSlice(b));
-  ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash());
+  for (int i = 0; i < 4; i++) {
+    fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write)
+            .move_as_ok();
+    boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31);
+    fd.close();
+    auto b = td::read_file_str(path).move_as_ok();
+
+    auto b_cell = vm::deserialize_boc(td::BufferSlice(b));
+    ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash());
+    if (i > 0) ASSERT_EQ(prev_b, b);
+    prev_b = b;
+  }
 }
 
 TEST(TonDb, DoNotMakeListsPrunned) {

crypto/vm/large-boc-serializer.cpp

@@ -115,7 +115,7 @@ td::Status LargeBocSerializer::import_cells() {
 
 td::Result<int> LargeBocSerializer::import_cell(Hash root_hash, int root_depth) {
   const int start_ind = cell_count;
-  td::HashMap<Hash, std::pair<int, bool>> current_depth_hashes;
+  td::BTreeMap<Hash, std::pair<int, bool>> current_depth_hashes;
 
   auto existing_it = cells.find(root_hash);
   if (existing_it != cells.end()) {
@@ -131,7 +131,7 @@ td::Result<int> LargeBocSerializer::import_cell(Hash root_hash, int root_depth)
     }
 
     cell_list.resize(cell_list.size() + current_depth_hashes.size());
-    td::HashMap<Hash, std::pair<int, bool>> next_depth_hashes;
+    td::BTreeMap<Hash, std::pair<int, bool>> next_depth_hashes;
     auto batch_start = current_depth_hashes.begin();
     while (batch_start != current_depth_hashes.end()) {
       std::vector<td::Slice> batch_hashes;

tdutils/td/utils/HashMap.h

@@ -23,8 +23,10 @@
 #if TD_HAVE_ABSL
 #include <absl/container/flat_hash_map.h>
 #include <absl/container/node_hash_map.h>
+#include <absl/container/btree_map.h>
 #else
 #include <unordered_map>
+#include <map>
 #endif
 
 namespace td {
@@ -34,11 +36,15 @@ template <class Key, class Value, class H = Hash<Key>>
 using HashMap = absl::flat_hash_map<Key, Value, H>;
 template <class Key, class Value, class H = Hash<Key>, class E = std::equal_to<>>
 using NodeHashMap = absl::node_hash_map<Key, Value, H, E>;
+template <class Key, class Value>
+using BTreeMap = absl::btree_map<Key, Value>;
 #else
 template <class Key, class Value, class H = Hash<Key>>
 using HashMap = std::unordered_map<Key, Value, H>;
 template <class Key, class Value, class H = Hash<Key>>
 using NodeHashMap = std::unordered_map<Key, Value, H>;
+template <class Key, class Value>
+using BTreeMap = std::map<Key, Value>;
 #endif
 
 }  // namespace td