fil-422 unsorted hamt (#480)

Signed-off-by: turuslan <[email protected]>

Author: turuslan

core/storage/hamt/hamt.cpp

@@ -26,6 +26,20 @@ OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::hamt, HamtError, e) {
 namespace fc::storage::hamt {
   using common::which;
 
+  inline auto leafFind(Node::Leaf &leaf, BytesIn key) {
+    return std::find_if(
+        leaf.begin(), leaf.end(), [&](auto &p) { return p.first == key; });
+  }
+
+  inline void leafInsert(Node::Leaf &leaf, Node::Leaf::value_type pair) {
+    leaf.emplace(
+        std::lower_bound(leaf.begin(),
+                         leaf.end(),
+                         pair,
+                         [&](auto &l, auto &r) { return l.first < r.first; }),
+        std::move(pair));
+  }
+
   CBOR2_ENCODE(Node) {
     Bits bits;
     auto l_items{s.list()};
@@ -93,7 +107,7 @@ namespace fc::storage::hamt {
         for (size_t j = 0; j < n_leaf; ++j) {
           auto l_pair{l_leaf.list()};
           auto key{l_pair.get<Bytes>()};
-          leaf.emplace(std::move(key), l_pair.raw());
+          leaf.emplace_back(std::move(key), l_pair.raw());
         }
         v.items[j] = std::move(leaf);
       }
@@ -140,7 +154,7 @@ namespace fc::storage::hamt {
         node = boost::get<Node::Ptr>(item);
       } else {
         auto &leaf = boost::get<Node::Leaf>(item);
-        auto it{leaf.find(key)};
+        auto it{leafFind(leaf, key)};
         if (it == leaf.end()) {
           return HamtError::kNotFound;
         }
@@ -219,10 +233,10 @@ namespace fc::storage::hamt {
           *boost::get<Node::Ptr>(item), consumeIndex(indices), key, value);
     }
     auto &leaf = boost::get<Node::Leaf>(item);
-    if (auto it2{leaf.find(key)}; it2 != leaf.end()) {
+    if (auto it2{leafFind(leaf, key)}; it2 != leaf.end()) {
       copy(it2->second, value);
     } else if (leaf.size() < kLeafMax) {
-      leaf.emplace(copy(key), copy(value));
+      leafInsert(leaf, {copy(key), copy(value)});
     } else {
       auto child = std::make_shared<Node>();
       child->v3 = v3();
@@ -255,13 +269,14 @@ namespace fc::storage::hamt {
       OUTCOME_TRY(cleanShard(item));
     } else {
       auto &leaf = boost::get<Node::Leaf>(item);
-      if (leaf.find(key) == leaf.end()) {
+      auto it2{leafFind(leaf, key)};
+      if (it2 == leaf.end()) {
         return HamtError::kNotFound;
       }
       if (leaf.size() == 1) {
         node.items.erase(index);
       } else {
-        leaf.erase(leaf.find(key));
+        leaf.erase(it2);
       }
     }
     return outcome::success();
@@ -281,7 +296,7 @@ namespace fc::storage::hamt {
           return outcome::success();
         }
         for (auto &pair : boost::get<Node::Leaf>(item2.second)) {
-          leaf.emplace(pair);
+          leafInsert(leaf, pair);
           if (leaf.size() > kLeafMax) {
             return outcome::success();
           }

core/storage/hamt/hamt.hpp

@@ -57,7 +57,7 @@ namespace fc::storage::hamt {
   /** Hamt node representation */
   struct Node {
     using Ptr = std::shared_ptr<Node>;
-    using Leaf = std::map<Bytes, Bytes, BytesLess>;
+    using Leaf = std::vector<std::pair<Bytes, Bytes>>;
     using Item = boost::variant<CID, Ptr, Leaf>;
 
     std::map<size_t, Item> items;

test/core/test_vectors/test.cpp

@@ -245,30 +245,6 @@ auto search() {
     const auto &path{item.path()};
     if (item.status().type() == boost::filesystem::file_type::regular_file
         && path.extension() == ".json") {
-      // Skip tests that fail in Fuhon
-      static std::vector<boost::filesystem::path> fail_in_fuhon{
-          // TODO (m.tagirov) FIL-422 test-vectors hamt have incorrect order
-          // Fuhon implementation loads and reorders hamt
-          // while Lotus implementation uses lazy approach and keeps initial incorrect order
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storagepower/CreateMiner/Ok/ext-0004-fil_1_storagepower-CreateMiner-Ok-6.json",
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storagepower/CreateMiner/Ok/ext-0004-fil_1_storagepower-CreateMiner-Ok-10.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storagepower/CreateMiner/Ok/ext-0001-fil_1_storagepower-CreateMiner-Ok-6.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storageminer/PreCommitSector/Ok/ext-0001-fil_1_storageminer-PreCommitSector-Ok-1.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storageminer/PreCommitSector/Ok/ext-0001-fil_1_storageminer-PreCommitSector-Ok-5.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storageminer/PreCommitSector/Ok/ext-0001-fil_1_storageminer-PreCommitSector-Ok-6.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storageminer/PreCommitSector/Ok/ext-0001-fil_1_storageminer-PreCommitSector-Ok-8.json",
-          kCorpusRoot / "extracted/0001-initial-extraction/fil_1_storageminer/PreCommitSector/Ok/ext-0001-fil_1_storageminer-PreCommitSector-Ok-9.json",
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storageminer/PreCommitSector/Ok/ext-0004-fil_1_storageminer-PreCommitSector-Ok-4.json",
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storageminer/PreCommitSector/Ok/ext-0004-fil_1_storageminer-PreCommitSector-Ok-5.json",
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storageminer/PreCommitSector/Ok/ext-0004-fil_1_storageminer-PreCommitSector-Ok-7.json",
-          kCorpusRoot / "extracted/0004-coverage-boost/fil_1_storageminer/PreCommitSector/Ok/ext-0004-fil_1_storageminer-PreCommitSector-Ok-9.json",
-      };
-
-      if (std::find(fail_in_fuhon.cbegin(), fail_in_fuhon.cend(), path)
-          != fail_in_fuhon.cend()) {
-        continue;
-      }
-
       // ignore broken/incorrect vectors that starts with "x--"
       if (boost::algorithm::starts_with(path.filename().string(), "x--")) {
         continue;