Hamt bit width (#99)

Signed-off-by: turuslan <[email protected]>

Author: turuslan

core/storage/hamt/hamt.cpp

@@ -24,26 +24,24 @@ OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::hamt, HamtError, e) {
 namespace fc::storage::hamt {
   using fc::common::which;
 
-  // assuming 8-bit indices
-  auto keyToIndices(const std::string &key, int n = -1) {
-    std::vector<uint8_t> key_bytes(key.begin(), key.end());
-    auto hash = crypto::murmur::hash(key_bytes);
-    return std::vector<size_t>(n == -1 ? hash.begin() : hash.end() - n + 1,
-                               hash.end());
-  }
-
   auto consumeIndex(gsl::span<const size_t> indices) {
     return indices.subspan(1);
   }
 
-  Hamt::Hamt(std::shared_ptr<ipfs::IpfsDatastore> store)
-      : store_(std::move(store)), root_(std::make_shared<Node>()) {}
+  Hamt::Hamt(std::shared_ptr<ipfs::IpfsDatastore> store, size_t bit_width)
+      : store_{std::move(store)},
+        root_{std::make_shared<Node>()},
+        bit_width_{bit_width} {}
 
   Hamt::Hamt(std::shared_ptr<ipfs::IpfsDatastore> store, Node::Ptr root)
-      : store_(std::move(store)), root_(std::move(root)) {}
+      : store_{std::move(store)},
+        root_{std::move(root)},
+        bit_width_{kDefaultBitWidth} {}
 
-  Hamt::Hamt(std::shared_ptr<ipfs::IpfsDatastore> store, const CID &root)
-      : store_(std::move(store)), root_(root) {}
+  Hamt::Hamt(std::shared_ptr<ipfs::IpfsDatastore> store,
+             const CID &root,
+             size_t bit_width)
+      : store_{std::move(store)}, root_{root}, bit_width_{bit_width} {}
 
   outcome::result<void> Hamt::set(const std::string &key,
                                   gsl::span<const uint8_t> value) {
@@ -84,6 +82,30 @@ namespace fc::storage::hamt {
     return boost::get<CID>(root_);
   }
 
+  std::vector<size_t> Hamt::keyToIndices(const std::string &key, int n) const {
+    std::vector<uint8_t> key_bytes(key.begin(), key.end());
+    auto hash = crypto::murmur::hash(key_bytes);
+    std::vector<size_t> indices;
+    constexpr auto byte_bits = 8;
+    auto max_bits = byte_bits * hash.size();
+    max_bits -= max_bits % bit_width_;
+    auto offset = 0;
+    if (n != -1) {
+      offset = max_bits - (n - 1) * bit_width_;
+    }
+    while (offset + bit_width_ <= max_bits) {
+      size_t index = 0;
+      for (auto i = 0u; i < bit_width_; ++i, ++offset) {
+        index <<= 1;
+        index |= 1
+                 & (hash[offset / byte_bits]
+                    >> (byte_bits - 1 - offset % byte_bits));
+      }
+      indices.push_back(index);
+    }
+    return indices;
+  }
+
   outcome::result<void> Hamt::set(Node &node,
                                   gsl::span<const size_t> indices,
                                   const std::string &key,

core/storage/hamt/hamt.hpp

@@ -25,6 +25,7 @@ namespace fc::storage::hamt {
   using Value = ipfs::IpfsDatastore::Value;
 
   constexpr size_t kLeafMax = 3;
+  constexpr size_t kDefaultBitWidth = 8;
 
   /** Hamt node representation */
   struct Node {
@@ -107,9 +108,12 @@ namespace fc::storage::hamt {
     using Visitor = std::function<outcome::result<void>(const std::string &,
                                                         const Value &)>;
 
-    explicit Hamt(std::shared_ptr<ipfs::IpfsDatastore> store);
+    Hamt(std::shared_ptr<ipfs::IpfsDatastore> store,
+         size_t bit_width = kDefaultBitWidth);
     Hamt(std::shared_ptr<ipfs::IpfsDatastore> store, Node::Ptr root);
-    Hamt(std::shared_ptr<ipfs::IpfsDatastore> store, const CID &root);
+    Hamt(std::shared_ptr<ipfs::IpfsDatastore> store,
+         const CID &root,
+         size_t bit_width = kDefaultBitWidth);
     /** Set value by key, does not write to storage */
     outcome::result<void> set(const std::string &key,
                               gsl::span<const uint8_t> value);
@@ -143,6 +147,7 @@ namespace fc::storage::hamt {
     }
 
    private:
+    std::vector<size_t> keyToIndices(const std::string &key, int n = -1) const;
     outcome::result<void> set(Node &node,
                               gsl::span<const size_t> indices,
                               const std::string &key,
@@ -157,6 +162,7 @@ namespace fc::storage::hamt {
 
     std::shared_ptr<ipfs::IpfsDatastore> store_;
     Node::Item root_;
+    size_t bit_width_;
   };
 }  // namespace fc::storage::hamt
 

test/core/storage/hamt/hamt_test.cpp

@@ -199,6 +199,20 @@ TEST_F(HamtTest, FlushCollisionChild) {
   EXPECT_OUTCOME_EQ(store_->contains(cidShard), true);
 }
 
+/** Go cid compatibility with bit width of 5 */
+TEST_F(HamtTest, CollisionChildBitWidth5) {
+  hamt_ = {store_, 5};
+  EXPECT_OUTCOME_EQ(hamt_.flush(), "0171a0e4022018fe6acc61a3a36b0c373c4a3a8ea64b812bf2ca9b528050909c78d408558a0c"_cid);
+  EXPECT_OUTCOME_TRUE_1(hamt_.set("ails", "01"_unhex));
+  EXPECT_OUTCOME_EQ(hamt_.flush(), "0171a0e40220963f12f112adbaa0e56ad77152a259224ac41eed6e5d909eaeb714f65add029f"_cid);
+  EXPECT_OUTCOME_TRUE_1(hamt_.set("aufx", "02"_unhex));
+  EXPECT_OUTCOME_EQ(hamt_.flush(), "0171a0e402207df36ed88d694b1bfe7161dd598bdfac05426a9beab0bf828d3e838d2c198e16"_cid);
+  EXPECT_OUTCOME_TRUE_1(hamt_.set("bmvm", "03"_unhex));
+  EXPECT_OUTCOME_EQ(hamt_.flush(), "0171a0e40220823751c22261961ca5a97917c9ce6c681e44a8265873d66ae2dadf3908c12357"_cid);
+  EXPECT_OUTCOME_TRUE_1(hamt_.set("cnyh", "04"_unhex));
+  EXPECT_OUTCOME_EQ(hamt_.flush(), "0171a0e40220f77809a02565e12cbadbab9ab8713958c5bb66b061508bbb01b37d73923f8c2b"_cid);
+}
+
 /** Visit all key value pairs */
 TEST_F(HamtTest, Visitor) {
   auto n = 0;