IPFS MerkleDAG (#77)

Signed-off-by: Sergey Kaprovich <[email protected]>

Author: sergkaprovich

.github/workflows/ci.yml

@@ -46,11 +46,12 @@ jobs:
           brew install ninja
         else
           sudo apt-get update || true
-          sudo apt-get install -y ninja-build
+          sudo apt-get install -y ninja-build python-setuptools
         fi
 
         pip3 -V || sudo python3 -m pip install --upgrade pip
-        sudo pip3 install scikit-build cmake requests gitpython gcovr pyyaml
+        sudo pip3 install scikit-build
+        sudo pip3 install cmake requests gitpython gcovr pyyaml
         sudo curl https://sh.rustup.rs -sSf | sh -s -- -y
     - name: cmake
       env:

cmake/Hunter/config.cmake

@@ -30,7 +30,7 @@ hunter_config(
 )
 
 hunter_config(libp2p
-    URL https://github.com/soramitsu/libp2p/archive/cdc092fe74ef5d8eeedf42e3e949488ff5052cb4.zip
-    SHA1 16605084e4f018ebdb09018e906a54aa1909c8e1
+    URL https://github.com/soramitsu/libp2p/archive/06a70dfcdc2649264f44c460a542e0c774db2290.zip
+    SHA1 c6bbd0e55659fb0a7a84091a97d219f59ae37d14
     CMAKE_ARGS TESTING=OFF
     )

cmake/dependencies.cmake

@@ -45,3 +45,7 @@ find_package(libp2p CONFIG REQUIRED)
 # https://docs.hunter.sh/en/latest/packages/pkg/cppcodec.html
 hunter_add_package(cppcodec)
 find_package(cppcodec CONFIG REQUIRED)
+
+# https://hunter.readthedocs.io/en/stable/packages/pkg/Protobuf.html
+hunter_add_package(Protobuf)
+find_package(Protobuf CONFIG REQUIRED)

core/storage/ipfs/CMakeLists.txt

@@ -29,3 +29,5 @@ add_library(ipfs_blockservice
 target_link_libraries(ipfs_blockservice
     buffer
     )
+
+add_subdirectory(merkledag)

core/storage/ipfs/block.hpp

@@ -6,7 +6,10 @@
 #ifndef FILECOIN_STORAGE_IPFS_BLOCK_HPP
 #define FILECOIN_STORAGE_IPFS_BLOCK_HPP
 
+#include <functional>
+
 #include "common/buffer.hpp"
+#include "common/outcome.hpp"
 #include "primitives/cid/cid.hpp"
 
 namespace fc::storage::ipfs {
@@ -31,7 +34,7 @@ namespace fc::storage::ipfs {
      * @brief Get block content
      * @return Block's raw data for store in the BlockService
      */
-    virtual const Content &getContent() const = 0;
+    virtual const Content &getRawBytes() const = 0;
   };
 }  // namespace fc::storage::ipfs
 

core/storage/ipfs/impl/blockservice_impl.cpp

@@ -15,7 +15,7 @@ namespace fc::storage::ipfs {
   }
 
   outcome::result<void> BlockServiceImpl::addBlock(const Block &block) {
-    auto result = local_storage_->set(block.getCID(), block.getContent());
+    auto result = local_storage_->set(block.getCID(), block.getRawBytes());
     if (result.has_error()) {
       return BlockServiceError::ADD_BLOCK_FAILED;
     }

core/storage/ipfs/merkledag/CMakeLists.txt

@@ -0,0 +1,34 @@
+find_package(Protobuf REQUIRED)
+set(PB_SCHEME "merkledag.proto")
+set(PB_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --cpp_out=${PB_BUILD_DIR} ${PB_SCHEME}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/impl/protobuf
+    RESULT_VARIABLE CMD_OUTPUT
+    )
+
+add_library(ipfs_merkledag_service_protobuf
+    ${PB_BUILD_DIR}/merkledag.pb.h
+    ${PB_BUILD_DIR}/merkledag.pb.cc
+    )
+target_include_directories(ipfs_merkledag_service_protobuf PUBLIC ${PB_BUILD_DIR})
+target_link_libraries(ipfs_merkledag_service_protobuf
+    protobuf::libprotobuf
+    )
+disable_clang_tidy(ipfs_merkledag_service_protobuf)
+
+add_library(ipfs_merkledag_service
+    impl/link_impl.cpp
+    impl/node_impl.cpp
+    impl/merkledag_service_impl.cpp
+    impl/pb_node_encoder.cpp
+    impl/pb_node_decoder.cpp
+    impl/leaf_impl.cpp
+    )
+target_link_libraries(ipfs_merkledag_service
+    cid
+    Boost::boost
+    ipfs_blockservice
+    ipfs_datastore_in_memory
+    ipfs_merkledag_service_protobuf
+    )

core/storage/ipfs/merkledag/impl/leaf_impl.cpp

@@ -0,0 +1,55 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "storage/ipfs/merkledag/impl/leaf_impl.hpp"
+
+namespace fc::storage::ipfs::merkledag {
+  LeafImpl::LeafImpl(common::Buffer data)
+      : content_{std::move(data)} {}
+
+  const common::Buffer &LeafImpl::content() const {
+    return content_;
+  }
+
+  size_t LeafImpl::count() const {
+    return children_.size();
+  }
+
+  outcome::result<std::reference_wrapper<const Leaf>> LeafImpl::subLeaf(
+      std::string_view name) const {
+    if (auto iter = children_.find(name); iter != children_.end()) {
+      return iter->second;
+    }
+    return LeafError::LEAF_NOT_FOUND;
+  }
+
+  std::vector<std::string_view> LeafImpl::getSubLeafNames() const {
+    std::vector<std::string_view> names;
+    for (const auto &child : children_) {
+      names.push_back(child.first);
+    }
+    return names;
+  }
+
+  outcome::result<void> LeafImpl::insertSubLeaf(std::string name,
+                                                 LeafImpl children) {
+    auto result = children_.emplace(std::move(name), std::move(children));
+    if (result.second) {
+      return outcome::success();
+    }
+    return LeafError::DUPLICATE_LEAF;
+  }
+}  // namespace fc::storage::ipfs::merkledag
+
+OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::ipfs::merkledag, LeafError, e) {
+  using fc::storage::ipfs::merkledag::LeafError;
+  switch (e) {
+    case (LeafError::LEAF_NOT_FOUND):
+      return "MerkleDAG leaf: children leaf not found";
+    case (LeafError::DUPLICATE_LEAF):
+      return "MerkleDAG leaf: duplicate leaf name";
+  }
+  return "MerkleDAG leaf: unknown error";
+}

core/storage/ipfs/merkledag/impl/leaf_impl.hpp

@@ -0,0 +1,46 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_LEAF_IMPL_HPP
+#define FILECOIN_STORAGE_IPFS_MERKLEDAG_LEAF_IMPL_HPP
+
+#include "storage/ipfs/merkledag/leaf.hpp"
+
+#include <map>
+#include <string>
+
+namespace fc::storage::ipfs::merkledag {
+  class LeafImpl : public Leaf {
+   public:
+    /**
+     * @brief Construct leaf
+     * @param data - leaf content
+     */
+    explicit LeafImpl(common::Buffer data);
+
+    const common::Buffer &content() const override;
+
+    size_t count() const override;
+
+    outcome::result<std::reference_wrapper<const Leaf>> subLeaf(
+        std::string_view name) const override;
+
+    std::vector<std::string_view> getSubLeafNames() const override;
+
+    /**
+     * @brief Insert children leaf
+     * @param name - id of the leaf
+     * @param children - leaf to insert
+     * @return operation result
+     */
+    outcome::result<void> insertSubLeaf(std::string name, LeafImpl children);
+
+   private:
+    common::Buffer content_;
+    std::map<std::string, LeafImpl, std::less<>> children_;
+  };
+}  // namespace fc::storage::ipfs::merkledag
+
+#endif

core/storage/ipfs/merkledag/impl/link_impl.cpp

@@ -0,0 +1,27 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "storage/ipfs/merkledag/impl/link_impl.hpp"
+
+namespace fc::storage::ipfs::merkledag {
+  LinkImpl::LinkImpl(libp2p::multi::ContentIdentifier id,
+                     std::string name,
+                     size_t size)
+      : cid_{std::move(id)},
+        name_{std::move(name)},
+        size_{size} {}
+
+  const std::string &LinkImpl::getName() const {
+    return name_;
+  }
+
+  const CID &LinkImpl::getCID() const {
+    return cid_;
+  }
+
+  size_t LinkImpl::getSize() const {
+    return size_;
+  }
+}  // namespace fc::storage::ipfs::merkledag