Feature/fs repository (#46)

Signed-off-by: Alexey-N-Chernyshov <[email protected]>

Author: Alexey-N-Chernyshov

core/storage/ipfs/CMakeLists.txt

@@ -2,10 +2,17 @@
 # Copyright Soramitsu Co., Ltd. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
+add_library(ipfs_datastore_in_memory
+    impl/in_memory_datastore.cpp
+    impl/ipfs_datastore_error.cpp
+    )
+target_link_libraries(ipfs_datastore_in_memory
+    p2p::p2p_cid
+    buffer
+    )
 
 add_library(ipfs_datastore_leveldb
     impl/datastore_leveldb.cpp
-    impl/in_memory_datastore.cpp
     impl/ipfs_datastore_error.cpp
     )
 target_link_libraries(ipfs_datastore_leveldb

core/storage/repository/CMakeLists.txt

@@ -8,3 +8,16 @@ add_library(repository
 target_link_libraries(repository
     outcome
     )
+
+add_library(filesystem_repository
+    impl/filesystem_repository.cpp
+    )
+target_link_libraries(filesystem_repository
+    Boost::filesystem
+    config
+    fslock
+    ipfs_datastore_leveldb
+    keystore
+    outcome
+    repository
+    )

core/storage/repository/impl/filesystem_repository.cpp

@@ -0,0 +1,110 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "storage/repository/impl/filesystem_repository.hpp"
+
+#include <utility>
+
+#include "boost/filesystem.hpp"
+#include "crypto/bls_provider/impl/bls_provider_impl.hpp"
+#include "libp2p/crypto/secp256k1_provider/secp256k1_provider_impl.hpp"
+#include "primitives/address/impl/address_verifier_impl.hpp"
+#include "storage/ipfs/impl/datastore_leveldb.hpp"
+#include "storage/keystore/impl/filesystem/filesystem_keystore.hpp"
+#include "storage/repository/repository_error.hpp"
+
+using fc::crypto::bls::impl::BlsProviderImpl;
+using fc::primitives::address::AddressVerifierImpl;
+using fc::storage::ipfs::LeveldbDatastore;
+using fc::storage::keystore::FileSystemKeyStore;
+using fc::storage::repository::FileSystemRepository;
+using fc::storage::repository::Repository;
+using fc::storage::repository::RepositoryError;
+using libp2p::crypto::secp256k1::Secp256k1ProviderImpl;
+using Version = fc::storage::repository::Repository::Version;
+
+FileSystemRepository::FileSystemRepository(
+    std::shared_ptr<IpfsDatastore> ipld_store,
+    std::shared_ptr<KeyStore> keystore,
+    std::shared_ptr<Config> config,
+    std::string repository_path,
+    std::unique_ptr<fslock::Locker> fs_locker)
+    : Repository{std::move(ipld_store), std::move(keystore), std::move(config)},
+      repository_path_{std::move(repository_path)},
+      fs_locker_{std::move(fs_locker)} {}
+
+fc::outcome::result<std::shared_ptr<Repository>> FileSystemRepository::create(
+    const Path &repo_path,
+    const std::string &api_address,
+    const leveldb::Options &leveldb_options) {
+  // check version if version file exists
+  auto version_filename =
+      repo_path + fc::storage::filestore::DELIMITER + kVersionFilename;
+  if (boost::filesystem::exists(version_filename)) {
+    boost::filesystem::ifstream version_is(version_filename);
+    std::string version_line;
+    std::getline(version_is, version_line);
+    // returns version number or 0 in case of failure
+    Version version = atoi(version_line.c_str());
+    if (version != kFileSystemRepositoryVersion)
+      return RepositoryError::WRONG_VERSION;
+  } else {
+    logger_->debug("Version file does not exist \"" + repo_path
+                   + fc::storage::filestore::DELIMITER + version_filename
+                   + "\". Will be created.");
+  }
+
+  // try lock
+  auto lock_filename =
+      repo_path + fc::storage::filestore::DELIMITER + kRepositoryLock;
+  std::unique_ptr<fslock::Locker> fs_locker =
+      std::make_unique<fslock::Locker>();
+  OUTCOME_TRY(fs_locker->lock(lock_filename));
+
+  // load config if exists
+  auto config_filename =
+      repo_path + fc::storage::filestore::DELIMITER + kConfigFilename;
+  auto config = std::make_shared<Config>();
+  if (boost::filesystem::exists(config_filename)) {
+    OUTCOME_TRY(config->load(config_filename));
+  }
+
+  // write api address
+  auto api_filename =
+      repo_path + fc::storage::filestore::DELIMITER + kApiFilename;
+  boost::filesystem::ofstream api_os(api_filename);
+  if (!api_os.is_open()) return RepositoryError::OPEN_FILE_ERROR;
+  api_os << api_address << std::endl;
+  api_os.close();
+
+  // write version
+  boost::filesystem::ofstream version_os(version_filename);
+  if (!version_os.is_open()) return RepositoryError::OPEN_FILE_ERROR;
+  version_os << kFileSystemRepositoryVersion << std::endl;
+  version_os.close();
+
+  // create datastore
+  auto datastore_path =
+      repo_path + fc::storage::filestore::DELIMITER + kDatastore;
+  OUTCOME_TRY(ipfs_datastore,
+              LeveldbDatastore::create(datastore_path, leveldb_options));
+
+  // create keystore
+  auto keystore_path =
+      repo_path + fc::storage::filestore::DELIMITER + kKeysDirectory;
+  boost::filesystem::create_directory(keystore_path);
+  auto keystore = std::make_shared<FileSystemKeyStore>(
+      keystore_path,
+      std::make_shared<BlsProviderImpl>(),
+      std::make_shared<Secp256k1ProviderImpl>(),
+      std::make_shared<AddressVerifierImpl>());
+
+  return std::make_shared<FileSystemRepository>(
+      ipfs_datastore, keystore, config, repo_path, std::move(fs_locker));
+}
+
+fc::outcome::result<Version> FileSystemRepository::getVersion() const {
+  return kFileSystemRepositoryVersion;
+}

core/storage/repository/impl/filesystem_repository.hpp

@@ -0,0 +1,73 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef FILECOIN_CORE_STORAGE_IMPL_FILESYSTEM_REPOSITORY_HPP
+#define FILECOIN_CORE_STORAGE_IMPL_FILESYSTEM_REPOSITORY_HPP
+
+#include <iostream>
+
+#include "fslock/fslock.hpp"
+#include "storage/filestore/path.hpp"
+#include "storage/leveldb/leveldb.hpp"
+#include "storage/repository/repository.hpp"
+
+namespace fc::storage::repository {
+
+  using filestore::Path;
+  using Version = Repository::Version;
+
+  /**
+   * @brief FileSystem implementation of Repository.
+   *
+   *  Directory structure of Filesystem Repository is:
+   * .ipfs/
+   * ├── api             <--- running daemon api addr
+   * ├── blocks/         <--- objects stored directly on disk
+   * │   └── aa          <--- prefix namespacing like git
+   * │       └── aa      <--- N tiers
+   * ├── config          <--- config file (json or toml)
+   * ├── hooks/          <--- hook scripts (not implemented yet)
+   * ├── keys/           <--- cryptographic keys
+   * │   ├── id.pri      <--- identity private key
+   * │   └── id.pub      <--- identity public key
+   * ├── datastore/      <--- datastore
+   * ├── logs/           <--- 1 or more files (log rotate)
+   * │   └── events.log  <--- can be tailed
+   * ├── repo.lock       <--- mutex for repo
+   * └── version         <--- version file
+   *
+   */
+  class FileSystemRepository : public Repository {
+   public:
+    inline static const std::string kApiFilename = "api";
+    inline static const std::string kConfigFilename = "config.json";
+    inline static const std::string kKeysDirectory = "keys";
+    inline static const std::string kDatastore = "datastore";
+    inline static const std::string kRepositoryLock = "repo.lock";
+    inline static const std::string kVersionFilename = "version";
+    inline static const Version kFileSystemRepositoryVersion = 1;
+
+    FileSystemRepository(std::shared_ptr<IpfsDatastore> ipld_store,
+                         std::shared_ptr<KeyStore> keystore,
+                         std::shared_ptr<Config> config,
+                         std::string repository_path,
+                         std::unique_ptr<fslock::Locker> fs_locker);
+
+    static outcome::result<std::shared_ptr<Repository>> create(
+        const Path &repo_path,
+        const std::string &api_address,
+        const leveldb::Options &leveldb_options);
+
+    outcome::result<Version> getVersion() const override;
+
+   private:
+    Path repository_path_;
+    std::unique_ptr<fslock::Locker> fs_locker_;
+    inline static common::Logger logger_ = common::createLogger("repository");
+  };
+
+}  // namespace fc::storage::repository
+
+#endif  // FILECOIN_CORE_STORAGE_IMPL_FILESYSTEM_REPOSITORY_HPP

core/storage/repository/impl/repository_error.cpp

@@ -11,6 +11,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::repository, RepositoryError, e) {
   switch (e) {
     case RepositoryError::WRONG_VERSION:
       return "RepositoryError: wrong version";
+    case RepositoryError::OPEN_FILE_ERROR:
+      return "RepositoryError: cannot open file";
     case RepositoryError::UNKNOWN:
       break;
   }

core/storage/repository/repository_error.hpp

@@ -15,6 +15,7 @@ namespace fc::storage::repository {
    */
   enum class RepositoryError {
     WRONG_VERSION = 1,
+    OPEN_FILE_ERROR,
 
     UNKNOWN
   };

test/core/fslock/fslock_test.cpp

@@ -18,7 +18,7 @@ using fc::storage::filestore::Path;
 
 class FSLockTest : public test::BaseFS_Test {
  public:
-  FSLockTest() : test::BaseFS_Test("/tmp/fc_filesystem_lock_file_test/") {}
+  FSLockTest() : test::BaseFS_Test("fc_filesystem_lock_file_test") {}
 
   /** Path to lock file */
   Path lock_file_path;

test/core/storage/CMakeLists.txt

@@ -7,3 +7,4 @@ add_subdirectory(hamt)
 add_subdirectory(keystore)
 add_subdirectory(ipfs)
 add_subdirectory(leveldb)
+add_subdirectory(repository)

test/core/storage/config/config_test.cpp

@@ -14,7 +14,7 @@ using fc::storage::config::ConfigError;
 
 class ConfigImplTest : public test::BaseFS_Test {
  public:
-  ConfigImplTest() : test::BaseFS_Test("/tmp/fc_config_test/") {}
+  ConfigImplTest() : test::BaseFS_Test("fc_config_test") {}
 
   /**
    * Create a test directory with an empty file.

test/core/storage/filestore/filesystem/filesystem_file_test.cpp

@@ -18,7 +18,7 @@ using fc::storage::filestore::Path;
 
 class FileSystemFileTest : public test::BaseFS_Test {
  public:
-  FileSystemFileTest() : test::BaseFS_Test("/tmp/fc_filesystem_file_test/") {}
+  FileSystemFileTest() : test::BaseFS_Test("fc_filesystem_file_test") {}
 
   /** Path to empty file */
   Path empty_file_path;