Miner deal fixes (#541)

* Fix StorageAsk

Signed-off-by: ortyomka <[email protected]>

* Fixes miner deals

Signed-off-by: ortyomka <[email protected]>

Author: ortyomka

cmake/Hunter/config.cmake

@@ -24,7 +24,7 @@ hunter_config(libp2p
     )
 
 hunter_config(jwt-cpp
-    URL https://github.com/Thalhammer/jwt-cpp/archive/ac0424b115721e4066d2fb99f72ba0cd58759882.tar.gz
+    URL https://github.com/soramitsu/fuhon-jwt-cpp/archive/ac0424b115721e4066d2fb99f72ba0cd58759882.tar.gz
     SHA1 92667cf319f7009234f98f27884db571c2eb1171
     CMAKE_ARGS JWT_BUILD_EXAMPLES=OFF
     )

core/api/rpc/json.hpp

@@ -555,6 +555,40 @@ namespace fc::api {
       v = BigInt{AsString(j)};
     }
 
+    ENCODE(MinerDeal) {
+      Value j{rapidjson::kObjectType};
+      Set(j, "Proposal", v.client_deal_proposal.proposal);
+      Set(j, "ClientSignature", v.client_deal_proposal.client_signature);
+      Set(j, "ProposalCid", v.proposal_cid);
+      Set(j, "AddFundsCid", v.add_funds_cid);
+      Set(j, "PublishCid", v.publish_cid);
+      Set(j, "Client", v.client);
+      Set(j, "State", v.state);
+      Set(j, "PiecePath", v.piece_path);
+      Set(j, "MetadataPath", v.metadata_path);
+      Set(j, "FastRetrieval", v.is_fast_retrieval);
+      Set(j, "Message", v.message);
+      Set(j, "Ref", v.ref);
+      Set(j, "DealId", v.deal_id);
+      return j;
+    }
+
+    DECODE(MinerDeal) {
+      Get(j, "Proposal", v.client_deal_proposal.proposal);
+      Get(j, "ClientSignature", v.client_deal_proposal.client_signature);
+      Get(j, "ProposalCid", v.proposal_cid);
+      Get(j, "AddFundsCid", v.add_funds_cid);
+      Get(j, "PublishCid", v.publish_cid);
+      Get(j, "Client", v.client);
+      Get(j, "State", v.state);
+      Get(j, "PiecePath", v.piece_path);
+      Get(j, "MetadataPath", v.metadata_path);
+      Get(j, "FastRetrieval", v.is_fast_retrieval);
+      Get(j, "Message", v.message);
+      Get(j, "Ref", v.ref);
+      Get(j, "DealId", v.deal_id);
+    }
+
     ENCODE(MinerInfo) {
       Value j{rapidjson::kObjectType};
       Set(j, "Owner", v.owner);

core/api/storage_miner/storage_api.cpp

@@ -70,6 +70,11 @@ namespace fc::api {
       return outcome::success();
     };
 
+    api->MarketListIncompleteDeals =
+        [=]() -> outcome::result<std::vector<MinerDeal>> {
+      return storage_market_provider->getLocalDeals();
+    };
+
     api->SectorsList = [=]() -> outcome::result<std::vector<SectorNumber>> {
       std::vector<SectorNumber> result;
       for (const auto &sector : miner->getSealing()->getListSectors()) {

core/api/storage_miner/storage_api.hpp

@@ -25,12 +25,14 @@ namespace fc::api {
   using boost::asio::io_context;
   using markets::retrieval::RetrievalAsk;
   using markets::retrieval::provider::RetrievalProvider;
+  using markets::storage::MinerDeal;
   using markets::storage::SignedStorageAsk;
   using markets::storage::provider::StorageProvider;
   using markets::storage::provider::StoredAsk;
   using miner::Miner;
   using mining::types::DealInfo;
   using mining::types::DealSchedule;
+  using mining::types::PieceLocation;
   using primitives::ChainEpoch;
   using primitives::DealId;
   using primitives::SectorNumber;
@@ -55,25 +57,14 @@ namespace fc::api {
   using sector_storage::stores::SectorIndex;
   using StorageInfo_ = sector_storage::stores::StorageInfo;
 
-  const static common::Logger kStorageApiLogger = common::createLogger("Storage API");
-
-  struct PieceLocation {
-    SectorNumber sector_number;
-    PaddedPieceSize offset;
-    PaddedPieceSize length;
-  };
-  CBOR_TUPLE(PieceLocation, sector_number, offset, length)
+  const static common::Logger kStorageApiLogger =
+      common::createLogger("Storage API");
 
   // TODO(ortyomka): [FIL-421] implement it
   struct ApiSectorInfo {
     mining::SealingState state = mining::SealingState::kStateUnknown;
   };
 
-  inline bool operator==(const PieceLocation &lhs, const PieceLocation &rhs) {
-    return lhs.sector_number == rhs.sector_number && lhs.offset == rhs.offset
-           && lhs.length == rhs.length;
-  }
-
   constexpr ApiVersion kMinerApiVersion = makeApiVersion(1, 0, 0);
 
   /**
@@ -116,6 +107,10 @@ namespace fc::api {
                void,
                const RetrievalAsk &)
 
+    API_METHOD(MarketListIncompleteDeals,
+               jwt::kReadPermission,
+               std::vector<MinerDeal>);
+
     API_METHOD(SectorsList, jwt::kReadPermission, std::vector<SectorNumber>)
 
     API_METHOD(
@@ -249,6 +244,7 @@ namespace fc::api {
     f(a.MarketGetRetrievalAsk);
     f(a.MarketSetAsk);
     f(a.MarketSetRetrievalAsk);
+    f(a.MarketListIncompleteDeals);
     f(a.SectorsList);
     f(a.SectorsStatus);
     f(a.StorageAttach);

core/common/local_ip.hpp

@@ -0,0 +1,28 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <boost/asio.hpp>
+
+namespace fc::common {
+  inline const std::string &localIp() {
+    static const std::string ip{[] {
+      using namespace boost::asio::ip;
+      boost::asio::io_context io;
+      tcp::resolver resolver{io};
+      boost::system::error_code ec;
+      tcp::resolver::iterator end;
+      for (auto it{resolver.resolve(host_name(), "", ec)}; it != end; ++it) {
+        auto addr{it->endpoint().address()};
+        if (addr.is_v4()) {
+          return addr.to_string();
+        }
+      }
+      return std::string{"127.0.0.1"};
+    }()};
+    return ip;
+  }
+}  // namespace fc::common

core/markets/storage/deal_protocol.hpp

@@ -8,6 +8,7 @@
 #include <libp2p/peer/peer_info.hpp>
 #include <libp2p/peer/protocol.hpp>
 #include "codec/cbor/streams_annotation.hpp"
+#include "common/libp2p/peer/cbor_peer_info.hpp"
 #include "crypto/signature/signature.hpp"
 #include "primitives/address/address.hpp"
 #include "primitives/cid/cid.hpp"
@@ -103,7 +104,7 @@ namespace fc::markets::storage {
     CID proposal_cid;
     boost::optional<CID> add_funds_cid;
     boost::optional<CID> publish_cid;
-    PeerInfo client;
+    PeerInfo client = codec::cbor::kDefaultT<PeerInfo>();
     StorageDealStatus state;
     Path piece_path;
     Path metadata_path;

core/markets/storage/provider/impl/provider_impl.cpp

@@ -153,6 +153,18 @@ namespace fc::markets::storage::provider {
     return StorageMarketProviderError::kLocalDealNotFound;
   }
 
+  outcome::result<std::vector<MinerDeal>> StorageProviderImpl::getLocalDeals()
+      const {
+    std::vector<MinerDeal> deals;
+    const auto fsm_deals = fsm_->list();
+    deals.reserve(fsm_deals.size());
+    for (const auto &it : fsm_deals) {
+      deals.push_back(*it.first);
+    }
+
+    return deals;
+  }
+
   outcome::result<void> StorageProviderImpl::importDataForDeal(
       const CID &proposal_cid, const boost::filesystem::path &path) {
     auto fsm_state_table = fsm_->list();
@@ -388,19 +400,19 @@ namespace fc::markets::storage::provider {
 
     for (const auto &ref : piece_refs) {
       OUTCOME_TRY(sector_info,
-                  sector_blocks_->getMiner()->getSectorInfo(ref.sector_number));
+                  sector_blocks_->getMiner()->getSectorInfo(ref.sector));
 
       if (sector_info->state == SealingState::kProving) {
         piece_location = ref;
         break;
       }
     }
 
-    if (!piece_location.has_value()) {
-      return StorageProviderError::kNotFoundSector;
+    if (piece_location.has_value()) {
+      return piece_location.get();
     }
 
-    return piece_location.get();
+    return StorageProviderError::kNotFoundSector;
   }
 
   outcome::result<void> StorageProviderImpl::recordPieceInfo(
@@ -418,9 +430,9 @@ namespace fc::markets::storage::provider {
     OUTCOME_TRY(piece_storage_->addDealForPiece(
         deal->client_deal_proposal.proposal.piece_cid,
         DealInfo{.deal_id = deal->deal_id,
-                 .sector_id = piece_location.sector_number,
+                 .sector_id = piece_location.sector,
                  .offset = piece_location.offset,
-                 .length = piece_location.length}));
+                 .length = piece_location.size}));
     return outcome::success();
   }
 
@@ -667,14 +679,18 @@ namespace fc::markets::storage::provider {
       StorageDealStatus to) {
     // TODO(a.chernyshov): hand off
     auto &p{deal->client_deal_proposal.proposal};
-    OUTCOME_EXCEPT(sector_blocks_->addPiece(
+    auto maybe_piece_location = sector_blocks_->addPiece(
         p.piece_size.unpadded(),
         deal->piece_path,
         mining::types::DealInfo{deal->publish_cid,
                                 deal->deal_id,
                                 p,
                                 {p.start_epoch, p.end_epoch},
-                                deal->is_fast_retrieval}));
+                                deal->is_fast_retrieval});
+    FSM_HALT_ON_ERROR(maybe_piece_location, "Unable to locate piece", deal);
+    FSM_HALT_ON_ERROR(recordPieceInfo(deal, maybe_piece_location.value()),
+                      "Record piece failed",
+                      deal);
     FSM_SEND(deal, ProviderEvent::ProviderEventDealHandedOff);
   }
 
@@ -697,11 +713,6 @@ namespace fc::markets::storage::provider {
       ProviderEvent event,
       StorageDealStatus from,
       StorageDealStatus to) {
-    auto maybe_piece_location = locatePiece(deal);
-    FSM_HALT_ON_ERROR(maybe_piece_location, "Unable to locate piece", deal);
-    FSM_HALT_ON_ERROR(recordPieceInfo(deal, maybe_piece_location.value()),
-                      "Record piece failed",
-                      deal);
     // TODO(a.chernyshov): wait expiration
   }
 

core/markets/storage/provider/impl/provider_impl.hpp

@@ -32,13 +32,13 @@ namespace fc::markets::storage::provider {
   using fc::storage::filestore::FileStore;
   using fc::storage::piece::PieceStorage;
   using libp2p::Host;
-  using vm::actor::builtin::types::market::deal_info_manager::DealInfoManager;
   using pieceio::PieceIO;
   using primitives::BigInt;
   using primitives::EpochDuration;
   using primitives::GasAmount;
   using primitives::sector::RegisteredSealProof;
   using sectorblocks::SectorBlocks;
+  using vm::actor::builtin::types::market::deal_info_manager::DealInfoManager;
   using ProviderTransition =
       fsm::Transition<ProviderEvent, void, StorageDealStatus, MinerDeal>;
   using ProviderFSM =
@@ -76,6 +76,9 @@ namespace fc::markets::storage::provider {
     auto getDeal(const CID &proposal_cid) const
         -> outcome::result<MinerDeal> override;
 
+    auto getLocalDeals() const
+        -> outcome::result<std::vector<MinerDeal>> override;
+
     auto importDataForDeal(const CID &proposal_cid,
                            const boost::filesystem::path &path)
         -> outcome::result<void> override;

core/markets/storage/provider/impl/stored_ask.cpp

@@ -10,6 +10,31 @@ namespace fc::markets::storage::provider {
   // Key to store last ask in datastore
   const Bytes kBestAskKey{codec::cbor::encode("latest-ask").value()};
 
+  outcome::result<std::shared_ptr<StoredAsk>> StoredAsk::newStoredAsk(
+      std::shared_ptr<Datastore> datastore,
+      std::shared_ptr<FullNodeApi> api,
+      Address actor_address) {
+    struct make_unique_enabler : StoredAsk {
+      make_unique_enabler(std::shared_ptr<Datastore> datastore,
+                          std::shared_ptr<FullNodeApi> api,
+                          Address actor_address)
+          : StoredAsk(
+              std::move(datastore), std::move(api), std::move(actor_address)){};
+    };
+
+    std::unique_ptr<StoredAsk> stored_ask =
+        std::make_unique<make_unique_enabler>(
+            std::move(datastore), std::move(api), std::move(actor_address));
+
+    OUTCOME_TRY(maybe_ask, stored_ask->tryLoadSignedAsk());
+
+    if (not maybe_ask.has_value()) {
+      OUTCOME_TRY(stored_ask->addAsk(kDefaultPrice, kDefaultDuration));
+    }
+
+    return std::move(stored_ask);
+  }
+
   StoredAsk::StoredAsk(std::shared_ptr<Datastore> datastore,
                        std::shared_ptr<FullNodeApi> api,
                        Address actor_address)
@@ -31,12 +56,21 @@ namespace fc::markets::storage::provider {
 
   outcome::result<void> StoredAsk::addAsk(const TokenAmount &price,
                                           ChainEpoch duration) {
+    auto min_size = kDefaultMinPieceSize;
+    auto max_size = kDefaultMaxPieceSize;
+
+    if (last_signed_storage_ask_.has_value()) {
+      auto &ask{(*last_signed_storage_ask_).ask};
+      min_size = ask.min_piece_size;
+      max_size = ask.max_piece_size;
+    }
+
     return addAsk(
         {
             .price = price,
             .verified_price = price,
-            .min_piece_size = kDefaultMinPieceSize,
-            .max_piece_size = kDefaultMaxPieceSize,
+            .min_piece_size = min_size,
+            .max_piece_size = max_size,
             .miner = actor_,
         },
         duration);
@@ -54,26 +88,18 @@ namespace fc::markets::storage::provider {
   }
 
   outcome::result<SignedStorageAsk> StoredAsk::loadSignedAsk() {
-    if (datastore_->contains(kBestAskKey)) {
-      OUTCOME_TRY(ask_bytes, datastore_->get(kBestAskKey));
-      OUTCOME_TRY(ask, codec::cbor::decode<SignedStorageAsk>(ask_bytes));
-      return std::move(ask);
-    }
+    OUTCOME_TRY(ask_bytes, datastore_->get(kBestAskKey));
+    OUTCOME_TRY(ask, codec::cbor::decode<SignedStorageAsk>(ask_bytes));
+    return std::move(ask);
+  }
 
-    // otherwise return default which 'not actively accepting deals'
-    OUTCOME_TRY(chain_head, api_->ChainHead());
-    ChainEpoch timestamp = chain_head->height();
-    ChainEpoch expiry = chain_head->height() + kDefaultDuration;
-    StorageAsk default_ask{.price = kDefaultPrice,
-                           .verified_price = kDefaultPrice,
-                           .min_piece_size = kDefaultMinPieceSize,
-                           .max_piece_size = kDefaultMaxPieceSize,
-                           .miner = actor_,
-                           .timestamp = timestamp,
-                           .expiry = expiry,
-                           .seq_no = 0};
-    OUTCOME_TRY(signed_ask, signAsk(default_ask, *chain_head));
-    return std::move(signed_ask);
+  outcome::result<boost::optional<SignedStorageAsk>>
+  StoredAsk::tryLoadSignedAsk() {
+    if (not datastore_->contains(kBestAskKey)) {
+      return boost::none;
+    }
+    OUTCOME_TRY(ask, loadSignedAsk());
+    return std::move(ask);
   }
 
   outcome::result<void> StoredAsk::saveSignedAsk(const SignedStorageAsk &ask) {

core/markets/storage/provider/provider.hpp

@@ -34,6 +34,9 @@ namespace fc::markets::storage::provider {
     virtual auto getDeal(const CID &proposal_cid) const
         -> outcome::result<MinerDeal> = 0;
 
+    virtual auto getLocalDeals() const
+        -> outcome::result<std::vector<MinerDeal>> = 0;
+
     /**
      * Imports data to proceed deal with 'manual' transfer type
      * @param proposal_cid - deal proposal CID