Skip to content

Commit d61f480

Browse files
Alexey-N-ChernyshovAlexey-N-Chernyshov
authored
new task types, resource table and API for Snap Deals (#579)
 Signed-off-by: Alexey Chernyshov <[email protected]>
1 parent 8b5ef04 commit d61f480

File tree

8 files changed

+179
-94
lines changed

8 files changed

+179
-94
lines changed

‎core/api/worker_api.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ namespace fc::api {
2929
using sector_storage::Range;
3030
using sector_storage::SectorCids;
3131
using sector_storage::SectorFileType;
32+
using sector_storage::Update1Output;
3233

3334
struct WorkerApi {
3435
API_METHOD(AddPiece,

‎core/primitives/resources/resources.hpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@ namespace fc::primitives {
124124
.base_min_memory = uint64_t(2) << 10,
125125
}},
126126
}},
127-
// TTPreCommit2
128127
{kTTPreCommit2,
129128
{
130129
{RegisteredSealProof::kStackedDrg64GiBV1,
@@ -232,7 +231,7 @@ namespace fc::primitives {
232231
{RegisteredSealProof::kStackedDrg512MiBV1,
233232
Resources{
234233
.min_memory = uint64_t(1) << 30,
235-
.max_memory = uint64_t(3) << 29,
234+
.max_memory = uint64_t(3) << 29, // 1.5G
236235
.threads = 1,
237236
.can_gpu = true,
238237
.base_min_memory = uint64_t(10) << 30,
@@ -301,6 +300,10 @@ namespace fc::primitives {
301300

302301
res[kTTUnseal] = res[kTTPreCommit1];
303302
res[kTTReadUnsealed] = res[kTTFetch];
303+
res[kTTReplicaUpdate] = res[kTTAddPiece];
304+
res[kTTProveReplicaUpdate1] = res[kTTCommit1];
305+
res[kTTProveReplicaUpdate2] = res[kTTCommit2];
306+
res[kTTRegenSectorKey] = res[kTTReplicaUpdate];
304307

305308
for (auto &[key, value] : res) {
306309
value[RegisteredSealProof::kStackedDrg2KiBV1_1] =

‎core/primitives/seal_tasks/task.hpp

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,30 @@ namespace fc::primitives {
2929

3030
const TaskType kTTFetch("seal/v0/fetch");
3131
const TaskType kTTUnseal("seal/v0/unseal");
32+
// TODO (a.chernyshov) ReadUnsealed was removed in Lotus
33+
// (https://github.com/filecoin-project/cpp-filecoin/issues/574)
3234
const TaskType kTTReadUnsealed("seal/v0/unsealread");
3335

36+
const TaskType kTTReplicaUpdate("seal/v0/replicaupdate");
37+
const TaskType kTTProveReplicaUpdate1("seal/v0/provereplicaupdate/1");
38+
const TaskType kTTProveReplicaUpdate2("seal/v0/provereplicaupdate/2");
39+
const TaskType kTTRegenSectorKey("seal/v0/regensectorkey");
40+
3441
inline bool operator<(const TaskType &lhs, const TaskType &rhs) {
35-
static std::unordered_map<std::string, int> order = {{kTTReadUnsealed, 0},
36-
{kTTUnseal, 0},
37-
{kTTFinalize, 1},
38-
{kTTFetch, 2},
39-
{kTTCommit1, 3},
40-
{kTTCommit2, 4},
41-
{kTTPreCommit2, 5},
42-
{kTTPreCommit1, 6},
43-
{kTTAddPiece, 7}};
42+
static std::unordered_map<std::string, int> order = {
43+
{kTTFinalize, -2},
44+
{kTTReadUnsealed, 0},
45+
{kTTFetch, -1},
46+
{kTTUnseal, 1},
47+
{kTTCommit1, 2},
48+
{kTTCommit2, 3},
49+
{kTTPreCommit2, 4},
50+
{kTTPreCommit1, 5},
51+
{kTTProveReplicaUpdate1, 6},
52+
{kTTProveReplicaUpdate2, 7},
53+
{kTTReplicaUpdate, 8},
54+
{kTTAddPiece, 9},
55+
{kTTRegenSectorKey, 10}};
4456

4557
return order[lhs] < order[rhs];
4658
}

‎core/remote_worker/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
add_executable(remote_worker_main
77
main.cpp
8+
remote_worker_api.cpp
89
)
910
target_link_libraries(remote_worker_main
1011
Boost::program_options

‎core/remote_worker/main.cpp

Lines changed: 18 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -26,33 +26,20 @@
2626
#include "config/profile_config.hpp"
2727
#include "primitives/address/config.hpp"
2828
#include "proofs/proof_param_provider.hpp"
29+
#include "remote_worker/remote_worker_api.hpp"
2930
#include "sector_storage/fetch_handler.hpp"
3031
#include "sector_storage/impl/local_worker.hpp"
3132
#include "sector_storage/stores/impl/remote_index_impl.hpp"
3233
#include "sector_storage/stores/impl/remote_store.hpp"
3334
#include "sector_storage/stores/impl/storage_impl.hpp"
3435

35-
namespace fc {
36+
namespace fc::remote_worker {
3637
using api::kMinerApiVersion;
37-
using api::VersionResult;
3838
using boost::asio::io_context;
3939
using config::configProfile;
4040
using libp2p::multi::Multiaddress;
41-
using primitives::piece::PieceInfo;
42-
using primitives::piece::UnpaddedByteIndex;
43-
using primitives::piece::UnpaddedPieceSize;
44-
using primitives::sector::SealRandomness;
45-
using primitives::sector::SectorRef;
4641
using proofs::ProofParamProvider;
47-
using sector_storage::AcquireMode;
48-
using sector_storage::Commit1Output;
49-
using sector_storage::InteractiveRandomness;
5042
using sector_storage::LocalWorker;
51-
using sector_storage::PathType;
52-
using sector_storage::PreCommit1Output;
53-
using sector_storage::Range;
54-
using sector_storage::SectorCids;
55-
using sector_storage::SectorFileType;
5643
using sector_storage::stores::LocalStore;
5744
namespace uuids = boost::uuids;
5845

@@ -63,7 +50,7 @@ namespace fc {
6350
boost::filesystem::path repo_path;
6451
std::pair<Multiaddress, std::string> miner_api{
6552
codec::cbor::kDefaultT<Multiaddress>(), {}};
66-
int api_port;
53+
int api_port{};
6754

6855
std::set<primitives::TaskType> tasks;
6956
bool need_download = false;
@@ -83,6 +70,8 @@ namespace fc {
8370
bool can_precommit2 = false;
8471
bool can_commit = false;
8572
bool can_unseal = false;
73+
bool can_replica_update = false;
74+
bool can_prove_replica_update2 = false;
8675
} raw;
8776

8877
po::options_description desc("Fuhon worker options");
@@ -95,6 +84,8 @@ namespace fc {
9584
option("precommit2", po::value(&raw.can_precommit2)->default_value(true));
9685
option("commit", po::value(&raw.can_commit)->default_value(true));
9786
option("unseal", po::value(&raw.can_unseal)->default_value(true));
87+
option("replica-update", po::value(&raw.can_replica_update));
88+
option("prove-replica-update2", po::value(&raw.can_prove_replica_update2));
9889
desc.add(configProfile());
9990
primitives::address::configCurrentNetwork(option);
10091

@@ -130,6 +121,12 @@ namespace fc {
130121
if (raw.can_unseal) {
131122
config.tasks.insert(primitives::kTTUnseal);
132123
}
124+
if (raw.can_replica_update) {
125+
config.tasks.insert(primitives::kTTReplicaUpdate);
126+
}
127+
if (raw.can_prove_replica_update2) {
128+
config.tasks.insert(primitives::kTTProveReplicaUpdate2);
129+
}
133130

134131
return config;
135132
}
@@ -211,66 +208,10 @@ namespace fc {
211208

212209
auto worker{std::make_shared<LocalWorker>(io, wconfig, mapi, remote_store)};
213210

214-
auto wapi{std::make_shared<api::WorkerApi>()};
215-
wapi->Version = []() { return VersionResult{"seal-worker", 0, 0}; };
216-
wapi->StorageAddLocal = [&](const std::string &path) {
217-
return local_store->openPath(path);
218-
};
219-
wapi->Fetch = [&](const SectorRef &sector,
220-
const SectorFileType &file_type,
221-
PathType path_type,
222-
AcquireMode mode) {
223-
return worker->fetch(sector, file_type, path_type, mode);
224-
};
225-
wapi->UnsealPiece = [&](const SectorRef &sector,
226-
UnpaddedByteIndex offset,
227-
const UnpaddedPieceSize &size,
228-
const SealRandomness &randomness,
229-
const CID &unsealed_cid) {
230-
return worker->unsealPiece(
231-
sector, offset, size, randomness, unsealed_cid);
232-
};
233-
wapi->MoveStorage = [&](const SectorRef &sector,
234-
const SectorFileType &types) {
235-
return worker->moveStorage(sector, types);
236-
};
237-
238-
wapi->Info = [&]() { return worker->getInfo(); };
239-
wapi->Paths = [&]() { return worker->getAccessiblePaths(); };
240-
wapi->TaskTypes = [&]() -> outcome::result<std::set<primitives::TaskType>> {
241-
OUTCOME_TRY(tasks, worker->getSupportedTask());
242-
// TODO(ortyomka): [FIL-344] Remove its
243-
tasks.extract(primitives::kTTAddPiece);
244-
return std::move(tasks);
245-
};
246-
247-
wapi->SealPreCommit1 = [&](const SectorRef &sector,
248-
const SealRandomness &ticket,
249-
std::vector<PieceInfo> pieces) {
250-
return worker->sealPreCommit1(sector, ticket, pieces);
251-
};
252-
wapi->SealPreCommit2 = [&](const SectorRef &sector,
253-
const PreCommit1Output &pre_commit_1_output) {
254-
return worker->sealPreCommit2(sector, pre_commit_1_output);
255-
};
256-
wapi->SealCommit1 = [&](const SectorRef &sector,
257-
const SealRandomness &ticket,
258-
const InteractiveRandomness &seed,
259-
std::vector<PieceInfo> pieces,
260-
const SectorCids &cids) {
261-
return worker->sealCommit1(sector, ticket, seed, pieces, cids);
262-
};
263-
wapi->SealCommit2 = [&](const SectorRef &sector,
264-
const Commit1Output &commit_1_output) {
265-
return worker->sealCommit2(sector, commit_1_output);
266-
};
267-
wapi->FinalizeSector = [&](const SectorRef &sector,
268-
std::vector<Range> keep_unsealed) {
269-
return worker->finalizeSector(sector, keep_unsealed);
270-
};
211+
auto worker_api = makeWorkerApi(local_store, worker);
271212

272213
std::map<std::string, std::shared_ptr<api::Rpc>> wrpc;
273-
wrpc.emplace("/rpc/v0", api::makeRpc(*wapi));
214+
wrpc.emplace("/rpc/v0", api::makeRpc(*worker_api));
274215
auto wroutes{std::make_shared<api::Routes>()};
275216

276217
wroutes->insert({"/remote",
@@ -304,9 +245,9 @@ namespace fc {
304245
io->run();
305246
return outcome::success();
306247
}
307-
} // namespace fc
248+
} // namespace fc::remote_worker
308249

309250
int main(int argc, char **argv) {
310-
OUTCOME_EXCEPT(config, fc::readConfig(argc, argv));
311-
OUTCOME_EXCEPT(fc::main(config));
251+
OUTCOME_EXCEPT(config, fc::remote_worker::readConfig(argc, argv));
252+
OUTCOME_EXCEPT(fc::remote_worker::main(config));
312253
}

‎core/remote_worker/remote_worker_api.cpp

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/**
2+
* Copyright Soramitsu Co., Ltd. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
#include "remote_worker/remote_worker_api.hpp"
7+
8+
namespace fc::remote_worker {
9+
using api::VersionResult;
10+
using primitives::piece::PieceInfo;
11+
using primitives::piece::UnpaddedByteIndex;
12+
using primitives::piece::UnpaddedPieceSize;
13+
using primitives::sector::SealRandomness;
14+
using primitives::sector::SectorRef;
15+
using sector_storage::AcquireMode;
16+
using sector_storage::Commit1Output;
17+
using sector_storage::InteractiveRandomness;
18+
using sector_storage::PathType;
19+
using sector_storage::PreCommit1Output;
20+
using sector_storage::Range;
21+
using sector_storage::SectorCids;
22+
using sector_storage::SectorFileType;
23+
using sector_storage::Update1Output;
24+
25+
std::shared_ptr<WorkerApi> makeWorkerApi(
26+
const std::shared_ptr<LocalStore> &local_store,
27+
const std::shared_ptr<LocalWorker> &worker) {
28+
auto worker_api{std::make_shared<api::WorkerApi>()};
29+
worker_api->Version = []() { return VersionResult{"seal-worker", 0, 0}; };
30+
worker_api->StorageAddLocal = [=](const std::string &path) {
31+
return local_store->openPath(path);
32+
};
33+
worker_api->Fetch = [=](const SectorRef &sector,
34+
const SectorFileType &file_type,
35+
PathType path_type,
36+
AcquireMode mode) {
37+
return worker->fetch(sector, file_type, path_type, mode);
38+
};
39+
worker_api->UnsealPiece = [=](const SectorRef &sector,
40+
UnpaddedByteIndex offset,
41+
const UnpaddedPieceSize &size,
42+
const SealRandomness &randomness,
43+
const CID &unsealed_cid) {
44+
return worker->unsealPiece(
45+
sector, offset, size, randomness, unsealed_cid);
46+
};
47+
worker_api->MoveStorage = [=](const SectorRef &sector,
48+
const SectorFileType &types) {
49+
return worker->moveStorage(sector, types);
50+
};
51+
52+
worker_api->Info = [=]() { return worker->getInfo(); };
53+
worker_api->Paths = [=]() { return worker->getAccessiblePaths(); };
54+
worker_api->TaskTypes =
55+
[=]() -> outcome::result<std::set<primitives::TaskType>> {
56+
return worker->getSupportedTask();
57+
};
58+
59+
worker_api->SealPreCommit1 = [=](const SectorRef &sector,
60+
const SealRandomness &ticket,
61+
const std::vector<PieceInfo> &pieces) {
62+
return worker->sealPreCommit1(sector, ticket, pieces);
63+
};
64+
worker_api->SealPreCommit2 =
65+
[=](const SectorRef &sector,
66+
const PreCommit1Output &pre_commit_1_output) {
67+
return worker->sealPreCommit2(sector, pre_commit_1_output);
68+
};
69+
worker_api->SealCommit1 = [=](const SectorRef &sector,
70+
const SealRandomness &ticket,
71+
const InteractiveRandomness &seed,
72+
const std::vector<PieceInfo> &pieces,
73+
const SectorCids &cids) {
74+
return worker->sealCommit1(sector, ticket, seed, pieces, cids);
75+
};
76+
worker_api->SealCommit2 = [=](const SectorRef &sector,
77+
const Commit1Output &commit_1_output) {
78+
return worker->sealCommit2(sector, commit_1_output);
79+
};
80+
worker_api->FinalizeSector = [=](const SectorRef &sector,
81+
std::vector<Range> keep_unsealed) {
82+
return worker->finalizeSector(sector, keep_unsealed);
83+
};
84+
85+
worker_api->ReplicaUpdate = [=](const SectorRef &sector,
86+
const std::vector<PieceInfo> &pieces) {
87+
return worker->replicaUpdate(sector, pieces);
88+
};
89+
worker_api->ProveReplicaUpdate1 = [=](const SectorRef &sector,
90+
const CID &sector_key,
91+
const CID &new_sealed,
92+
const CID &new_unsealed) {
93+
return worker->proveReplicaUpdate1(
94+
sector, sector_key, new_sealed, new_unsealed);
95+
};
96+
worker_api->ProveReplicaUpdate2 = [=](const SectorRef &sector,
97+
const CID &sector_key,
98+
const CID &new_sealed,
99+
const CID &new_unsealed,
100+
const Update1Output &update1_output) {
101+
return worker->proveReplicaUpdate2(
102+
sector, sector_key, new_sealed, new_unsealed, update1_output);
103+
};
104+
105+
return worker_api;
106+
};
107+
108+
} // namespace fc::remote_worker

‎core/remote_worker/remote_worker_api.hpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright Soramitsu Co., Ltd. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
#pragma once
7+
8+
#include "api/worker_api.hpp"
9+
#include "sector_storage/impl/local_worker.hpp"
10+
11+
namespace fc::remote_worker {
12+
using fc::api::WorkerApi;
13+
using sector_storage::LocalWorker;
14+
using sector_storage::stores::LocalStore;
15+
16+
std::shared_ptr<WorkerApi> makeWorkerApi(
17+
const std::shared_ptr<LocalStore> &local_store,
18+
const std::shared_ptr<LocalWorker> &worker);
19+
20+
} // namespace fc::remote_worker

‎core/sector_storage/worker.hpp

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,19 @@ namespace fc::sector_storage {
2424
using primitives::piece::PieceInfo;
2525
using primitives::piece::UnpaddedByteIndex;
2626
using primitives::piece::UnpaddedPieceSize;
27+
using primitives::sector::InteractiveRandomness;
28+
using primitives::sector::Proof;
2729
using primitives::sector::SealRandomness;
2830
using primitives::sector::SectorId;
2931
using primitives::sector::SectorRef;
3032
using primitives::sector_file::SectorFileType;
33+
using stores::AcquireMode;
34+
using stores::PathType;
35+
using stores::StoreError;
3136
using PreCommit1Output = proofs::Phase1Output;
3237
using Commit1Output = proofs::Phase1Output;
3338
using Update1Output = proofs::UpdateProofs1;
3439
using SectorCids = proofs::SealedAndUnsealedCID;
35-
using primitives::sector::InteractiveRandomness;
36-
using primitives::sector::Proof;
37-
using primitives::sector::SealRandomness;
38-
using stores::AcquireMode;
39-
using stores::PathType;
40-
using stores::StoreError;
4140

4241
struct Range {
4342
UnpaddedPieceSize offset;

0 commit comments

Comments
 (0)