Skip to content

Commit 868797d

Browse files
ortyomkaAlexey-N-Chernyshov
ortyomka
and
Alexey-N-Chernyshov
authored
Fixes Miner (#555)
* Fixes Miner Signed-off-by: ortyomka <[email protected]> * clang-tidy fixes (#557) * clang-tidy fixes Signed-off-by: Alexey Chernyshov <[email protected]> * PR fixes Signed-off-by: Alexey Chernyshov <[email protected]> * clang-tidy fixes (#563) Signed-off-by: Alexey Chernyshov <[email protected]> * Speed fix Signed-off-by: ortyomka <[email protected]> * Optimize pad pieces addition Signed-off-by: ortyomka <[email protected]> Co-authored-by: Alexey <[email protected]>
1 parent 3e51b22 commit 868797d

36 files changed

+1158
-820
lines changed

‎core/api/rpc/info.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ namespace fc::api::rpc {
1717

1818
inline Outcome<std::pair<Multiaddress, std::string>> loadInfo(
1919
const boost::filesystem::path &repo, std::string_view env) {
20-
std::string address, token;
20+
std::string address;
21+
std::string token;
2122
if (auto _info{getenv(env.data())}) {
2223
std::string_view info{_info};
2324
auto i{info.find(":")};

‎core/api/storage_miner/storage_api.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ namespace fc::api {
3333
return miner_info.sector_size;
3434
};
3535

36-
api->PledgeSector = [&]() -> outcome::result<void> {
36+
api->PledgeSector = [=]() -> outcome::result<void> {
3737
return miner->getSealing()->pledgeSector();
3838
};
3939

‎core/api/utils.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ namespace fc::api {
3333
using FunctionSimpleSignature = OutcomeResult(Ts...);
3434
using FunctionSignature = void(Callback, Ts...);
3535

36+
// TODO (ortyomka): [Issue 550] split sync blocking calls
3637
OutcomeResult operator()(Ts... args) const {
3738
if (!f_) {
3839
return ERROR_TEXT("API not set up");
@@ -53,7 +54,7 @@ namespace fc::api {
5354

5455
ApiMethod &operator=(std::function<FunctionSimpleSignature> &&f) {
5556
if (f) {
56-
f_ = [wf{std::move(f)}](auto &&cb, auto &&... args) -> void {
57+
f_ = [wf{std::move(f)}](auto &&cb, auto &&...args) -> void {
5758
cb(wf(std::forward<decltype(args)>(args)...));
5859
};
5960
} else {

‎core/const.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ namespace fc {
4646
const TokenAmount kMinimumBaseFee{100};
4747
constexpr auto kPackingEfficiencyDenom{5};
4848
constexpr auto kPackingEfficiencyNum{4};
49+
const TokenAmount kOneNanoFil{1000000000};
4950

5051
// ******************
5152
// Network versions

‎core/crypto/blake2/blake2b160.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,14 +147,14 @@ namespace fc::crypto::blake2b {
147147

148148
Ctx ctx{BLAKE2B512_HASH_LENGTH};
149149

150-
const int one_kb = 1024;
151-
std::string bytes(one_kb, ' ');
152-
file_stream.read(bytes.data(), one_kb);
150+
constexpr size_t buffer_size = 32 * 1024;
151+
std::string bytes(buffer_size, ' ');
152+
file_stream.read(bytes.data(), buffer_size);
153153
auto currently_read = file_stream.gcount();
154154
while (currently_read != 0) {
155155
ctx.update(gsl::make_span(common::span::cast<const uint8_t>(bytes.data()),
156156
currently_read));
157-
file_stream.read(bytes.data(), one_kb);
157+
file_stream.read(bytes.data(), buffer_size);
158158
currently_read = file_stream.gcount();
159159
}
160160

‎core/miner/impl/miner_impl.cpp

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -33,17 +33,18 @@ namespace fc::miner {
3333
using vm::actor::builtin::types::miner::kMaxSectorExpirationExtension;
3434
using vm::actor::builtin::types::miner::kWPoStProvingPeriod;
3535

36-
MinerImpl::MinerImpl(std::shared_ptr<FullNodeApi> api,
37-
std::shared_ptr<Sealing> sealing)
38-
: api_{std::move(api)}, sealing_{std::move(sealing)} {}
36+
MinerImpl::MinerImpl(std::shared_ptr<Sealing> sealing)
37+
: sealing_{std::move(sealing)} {}
3938

4039
outcome::result<std::shared_ptr<SectorInfo>> MinerImpl::getSectorInfo(
4140
SectorNumber sector_id) const {
4241
return sealing_->getSectorInfo(sector_id);
4342
}
4443

4544
outcome::result<PieceLocation> MinerImpl::addPieceToAnySector(
46-
UnpaddedPieceSize size, PieceData piece_data, DealInfo deal) {
45+
const UnpaddedPieceSize &size,
46+
PieceData piece_data,
47+
const DealInfo &deal) {
4748
return sealing_->addPieceToAnySector(size, std::move(piece_data), deal);
4849
}
4950

@@ -56,14 +57,14 @@ namespace fc::miner {
5657
}
5758

5859
outcome::result<std::shared_ptr<MinerImpl>> MinerImpl::newMiner(
59-
std::shared_ptr<FullNodeApi> api,
60+
const std::shared_ptr<FullNodeApi> &api,
6061
const Address &miner_address,
6162
const Address &worker_address,
62-
std::shared_ptr<Counter> counter,
63-
std::shared_ptr<BufferMap> sealing_fsm_kv,
64-
std::shared_ptr<Manager> sector_manager,
63+
const std::shared_ptr<Counter> &counter,
64+
const std::shared_ptr<BufferMap> &sealing_fsm_kv,
65+
const std::shared_ptr<Manager> &sector_manager,
6566
const std::shared_ptr<Scheduler> &scheduler,
66-
std::shared_ptr<boost::asio::io_context> context,
67+
const std::shared_ptr<boost::asio::io_context> &context,
6768
const mining::Config &config,
6869
const std::vector<Address>
6970
&precommit_control) { // TODO(ortyomka): Commit Batcher extension
@@ -92,40 +93,36 @@ namespace fc::miner {
9293
fee_config->max_precommit_batch_gas_fee.per_sector =
9394
TokenAmount{"2000000000000000"};
9495
fee_config->max_precommit_gas_fee = TokenAmount{"25000000000000000"};
96+
9597
std::shared_ptr<PreCommitBatcher> precommit_batcher =
96-
std::make_shared<PreCommitBatcherImpl>(
97-
std::chrono::seconds(60),
98-
api,
99-
miner_address,
100-
scheduler,
101-
[=](const MinerInfo &miner_info,
102-
const TokenAmount &good_funds,
103-
const std::shared_ptr<FullNodeApi> &api)
104-
-> outcome::result<Address> {
105-
return SelectAddress(miner_info, good_funds, api);
106-
},
107-
fee_config);
98+
std::make_shared<PreCommitBatcherImpl>(std::chrono::seconds(60),
99+
api,
100+
miner_address,
101+
scheduler,
102+
SelectAddress,
103+
fee_config);
108104
OUTCOME_TRY(sealing,
109105
SealingImpl::newSealing(api,
110106
events,
111107
miner_address,
112-
std::move(counter),
113-
std::move(sealing_fsm_kv),
114-
std::move(sector_manager),
108+
counter,
109+
sealing_fsm_kv,
110+
sector_manager,
115111
precommit_policy,
116-
std::move(context),
112+
context,
117113
scheduler,
118114
precommit_batcher,
115+
SelectAddress,
116+
fee_config,
119117
config));
120118

121119
struct make_unique_enabler : public MinerImpl {
122-
make_unique_enabler(std::shared_ptr<FullNodeApi> api,
123-
std::shared_ptr<Sealing> sealing)
124-
: MinerImpl{std::move(api), std::move(sealing)} {};
120+
explicit make_unique_enabler(std::shared_ptr<Sealing> sealing)
121+
: MinerImpl{std::move(sealing)} {};
125122
};
126123

127124
std::shared_ptr<MinerImpl> miner =
128-
std::make_shared<make_unique_enabler>(api, sealing);
125+
std::make_shared<make_unique_enabler>(std::move(sealing));
129126

130127
return std::move(miner);
131128
}

‎core/miner/impl/miner_impl.hpp

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,33 +33,32 @@ namespace fc::miner {
3333
class MinerImpl : public Miner {
3434
public:
3535
static outcome::result<std::shared_ptr<MinerImpl>> newMiner(
36-
std::shared_ptr<FullNodeApi> api,
36+
const std::shared_ptr<FullNodeApi> &api,
3737
const Address &miner_address,
3838
const Address &worker_address,
39-
std::shared_ptr<Counter> counter,
40-
std::shared_ptr<BufferMap> sealing_fsm_kv,
41-
std::shared_ptr<Manager> sector_manager,
39+
const std::shared_ptr<Counter> &counter,
40+
const std::shared_ptr<BufferMap> &sealing_fsm_kv,
41+
const std::shared_ptr<Manager> &sector_manager,
4242
const std::shared_ptr<Scheduler> &scheduler,
43-
std::shared_ptr<boost::asio::io_context> context,
43+
const std::shared_ptr<boost::asio::io_context> &context,
4444
const mining::Config &config,
4545
const std::vector<Address> &precommit_control);
4646

4747
outcome::result<std::shared_ptr<SectorInfo>> getSectorInfo(
4848
SectorNumber sector_id) const override;
4949

50-
outcome::result<PieceLocation> addPieceToAnySector(UnpaddedPieceSize size,
51-
PieceData piece_data,
52-
DealInfo deal) override;
50+
outcome::result<PieceLocation> addPieceToAnySector(
51+
const UnpaddedPieceSize &size,
52+
PieceData piece_data,
53+
const DealInfo &deal) override;
5354

5455
Address getAddress() const override;
5556

5657
std::shared_ptr<Sealing> getSealing() const override;
5758

5859
private:
59-
MinerImpl(std::shared_ptr<FullNodeApi> api,
60-
std::shared_ptr<Sealing> sealing);
60+
explicit MinerImpl(std::shared_ptr<Sealing> sealing);
6161

62-
std::shared_ptr<FullNodeApi> api_;
6362
std::shared_ptr<Sealing> sealing_;
6463
};
6564

‎core/miner/main/main.cpp

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ namespace fc {
9494
boost::optional<Address> actor, owner, worker;
9595
boost::optional<RegisteredSealProof> seal_type;
9696
std::vector<Address> precommit_control;
97-
int api_port;
97+
int api_port{};
9898

9999
/** Path to presealed sectors */
100100
boost::optional<boost::filesystem::path> preseal_path;
@@ -125,8 +125,8 @@ namespace fc {
125125
for (const auto &elem : meta.sectors) {
126126
// TODO(ortyomka): migrate sealing info
127127

128-
if (max_sector < elem.sector_id) {
129-
max_sector = elem.sector_id;
128+
if (max_sector <= elem.sector_id) {
129+
max_sector = elem.sector_id + 1;
130130
}
131131
}
132132
OUTCOME_TRY(sc.setNumber(max_sector));
@@ -204,6 +204,7 @@ namespace fc {
204204
return config;
205205
}
206206

207+
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
207208
outcome::result<void> setupMiner(Config &config,
208209
BufferMap &kv,
209210
const PeerId &peer_id) {
@@ -335,6 +336,7 @@ namespace fc {
335336
return outcome::success();
336337
}
337338

339+
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
338340
outcome::result<void> main(Config &config) {
339341
log()->debug("Starting ", miner::kMinerVersion);
340342

@@ -426,6 +428,18 @@ namespace fc {
426428
sector_storage::SchedulerImpl::newScheduler(
427429
io_thread2.io, prefixed("scheduler_works/")));
428430
IoThread io_thread3;
431+
432+
{
433+
uint64_t cpus = std::thread::hardware_concurrency();
434+
if (cpus == 0) {
435+
cpus = sysconf(_SC_NPROCESSORS_ONLN);
436+
}
437+
438+
for (size_t i{0}; i < cpus; ++i) {
439+
pool.emplace_back(std::thread{[&] { io_thread3.io->run(); }});
440+
}
441+
}
442+
429443
OUTCOME_TRY(
430444
manager,
431445
sector_storage::ManagerImpl::newManager(
@@ -435,7 +449,8 @@ namespace fc {
435449
mining::Config default_config{.max_wait_deals_sectors = 2,
436450
.max_sealing_sectors = 0,
437451
.max_sealing_sectors_for_deals = 0,
438-
.wait_deals_delay = std::chrono::hours(6)};
452+
.wait_deals_delay = std::chrono::hours(6),
453+
.batch_pre_commits = true};
439454
OUTCOME_TRY(miner,
440455
miner::MinerImpl::newMiner(
441456
napi,

‎core/miner/miner.hpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ namespace fc::miner {
2828
SectorNumber sector_id) const = 0;
2929

3030
virtual outcome::result<PieceLocation> addPieceToAnySector(
31-
UnpaddedPieceSize size, PieceData piece_data, DealInfo deal) = 0;
31+
const UnpaddedPieceSize &size,
32+
PieceData piece_data,
33+
const DealInfo &deal) = 0;
3234

3335
virtual Address getAddress() const = 0;
3436

‎core/miner/storage_fsm/impl/precommit_batcher_impl.cpp

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@
55

66
#include "miner/storage_fsm/impl/precommit_batcher_impl.hpp"
77

8+
#include <utility>
9+
810
#include "vm/actor/actor.hpp"
11+
#include "vm/actor/builtin/types/miner/v6/monies.hpp"
912
#include "vm/actor/builtin/v5/miner/miner_actor.hpp"
1013

1114
namespace fc::mining {
@@ -14,25 +17,26 @@ namespace fc::mining {
1417
using vm::actor::MethodParams;
1518
using vm::actor::builtin::types::miner::kChainFinality;
1619
using vm::actor::builtin::v5::miner::PreCommitBatch;
20+
using vm::actor::builtin::v6::miner::aggregatePreCommitNetworkFee;
1721

18-
PreCommitBatcherImpl::PreCommitEntry::PreCommitEntry(
19-
const TokenAmount &number, const SectorPreCommitInfo &info)
20-
: deposit(number), precommit_info(info){};
22+
PreCommitBatcherImpl::PreCommitEntry::PreCommitEntry(TokenAmount number,
23+
SectorPreCommitInfo info)
24+
: deposit(std::move(number)), precommit_info(std::move(info)){};
2125

2226
PreCommitBatcherImpl::PreCommitBatcherImpl(
2327
const std::chrono::milliseconds &max_time,
2428
std::shared_ptr<FullNodeApi> api,
25-
const Address &miner_address,
26-
const std::shared_ptr<Scheduler> &scheduler,
27-
const AddressSelector &address_selector,
29+
Address miner_address,
30+
std::shared_ptr<Scheduler> scheduler,
31+
AddressSelector address_selector,
2832
std::shared_ptr<FeeConfig> fee_config)
2933
: max_delay_(max_time),
3034
api_(std::move(api)),
31-
miner_address_(miner_address),
32-
scheduler_(scheduler),
35+
miner_address_(std::move(miner_address)),
36+
scheduler_(std::move(scheduler)),
3337
closest_cutoff_(max_time),
3438
fee_config_(std::move(fee_config)),
35-
address_selector_(address_selector) {
39+
address_selector_(std::move(address_selector)) {
3640
cutoff_start_ = std::chrono::system_clock::now();
3741
logger_ = common::createLogger("batcher");
3842
logger_->info("Batcher has been started");
@@ -58,36 +62,48 @@ namespace fc::mining {
5862
}
5963

6064
outcome::result<CID> PreCommitBatcherImpl::sendBatch() {
61-
if (batch_storage_.size() != 0) {
65+
if (!batch_storage_.empty()) {
6266
logger_->info("Sending procedure started");
6367
OUTCOME_TRY(head, api_->ChainHead());
6468
OUTCOME_TRY(minfo, api_->StateMinerInfo(miner_address_, head->key));
6569

6670
PreCommitBatch::Params params;
67-
71+
TokenAmount mutual_deposit;
6872
for (const auto &data : batch_storage_) {
69-
mutual_deposit_ += data.second.deposit;
73+
mutual_deposit += data.second.deposit;
7074
params.sectors.push_back(data.second.precommit_info);
7175
}
7276
TokenAmount max_fee =
7377
fee_config_->max_precommit_batch_gas_fee.FeeForSector(
7478
params.sectors.size());
75-
TokenAmount good_funds = mutual_deposit_ + max_fee;
79+
80+
const auto base_fee = head->blks[0].parent_base_fee;
81+
const TokenAmount agg_fee_raw =
82+
aggregatePreCommitNetworkFee(params.sectors.size(), base_fee);
83+
84+
static TokenAmount kAggFeeNum{110};
85+
static TokenAmount kAggFeeDen{100};
86+
const TokenAmount agg_fee = bigdiv(agg_fee_raw * kAggFeeNum, kAggFeeDen);
87+
88+
const auto need_funds = mutual_deposit + agg_fee;
89+
90+
// TODO(ortyomka): Collateral Send Amount
91+
92+
const TokenAmount good_funds = max_fee + need_funds;
7693
OUTCOME_TRY(encodedParams, codec::cbor::encode(params));
7794
OUTCOME_TRY(address, address_selector_(minfo, good_funds, api_));
7895
OUTCOME_TRY(signed_message,
7996
api_->MpoolPushMessage(
8097
vm::message::UnsignedMessage(miner_address_,
8198
address,
8299
0,
83-
mutual_deposit_,
100+
need_funds,
84101
max_fee,
85102
{},
86103
PreCommitBatch::Number,
87104
MethodParams{encodedParams}),
88105
kPushNoSpec));
89106

90-
mutual_deposit_ = 0;
91107
batch_storage_.clear();
92108
logger_->info("Sending procedure completed");
93109
cutoff_start_ = std::chrono::system_clock::now();
@@ -154,7 +170,7 @@ namespace fc::mining {
154170

155171
const auto &sn = sector_info.sector_number;
156172
batch_storage_[sn] = PreCommitEntry(deposit, precommit_info);
157-
callbacks_[sn] = callback; // TODO: batcher upper limit
173+
callbacks_[sn] = callback; // TODO(Elestrias): batcher upper limit
158174
setPreCommitCutoff(head->epoch(), sector_info);
159175
return outcome::success();
160176
}

0 commit comments

Comments
 (0)