Skip to content

Commit 7fefaa9

Browse files
DanShadersDanShaders
committed
Add initial support for split persistent states in ArchiveManager
For now, we only make implementation ready for them without changing public API in any way. In particular, it is not possible to create a split state or request it.
1 parent 774371b commit 7fefaa9

File tree

7 files changed

+223
-20
lines changed

7 files changed

+223
-20
lines changed

‎tl/generate/scheme/ton_api.tl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -542,7 +542,9 @@ db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:i
542542
db.filedb.key.empty = db.filedb.Key;
543543
db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key;
544544
db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key;
545-
db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key;
545+
db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key;
546+
db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key;
547+
db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key;
546548
db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key;
547549
db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key;
548550
db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key;

‎tl/generate/scheme/ton_api.tlo‎

364 Bytes
Binary file not shown.

‎validator/db/archive-manager.cpp‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -311,9 +311,6 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td:
311311
}
312312

313313
void ArchiveManager::register_perm_state(FileReferenceShort id) {
314-
BlockSeqno masterchain_seqno = 0;
315-
id.ref().visit(td::overloaded(
316-
[&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {}));
317314
td::uint64 size;
318315
auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short());
319316
if (r_stat.is_error()) {
@@ -322,7 +319,7 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) {
322319
} else {
323320
size = r_stat.ok().size_;
324321
}
325-
perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size};
322+
perm_states_[{id.seqno_of_persistent_state(), id.hash()}] = {.id = id, .size = size};
326323
}
327324

328325
void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
@@ -347,14 +344,33 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
347344
.release();
348345
}
349346

347+
namespace {
348+
349+
FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc_block_id, PersistentStateType type) {
350+
FileReferenceShort result;
351+
type.visit(td::overloaded(
352+
[&](UnsplitStateType const &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); },
353+
[&](SplitAccountStateType const &account_state) {
354+
result = fileref::SplitAccountState::create(block_id, mc_block_id, account_state.effective_shard_id);
355+
},
356+
[&](SplitPersistentStateType const &persistent_state) {
357+
result = fileref::SplitPersistentState::create(block_id, mc_block_id);
358+
}));
359+
return result;
360+
}
361+
362+
} // namespace
363+
350364
void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
351365
td::Promise<td::Unit> promise) {
352366
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
353367
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data),
354368
std::move(P))
355369
.release();
356370
};
357-
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
371+
// TODO: Allow specifying state type.
372+
add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}),
373+
std::move(promise), std::move(create_writer));
358374
}
359375

360376
void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
@@ -365,23 +381,22 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma
365381
std::move(write_state), std::move(P))
366382
.release();
367383
};
368-
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
384+
// TODO: Allow specifying state type.
385+
add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}),
386+
std::move(promise), std::move(create_writer));
369387
}
370388

371389
void ArchiveManager::add_persistent_state_impl(
372-
BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
390+
FileReferenceShort const &id, td::Promise<td::Unit> promise,
373391
std::function<void(std::string, td::Promise<std::string>)> create_writer) {
374-
auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
375-
BlockSeqno masterchain_seqno = masterchain_block_id.seqno();
376-
auto hash = id.hash();
377-
if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) {
392+
if (perm_states_.find({id.seqno_of_persistent_state(), id.hash()}) != perm_states_.end()) {
378393
promise.set_value(td::Unit());
379394
return;
380395
}
381396

382397
auto path = db_root_ + "/archive/states/" + id.filename_short();
383398
auto P = td::PromiseCreator::lambda(
384-
[SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result<std::string> R) mutable {
399+
[SelfId = actor_id(this), id = id, promise = std::move(promise)](td::Result<std::string> R) mutable {
385400
if (R.is_error()) {
386401
promise.set_error(R.move_as_error());
387402
} else {
@@ -436,7 +451,8 @@ void ArchiveManager::get_previous_persistent_state_files(
436451

437452
void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
438453
td::Promise<td::BufferSlice> promise) {
439-
auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
454+
// TODO: Allow specifying state type.
455+
auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{});
440456
auto hash = id.hash();
441457
if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) {
442458
promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
@@ -449,7 +465,8 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master
449465

450466
void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
451467
td::int64 max_size, td::Promise<td::BufferSlice> promise) {
452-
auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
468+
// TODO: Allow specifying state type.
469+
auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{});
453470
auto hash = id.hash();
454471
if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) {
455472
promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
@@ -462,7 +479,8 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt
462479

463480
void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id,
464481
td::Promise<td::uint64> promise) {
465-
auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
482+
// TODO: Allow specifying state type.
483+
auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{});
466484
auto hash = id.hash();
467485
auto it = perm_states_.find({masterchain_block_id.seqno(), hash});
468486
if (it == perm_states_.end()) {
@@ -918,6 +936,11 @@ void ArchiveManager::start_up() {
918936
R = FileReferenceShort::create(newfname);
919937
R.ensure();
920938
}
939+
if (!R.ok().is_state_like()) {
940+
LOG(ERROR) << "deleting file that is not state-like '" << fname << "'";
941+
td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore();
942+
return;
943+
}
921944
register_perm_state(R.move_as_ok());
922945
}
923946
}).ensure();
@@ -1037,6 +1060,14 @@ void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
10371060
res = 0;
10381061
seqno = x.masterchain_seqno;
10391062
},
1063+
[&](const fileref::SplitAccountState &x) {
1064+
res = 0;
1065+
seqno = x.masterchain_seqno;
1066+
},
1067+
[&](const fileref::SplitPersistentState &x) {
1068+
res = 0;
1069+
seqno = x.masterchain_seqno;
1070+
},
10401071
[&](const auto &obj) { res = -1; }));
10411072

10421073
if (res == -1) {
@@ -1315,6 +1346,8 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han
13151346
it->second.id.ref().visit(td::overloaded(
13161347
[&](const fileref::ZeroStateShort &x) { res = -1; },
13171348
[&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; },
1349+
[&](const fileref::SplitPersistentState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; },
1350+
[&](const fileref::SplitAccountState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; },
13181351
[&](const auto &obj) { res = 1; }));
13191352
if (res <= 0) {
13201353
it++;

‎validator/db/archive-manager.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#pragma once
2020

2121
#include "archive-slice.hpp"
22+
#include "interfaces/persistent-state.h"
2223

2324
namespace ton {
2425

@@ -219,7 +220,7 @@ class ArchiveManager : public td::actor::Actor {
219220
PackageId get_max_temp_file_desc_idx();
220221
PackageId get_prev_temp_file_desc_idx(PackageId id);
221222

222-
void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
223+
void add_persistent_state_impl(FileReferenceShort const &id, td::Promise<td::Unit> promise,
223224
std::function<void(std::string, td::Promise<std::string>)> create_writer);
224225
void register_perm_state(FileReferenceShort id);
225226

‎validator/db/fileref.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,16 @@ std::string ZeroStateShort::filename_short() const {
106106
return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex();
107107
}
108108

109+
std::string SplitAccountState::filename_short() const {
110+
return PSTRING() << "stateaccount_" << masterchain_seqno << "_" << shard_id.workchain << "_"
111+
<< shard_to_str(shard_id.shard) << "_" << shard_to_str(effective_shard) << "_" << hash().to_hex();
112+
}
113+
114+
std::string SplitPersistentState::filename_short() const {
115+
return PSTRING() << "statesplit_" << masterchain_seqno << "_" << shard_id.workchain << "_"
116+
<< shard_to_str(shard_id.shard) << "_" << hash().to_hex();
117+
}
118+
109119
PersistentStateShort PersistentState::shortref() const {
110120
return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()};
111121
}
@@ -288,6 +298,24 @@ ShardIdFull FileReferenceShort::shard() const {
288298
return h;
289299
}
290300

301+
BlockSeqno FileReferenceShort::seqno_of_persistent_state() const {
302+
BlockSeqno result;
303+
ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = x.masterchain_seqno; },
304+
[&](const fileref::SplitAccountState& x) { result = x.masterchain_seqno; },
305+
[&](const fileref::SplitPersistentState& x) { result = x.masterchain_seqno; },
306+
[&](const fileref::ZeroStateShort) { result = 0; }, [&](const auto&) { CHECK(false); }));
307+
return result;
308+
}
309+
310+
bool FileReferenceShort::is_state_like() const {
311+
bool result = false;
312+
ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = true; },
313+
[&](const fileref::SplitAccountState& x) { result = true; },
314+
[&](const fileref::SplitPersistentState& x) { result = true; },
315+
[&](const fileref::ZeroStateShort) { result = true; }, [&](const auto&) {}));
316+
return result;
317+
}
318+
291319
std::string FileReference::filename() const {
292320
std::string h;
293321
ref_.visit([&](const auto& obj) { h = obj.filename(); });
@@ -420,6 +448,35 @@ td::Result<FileReferenceShort> FileReferenceShort::create(std::string filename)
420448
} else {
421449
return td::Status::Error(ErrorCode::protoviolation, "too big file name");
422450
}
451+
} else if (token == "stateaccount") {
452+
std::getline(ss, token, '_');
453+
TRY_RESULT(masterchain_seqno, td::to_integer_safe<BlockSeqno>(token));
454+
std::getline(ss, token, '_');
455+
TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
456+
std::getline(ss, token, '_');
457+
TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
458+
std::getline(ss, token, '_');
459+
TRY_RESULT(effective_shard, td::hex_to_integer_safe<ShardId>(token));
460+
TRY_RESULT(vhash, get_token_hash(ss));
461+
if (!ss.eof()) {
462+
return td::Status::Error(ErrorCode::protoviolation, "too big file name");
463+
}
464+
if (!shard_is_proper_ancestor(shard, effective_shard)) {
465+
return td::Status::Error(ErrorCode::protoviolation, "shard is not a proper ancestor of effective shard");
466+
}
467+
return fileref::SplitAccountState{ShardIdFull{workchain, shard}, effective_shard, masterchain_seqno, vhash};
468+
} else if (token == "statesplit") {
469+
std::getline(ss, token, '_');
470+
TRY_RESULT(masterchain_seqno, td::to_integer_safe<BlockSeqno>(token));
471+
std::getline(ss, token, '_');
472+
TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
473+
std::getline(ss, token, '_');
474+
TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
475+
TRY_RESULT(vhash, get_token_hash(ss));
476+
if (!ss.eof()) {
477+
return td::Status::Error(ErrorCode::protoviolation, "too big file name");
478+
}
479+
return fileref::SplitPersistentState{ShardIdFull{workchain, shard}, masterchain_seqno, vhash};
423480
} else if (token == "state") {
424481
std::getline(ss, token, '_');
425482
TRY_RESULT(masterchain_seqno, td::to_integer_safe<BlockSeqno>(token));

‎validator/db/fileref.hpp‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,77 @@ class ZeroState {
110110
BlockIdExt block_id;
111111
};
112112

113+
class SplitAccountState {
114+
public:
115+
static SplitAccountState create(BlockIdExt block_id, BlockIdExt masterchain_block_id, ShardId effective_shard) {
116+
auto hash = create_hash_tl_object<ton_api::db_filedb_key_splitAccountStateFile>(
117+
create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id), effective_shard);
118+
119+
return {
120+
.shard_id = block_id.shard_full(),
121+
.effective_shard = effective_shard,
122+
.masterchain_seqno = masterchain_block_id.seqno(),
123+
.hashv = hash,
124+
};
125+
}
126+
127+
FileHash hash() const {
128+
return hashv;
129+
}
130+
131+
ShardIdFull shard() const {
132+
return {shard_id.workchain, effective_shard};
133+
}
134+
135+
std::string filename_short() const;
136+
137+
ShardIdFull shard_id;
138+
ShardId effective_shard;
139+
BlockSeqno masterchain_seqno;
140+
FileHash hashv;
141+
};
142+
143+
class SplitPersistentState {
144+
public:
145+
static SplitPersistentState create(BlockIdExt block_id, BlockIdExt masterchain_block_id) {
146+
auto hash = create_hash_tl_object<ton_api::db_filedb_key_splitPersistentStateFile>(
147+
create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id));
148+
149+
return {
150+
.shard_id = block_id.shard_full(),
151+
.masterchain_seqno = masterchain_block_id.seqno(),
152+
.hashv = hash,
153+
};
154+
}
155+
156+
FileHash hash() const {
157+
return hashv;
158+
}
159+
160+
ShardIdFull shard() const {
161+
return shard_id;
162+
}
163+
164+
std::string filename_short() const;
165+
166+
ShardIdFull shard_id;
167+
BlockSeqno masterchain_seqno;
168+
FileHash hashv;
169+
};
170+
113171
class PersistentStateShort {
114172
public:
173+
static PersistentStateShort create(BlockIdExt block_id, BlockIdExt masterchain_block_id) {
174+
auto hash = create_hash_tl_object<ton_api::db_filedb_key_persistentStateFile>(
175+
create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id));
176+
177+
return {
178+
.shard_id = block_id.shard_full(),
179+
.masterchain_seqno = masterchain_block_id.seqno(),
180+
.hashv = hash,
181+
};
182+
}
183+
115184
FileHash hash() const {
116185
return hashv;
117186
}
@@ -346,9 +415,10 @@ class BlockInfo {
346415

347416
class FileReferenceShort {
348417
private:
349-
td::Variant<fileref::Empty, fileref::BlockShort, fileref::ZeroStateShort, fileref::PersistentStateShort,
350-
fileref::ProofShort, fileref::ProofShort, fileref::ProofLinkShort, fileref::SignaturesShort,
351-
fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort>
418+
td::Variant<fileref::Empty, fileref::BlockShort, fileref::ZeroStateShort, fileref::SplitAccountState,
419+
fileref::SplitPersistentState, fileref::PersistentStateShort, fileref::ProofShort, fileref::ProofShort,
420+
fileref::ProofLinkShort, fileref::SignaturesShort, fileref::CandidateShort, fileref::CandidateRefShort,
421+
fileref::BlockInfoShort>
352422
ref_;
353423

354424
public:
@@ -366,6 +436,8 @@ class FileReferenceShort {
366436

367437
td::Bits256 hash() const;
368438
ShardIdFull shard() const;
439+
BlockSeqno seqno_of_persistent_state() const;
440+
bool is_state_like() const;
369441
std::string filename_short() const;
370442
};
371443

‎validator/interfaces/persistent-state.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
This file is part of TON Blockchain Library.
3+
4+
TON Blockchain Library is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU Lesser General Public License as published by
6+
the Free Software Foundation, either version 2 of the License, or
7+
(at your option) any later version.
8+
9+
TON Blockchain Library is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU Lesser General Public License for more details.
13+
14+
You should have received a copy of the GNU Lesser General Public License
15+
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
#pragma once
18+
19+
#include "td/utils/Variant.h"
20+
#include "ton/ton-types.h"
21+
22+
namespace ton {
23+
24+
namespace validator {
25+
26+
struct UnsplitStateType {};
27+
28+
struct SplitAccountStateType {
29+
ShardId effective_shard_id;
30+
};
31+
32+
struct SplitPersistentStateType {};
33+
34+
using PersistentStateType = td::Variant<UnsplitStateType, SplitAccountStateType, SplitPersistentStateType>;
35+
36+
} // namespace validator
37+
38+
} // namespace ton

0 commit comments

Comments
 (0)