Merge pull request #1709 from DanShaders/state-split

Support splitting persistent states

Author: EmelyanenkoK

create-hardfork/create-hardfork.cpp

@@ -258,8 +258,8 @@ class HardforkCreator : public td::actor::Actor {
                                td::Promise<td::BufferSlice> promise) override {
       }
       void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id,
-                                     td::uint32 priority, td::Timestamp timeout,
-                                     td::Promise<td::BufferSlice> promise) override {
+                                     ton::validator::PersistentStateType type, td::uint32 priority,
+                                     td::Timestamp timeout, td::Promise<td::BufferSlice> promise) override {
       }
       void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,
                                 td::Promise<td::BufferSlice> promise) override {

crypto/block/block.tlb

@@ -670,6 +670,7 @@ workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8)
   zerostate_root_hash:bits256 zerostate_file_hash:bits256
   version:uint32 format:(WorkchainFormat basic)
   split_merge_timings:WcSplitMergeTimings
+  persistent_state_split_depth:(## 8) { persistent_state_split_depth <= 63 }
   = WorkchainDescr;
 
 _ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12;

crypto/block/mc-config.cpp

@@ -2199,6 +2199,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) {
     split_merge_interval = rec.split_merge_interval;
     min_split_merge_interval = rec.min_split_merge_interval;
     max_split_merge_delay = rec.max_split_merge_delay;
+    persistent_state_split_depth = info.persistent_state_split_depth;
     return true;
   };
   block::gen::WorkchainDescr::Record_workchain info_v1;

crypto/block/mc-config.h

@@ -438,6 +438,8 @@ struct WorkchainInfo : public td::CntObject {
   unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds
   unsigned max_split_merge_delay = 1000;  // end of split/merge interval must be at most 1000 seconds in the future
 
+  td::uint32 persistent_state_split_depth = 0;
+
   bool is_valid() const {
     return workchain != ton::workchainInvalid;
   }

crypto/vm/cells/MerkleProof.cpp

@@ -17,6 +17,7 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "vm/cells/MerkleProof.h"
+#include "td/utils/Status.h"
 #include "vm/cells/CellBuilder.h"
 #include "vm/cells/CellSlice.h"
 #include "vm/boc.h"
@@ -152,6 +153,11 @@ Ref<Cell> MerkleProof::virtualize(Ref<Cell> cell, int virtualization) {
   return virtualize_raw(r_raw.move_as_ok(), {0 /*level*/, static_cast<td::uint8>(virtualization)});
 }
 
+td::Result<Ref<Cell>> MerkleProof::try_virtualize(Ref<Cell> cell, int virtualization) {
+  TRY_RESULT(unpacked_cell, unpack_proof(std::move(cell)));
+  return unpacked_cell->virtualize({0 /*level*/, static_cast<td::uint8>(virtualization)});
+}
+
 class MerkleProofCombineFast {
  public:
   MerkleProofCombineFast(Ref<Cell> a, Ref<Cell> b) : a_(std::move(a)), b_(std::move(b)) {

crypto/vm/cells/MerkleProof.h

@@ -36,6 +36,7 @@ class MerkleProof {
 
   // cell must have zero level and must be a MerkleProof
   static Ref<Cell> virtualize(Ref<Cell> cell, int virtualization);
+  static td::Result<Ref<Cell>> try_virtualize(Ref<Cell> cell, int virtualization = 1);
 
   static Ref<Cell> combine(Ref<Cell> a, Ref<Cell> b);
   static td::Result<Ref<Cell>> combine_status(Ref<Cell> a, Ref<Cell> b);

crypto/vm/dict.cpp

@@ -2790,6 +2790,12 @@ Ref<CellSlice> AugmentedDictionary::get_root() const {
   return root;
 }
 
+Ref<Cell> AugmentedDictionary::get_wrapped_dict_root() const {
+  vm::CellBuilder cb;
+  cb.append_cellslice(get_root());
+  return cb.finalize();
+}
+
 Ref<CellSlice> AugmentedDictionary::extract_root() && {
   if (!(flags & f_root_cached) && !compute_root()) {
     return {};

crypto/vm/dict.h

@@ -572,6 +572,7 @@ class AugmentedDictionary final : public DictionaryFixed {
   AugmentedDictionary(DictNonEmpty, Ref<CellSlice> _root, int _n, const AugmentationData& _aug, bool validate = true);
   Ref<CellSlice> get_empty_dictionary() const;
   Ref<CellSlice> get_root() const;
+  Ref<Cell> get_wrapped_dict_root() const;
   Ref<CellSlice> extract_root() &&;
   bool append_dict_to_bool(CellBuilder& cb) &&;
   bool append_dict_to_bool(CellBuilder& cb) const &;

test/test-ton-collator.cpp

@@ -359,8 +359,8 @@ class TestNode : public td::actor::Actor {
                                td::Promise<td::BufferSlice> promise) override {
       }
       void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id,
-                                     td::uint32 priority, td::Timestamp timeout,
-                                     td::Promise<td::BufferSlice> promise) override {
+                                     ton::validator::PersistentStateType type, td::uint32 priority,
+                                     td::Timestamp timeout, td::Promise<td::BufferSlice> promise) override {
       }
       void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,
                                 td::Promise<td::BufferSlice> promise) override {

tl/generate/scheme/ton_api.tl

@@ -409,7 +409,9 @@ tonNode.preparedProof = tonNode.PreparedProof;
 tonNode.preparedProofLink = tonNode.PreparedProof;
 tonNode.preparedState = tonNode.PreparedState;
 tonNode.notFoundState = tonNode.PreparedState;
+tonNode.persistentStateIdV2 block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt effective_shard:long = tonNode.PersistentStateIdV2;
 tonNode.persistentStateSize size:long = tonNode.PersistentStateSize;
+tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize;
 tonNode.prepared = tonNode.Prepared;
 tonNode.notFound = tonNode.Prepared;
 tonNode.data data:bytes = tonNode.Data;
@@ -475,15 +477,11 @@ tonNode.prepareBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool
 tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool = tonNode.PreparedProof;
 tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared;
 tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared;
-tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState;
-tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize;
 tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState;
 tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks;
 tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull;
 tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull;
 tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data;
 tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data;
@@ -495,6 +493,14 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
 tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt)
     limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof;
 
+tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data;
+tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data;
+tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize;
+tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState;
+
+tonNode.downloadPersistentStateSliceV2 state:tonNode.persistentStateIdV2 offset:long max_size:long = tonNode.Data;
+tonNode.getPersistentStateSizeV2 state:tonNode.persistentStateIdV2 = tonNode.PersistentStateSize;
+
 tonNode.getCapabilities = tonNode.Capabilities;
 
 tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success;
@@ -542,7 +548,9 @@ db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:i
 db.filedb.key.empty = db.filedb.Key;
 db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; 
 db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; 
-db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; 
+db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key;
+db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key;
+db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key;
@@ -560,6 +568,8 @@ db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_t
 db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks;
 db.state.dbVersion version:int = db.state.DbVersion;
 db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards;
+db.state.persistentStateDescriptionShard block:tonNode.blockIdExt split_depth:int = db.state.PersistentStateDescriptionShard;
+db.state.persistentStateDescriptionShardsV2 shard_blocks:(vector db.state.PersistentStateDescriptionShard) = db.state.PersistentStateDescriptionShards;
 db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader;
 db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList;