Skip to content

Compression depth balance #1892

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: testnet
Choose a base branch
from
+378 −88
Open

Compression depth balance #1892

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a5152ce
Fix depth balance decompression
Mustang98 Nov 17, 2025
76ababb
[compression-depth-balance] Bug fix and refactor depth balance decomp…
Mustang98 Nov 17, 2025
88a23bc
[compression-depth-balance] Refactor balance parsing, change CMakeLis…
Mustang98 Nov 20, 2025
32189f1
[compression-depth-balance] Optimize decompression
Mustang98 Nov 20, 2025
eaba927
[compression-depth-balance] Add benchmark logs for testing on devnet
Mustang98 Nov 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crypto/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ set(TON_CRYPTO_CORE_SOURCE
openssl/residue.cpp
openssl/rand.cpp
vm/boc.cpp
vm/boc-compression.cpp
vm/large-boc-serializer.cpp
tl/tlblib.cpp

Expand Down Expand Up @@ -82,6 +81,7 @@ set(TON_CRYPTO_CORE_SOURCE
ellcurve/p256.cpp)

set(TON_CRYPTO_SOURCE
vm/boc-compression.cpp
vm/stack.cpp
vm/atom.cpp
vm/continuation.cpp
Expand Down
273 changes: 240 additions & 33 deletions crypto/vm/boc-compression.cpp
View file
Open in desktop

Large diffs are not rendered by default.

35 changes: 28 additions & 7 deletions validator-session/candidate-serializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,16 @@ namespace ton::validatorsession {
td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate>& block,
bool compression_enabled) {
if (!compression_enabled) {
return serialize_tl_object(block, true);
LOG(INFO) << "COMPR_BENCHMARK serialize_candidate START_COMPRESS block_id=" << block->root_hash_.to_hex();
auto res = serialize_tl_object(block, true);
LOG(INFO) << "COMPR_BENCHMARK serialize_candidate END_COMPRESS block_id=" << block->root_hash_.to_hex()
<< " compression_enabled=" << compression_enabled
<< " data_size_bytes=" << block->data_.size() + block->collated_data_.size()
<< " res_size=" << block->data_.size() + block->collated_data_.size();
return res;
}
size_t decompressed_size;
TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size))
TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size, block->root_hash_.to_hex()))
return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed));
}
Expand All @@ -40,7 +46,12 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
int max_decompressed_data_size,
int proto_version) {
if (!compression_enabled) {
return fetch_tl_object<ton_api::validatorSession_candidate>(data, true);
TRY_RESULT(res, fetch_tl_object<ton_api::validatorSession_candidate>(data, true));
LOG(INFO) << "COMPR_BENCHMARK deserialize_candidate START_DECOMPRESS block_id=" << res->root_hash_.to_hex();
LOG(INFO) << "COMPR_BENCHMARK deserialize_candidate END_DECOMPRESS block_id=" << res->root_hash_.to_hex()
<< " compression_enabled=" << compression_enabled
<< " received_size=" << res->data_.size() + res->collated_data_.size();
return std::move(res);
}
TRY_RESULT(f, fetch_tl_object<ton_api::validatorSession_Candidate>(data, true));
td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> res;
Expand All @@ -55,7 +66,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
return td::Status::Error("decompressed size is too big");
}
TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_,
max_decompressed_data_size, proto_version));
max_decompressed_data_size, proto_version, c.root_hash_.to_hex()));
return create_tl_object<ton_api::validatorSession_candidate>(c.src_, c.round_, c.root_hash_,
std::move(p.first), std::move(p.second));
}();
Expand All @@ -65,7 +76,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
if (c.data_.size() > max_decompressed_data_size) {
return td::Status::Error("Compressed data is too big");
}
TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version));
TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version, c.root_hash_.to_hex()));
return create_tl_object<ton_api::validatorSession_candidate>(c.src_, c.round_, c.root_hash_,
std::move(p.first), std::move(p.second));
}();
Expand All @@ -74,7 +85,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
}

td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data,
size_t& decompressed_size) {
size_t& decompressed_size, std::string root_hash) {
vm::BagOfCells boc1, boc2;
TRY_STATUS(boc1.deserialize(block));
if (boc1.get_root_count() != 1) {
Expand All @@ -85,25 +96,35 @@ td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice c
for (int i = 0; i < boc2.get_root_count(); ++i) {
roots.push_back(boc2.get_root_cell(i));
}
LOG(INFO) << "COMPR_BENCHMARK compress_candidate_data START_COMPRESS block_id=" << root_hash;
TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
decompressed_size = data.size();
td::BufferSlice compressed = td::lz4_compress(data);
LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size();
LOG(INFO) << "COMPR_BENCHMARK compress_candidate_data END_COMPRESS block_id=" << root_hash
<< " compression_enabled=" << true
<< " data_size_bytes=" << block.size() + collated_data.size()
<< " res_size=" << compressed.size();
return compressed;
}

td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(td::Slice compressed,
bool improved_compression,
int decompressed_size,
int max_decompressed_size,
int proto_version) {
int proto_version,
std::string root_hash) {
std::vector<td::Ref<vm::Cell>> roots;
if (!improved_compression) {
LOG(INFO) << "COMPR_BENCHMARK decompress_candidate_data START_DECOMPRESS block_id=" << root_hash;
TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size));
if (decompressed.size() != (size_t)decompressed_size) {
return td::Status::Error("decompressed size mismatch");
}
TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed));
LOG(INFO) << "COMPR_BENCHMARK decompress_candidate_data END_DECOMPRESS block_id=" << root_hash
<< " compression_enabled=" << true
<< " received_size=" << compressed.size();
} else {
TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size));
}
Expand Down
5 changes: 3 additions & 2 deletions validator-session/candidate-serializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
int proto_version);

td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data,
size_t& decompressed_size);
size_t& decompressed_size, std::string root_hash);
td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(td::Slice compressed,
bool improved_compression,
int decompressed_size,
int max_decompressed_size,
int proto_version);
int proto_version,
std::string root_hash);

} // namespace ton::validatorsession
8 changes: 5 additions & 3 deletions validator/collator-node/utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ tl_object_ptr<ton_api::collatorNode_Candidate> serialize_candidate(const BlockCa
}
size_t decompressed_size;
td::BufferSlice compressed =
validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok();
validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size, block.id.to_str()).move_as_ok();
return create_tl_object<ton_api::collatorNode_compressedCandidate>(
0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id),
(int)decompressed_size, std::move(compressed));
Expand Down Expand Up @@ -63,7 +63,8 @@ td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collator
return td::Status::Error("decompressed size is too big");
}
TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_,
max_decompressed_data_size, proto_version));
max_decompressed_data_size, proto_version,
create_block_id(c.id_).to_str()));
auto collated_data_hash = td::sha256_bits256(p.second);
auto key = PublicKey{c.source_};
if (!key.is_ed25519()) {
Expand All @@ -77,7 +78,8 @@ td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collator
[&](ton_api::collatorNode_compressedCandidateV2& c) {
res = [&]() -> td::Result<BlockCandidate> {
TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0,
max_decompressed_data_size, proto_version));
max_decompressed_data_size, proto_version,
create_block_id(c.id_).to_str()));
auto collated_data_hash = td::sha256_bits256(p.second);
auto key = PublicKey{c.source_};
if (!key.is_ed25519()) {
Expand Down
8 changes: 4 additions & 4 deletions validator/full-node-fast-sync-overlays.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN
}

void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), "fast-sync");
if (B.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
return;
Expand Down Expand Up @@ -109,7 +109,7 @@ void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash sr
td::uint32 validator_set_hash;
td::BufferSlice data;
auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data,
overlay::Overlays::max_fec_broadcast_size());
overlay::Overlays::max_fec_broadcast_size(), "fast-sync");
if (S.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << S;
return;
Expand Down Expand Up @@ -191,7 +191,7 @@ void FullNodeFastSyncOverlay::send_broadcast(BlockBroadcast broadcast) {
}
VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in fast sync overlay (with compression): "
<< broadcast.block_id.to_str();
auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true
auto B = serialize_block_broadcast(broadcast, true, "fast-sync"); // compression_enabled = true
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
return;
Expand All @@ -206,7 +206,7 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain
return;
}
auto B =
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, "fast-sync"); // compression enabled
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error();
return;
Expand Down
16 changes: 8 additions & 8 deletions validator/full-node-private-overlay.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src,
}

void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), "private");
if (B.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
return;
Expand Down Expand Up @@ -81,7 +81,7 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas
td::uint32 validator_set_hash;
td::BufferSlice data;
auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data,
overlay::Overlays::max_fec_broadcast_size());
overlay::Overlays::max_fec_broadcast_size(), "private");
if (S.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << S;
return;
Expand Down Expand Up @@ -166,7 +166,7 @@ void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, Catc
return;
}
auto B =
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, "private"); // compression enabled
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error();
return;
Expand All @@ -182,7 +182,7 @@ void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) {
}
VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay"
<< (enable_compression_ ? " (with compression)" : "") << ": " << broadcast.block_id.to_str();
auto B = serialize_block_broadcast(broadcast, enable_compression_);
auto B = serialize_block_broadcast(broadcast, enable_compression_, "private");
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
return;
Expand Down Expand Up @@ -323,7 +323,7 @@ void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::
<< src;
return;
}
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), "custom");
if (B.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
return;
Expand Down Expand Up @@ -371,7 +371,7 @@ void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src,
td::uint32 validator_set_hash;
td::BufferSlice data;
auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data,
overlay::Overlays::max_fec_broadcast_size());
overlay::Overlays::max_fec_broadcast_size(), "custom");
if (S.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << S;
return;
Expand Down Expand Up @@ -423,7 +423,7 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) {
}
VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_
<< "\": " << broadcast.block_id.to_str();
auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true
auto B = serialize_block_broadcast(broadcast, true, "custom"); // compression_enabled = true
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
return;
Expand All @@ -438,7 +438,7 @@ void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSe
return;
}
auto B =
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled
serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, "custom"); // compression enabled
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error();
return;
Expand Down
Loading