Activate improved compression.

Author: Mustang98

validator-session/candidate-serializer.cpp

@@ -28,10 +28,9 @@ td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::val
   if (!compression_enabled) {
     return serialize_tl_object(block, true);
   }
-  size_t decompressed_size;
-  TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size))
-  return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
-      0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed));
+  TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_))
+  return create_serialize_tl_object<ton_api::validatorSession_compressedCandidateV2>(
+      0, block->src_, block->round_, block->root_hash_, std::move(compressed));
 }
 
 td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
@@ -72,8 +71,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
   return res;
 }
 
-td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data,
-                                                    size_t& decompressed_size) {
+td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data) {
   vm::BagOfCells boc1, boc2;
   TRY_STATUS(boc1.deserialize(block));
   if (boc1.get_root_count() != 1) {
@@ -84,9 +82,7 @@ td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice c
   for (int i = 0; i < boc2.get_root_count(); ++i) {
     roots.push_back(boc2.get_root_cell(i));
   }
-  TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
-  decompressed_size = data.size();
-  td::BufferSlice compressed = td::lz4_compress(data);
+  TRY_RESULT(compressed, vm::boc_compress(roots, vm::CompressionAlgorithm::ImprovedStructureLZ4));
   LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size();
   return compressed;
 }

validator-session/candidate-serializer.h

@@ -27,8 +27,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
                                                                                      int max_decompressed_data_size,
                                                                                      int proto_version);
 
-td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data,
-                                                    size_t& decompressed_size);
+td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data);
 td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(td::Slice compressed,
                                                                                   bool improved_compression,
                                                                                   int decompressed_size,

validator/collator-node/utils.cpp

@@ -28,12 +28,10 @@ tl_object_ptr<ton_api::collatorNode_Candidate> serialize_candidate(const BlockCa
         PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(),
         block.collated_data.clone());
   }
-  size_t decompressed_size;
   td::BufferSlice compressed =
-      validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok();
-  return create_tl_object<ton_api::collatorNode_compressedCandidate>(
-      0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id),
-      (int)decompressed_size, std::move(compressed));
+      validatorsession::compress_candidate_data(block.data, block.collated_data).move_as_ok();
+  return create_tl_object<ton_api::collatorNode_compressedCandidateV2>(
+      0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), std::move(compressed));
 }
 
 td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collatorNode_Candidate> f,

validator/full-node-serializer.cpp

@@ -40,16 +40,13 @@ td::Result<td::BufferSlice> serialize_block_broadcast(const BlockBroadcast& broa
 
   TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof));
   TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data));
-  TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
-  td::BufferSlice data =
-      create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed_data>(std::move(sigs), std::move(boc));
-  td::BufferSlice compressed = td::lz4_compress(data);
+  TRY_RESULT(compressed_boc, vm::boc_compress({proof_root, data_root}, vm::CompressionAlgorithm::ImprovedStructureLZ4));
   VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: "
                         << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> "
-                        << compressed.size();
-  return create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed>(
-      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0,
-      std::move(compressed));
+                        << compressed_boc.size() + broadcast.signatures.size() * 96;
+  return create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressedV2>(
+      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
+      0, std::move(compressed_boc));
 }
 
 static td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) {
@@ -134,10 +131,10 @@ td::Result<td::BufferSlice> serialize_block_full(const BlockIdExt& id, td::Slice
   }
   TRY_RESULT(proof_root, vm::std_boc_deserialize(proof));
   TRY_RESULT(data_root, vm::std_boc_deserialize(data));
-  TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
-  td::BufferSlice compressed = td::lz4_compress(boc);
+  TRY_RESULT(compressed, vm::boc_compress({proof_root, data_root}, vm::CompressionAlgorithm::ImprovedStructureLZ4));
+
   VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size();
-  return create_serialize_tl_object<ton_api::tonNode_dataFullCompressed>(create_tl_block_id(id), 0,
+  return create_serialize_tl_object<ton_api::tonNode_dataFullCompressedV2>(create_tl_block_id(id), 0,
                                                                          std::move(compressed), is_proof_link);
 }
 
@@ -204,10 +201,9 @@ td::Result<td::BufferSlice> serialize_block_candidate_broadcast(BlockIdExt block
         create_tl_object<ton_api::tonNode_blockSignature>(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data));
   }
   TRY_RESULT(root, vm::std_boc_deserialize(data));
-  TRY_RESULT(data_new, vm::std_boc_serialize(root, 2));
-  td::BufferSlice compressed = td::lz4_compress(data_new);
+  TRY_RESULT(compressed, vm::boc_compress({root}, vm::CompressionAlgorithm::ImprovedStructureLZ4));
   VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size();
-  return create_serialize_tl_object<ton_api::tonNode_newBlockCandidateBroadcastCompressed>(
+  return create_serialize_tl_object<ton_api::tonNode_newBlockCandidateBroadcastCompressedV2>(
       create_tl_block_id(block_id), cc_seqno, validator_set_hash,
       create_tl_object<ton_api::tonNode_blockSignature>(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed));
 }