Activate improved compression #1948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

Mustang98 wants to merge 57 commits into ton-blockchain:testnet from Mustang98:activate-improved-compression

crypto/vm/boc-compression.cpp

Large diffs are not rendered by default.

crypto/vm/boc-compression.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -27,18 +27,32 @@ This file is part of TON Blockchain Library. @@
     namespace vm {
     constexpr size_t kDecompressedSizeBytes = 4;
+    constexpr size_t kMUCellOrderInRoot = 2;
-    enum class CompressionAlgorithm : int { BaselineLZ4 = 0, ImprovedStructureLZ4 = 1 };
+    enum class CompressionAlgorithm : int {
+      BaselineLZ4 = 0,
+      ImprovedStructureLZ4 = 1,
+      ImprovedStructureLZ4WithState = 2,
+    };
     td::Result<td::BufferSlice> boc_compress_baseline_lz4(const std::vector<td::Ref<vm::Cell>>& boc_roots);
     td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size);
-    td::Result<td::BufferSlice> boc_compress_improved_structure_lz4(const std::vector<td::Ref<vm::Cell>>& boc_roots);
-    td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress_improved_structure_lz4(td::Slice compressed,
-                                                                                     int max_decompressed_size);
+    td::Result<td::BufferSlice> boc_compress_improved_structure_lz4(const std::vector<td::Ref<vm::Cell>>& boc_roots,
+                                                                    bool compress_merkle_update = false,
+                                                                    td::Ref<vm::Cell> state = td::Ref<vm::Cell>());
+    td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress_improved_structure_lz4(
+        td::Slice compressed, int max_decompressed_size, bool decompress_merkle_update = false,
+        td::Ref<vm::Cell> state = td::Ref<vm::Cell>());
     td::Result<td::BufferSlice> boc_compress(const std::vector<td::Ref<vm::Cell>>& boc_roots,
-                                             CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4);
-    td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress(td::Slice compressed, int max_decompressed_size);
+                                             CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4,
+                                             td::Ref<vm::Cell> state = td::Ref<vm::Cell>());
+    td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress(td::Slice compressed, int max_decompressed_size,
+                                                              td::Ref<vm::Cell> state = td::Ref<vm::Cell>());
+    td::Result<bool> boc_need_state_for_decompression(const td::Slice& compressed);
+    td::Result<std::string> boc_get_algorithm_name(const td::Slice& compressed);
     }  // namespace vm

tl/generate/scheme/ton_api.tl

-Original file line number
+Diff line change
@@ Expand Up @@
     tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull;
     tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull;
-    tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull;
+    tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# proof:bytes block_compressed:bytes is_link:Bool = tonNode.DataFull;
     tonNode.dataFullEmpty = tonNode.DataFull;
     tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities;
@@ Expand Down @@

validator-session/candidate-serializer.cpp

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -25,24 +25,28 @@
  
    namespace ton::validatorsession {

    namespace {

    constexpr const char* k_called_from_validator_session = "validator_session";

    }  // namespace

    td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate>& block,

                                                    bool compression_enabled) {

      if (!compression_enabled) {

        auto t_compression_start = td::Time::now();

        auto res = serialize_tl_object(block, true);

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark serialize_candidate block_id="

                                          << block->root_hash_.to_hex()

                                          << " called_from=" << k_called_from_validator_session

                                          << " time_sec=" << (td::Time::now() - t_compression_start)

                                          << " compression=" << "none"

                                          << " original_size=" << block->data_.size() + block->collated_data_.size()

                                          << " compressed_size=" << block->data_.size() + block->collated_data_.size();

        return res;

      }

      size_t decompressed_size;

      TRY_RESULT(compressed,

                 compress_candidate_data(block->data_, block->collated_data_, decompressed_size, block->root_hash_))

      return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(

          0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed));

      TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, k_called_from_validator_session,

                                                     block->root_hash_))

      return create_serialize_tl_object<ton_api::validatorSession_compressedCandidateV2>(

          0, block->src_, block->round_, block->root_hash_, std::move(compressed));

    }

    td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,

    @@ -52,7 +56,7 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
  
        auto t_decompression_start = td::Time::now();

        TRY_RESULT(res, fetch_tl_object<ton_api::validatorSession_candidate>(data, true));

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark deserialize_candidate block_id="

                                          << res->root_hash_.to_hex()

                                          << res->root_hash_.to_hex() << " called_from=" << k_called_from_validator_session

                                          << " time_sec=" << (td::Time::now() - t_decompression_start)

                                          << " compression=" << "none"

                                          << " compressed_size=" << res->data_.size() + res->collated_data_.size();

    @@ -70,8 +74,9 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
  
                      if (c.decompressed_size_ > max_decompressed_data_size) {

                        return td::Status::Error("decompressed size is too big");

                      }

                      TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_,

                                                              max_decompressed_data_size, c.root_hash_));

                      TRY_RESULT(p,

                                 decompress_candidate_data(c.data_, false, c.decompressed_size_, max_decompressed_data_size,

                                                           k_called_from_validator_session, c.root_hash_));

                      return create_tl_object<ton_api::validatorSession_candidate>(c.src_, c.round_, c.root_hash_,

                                                                                   std::move(p.first), std::move(p.second));

                    }();

    @@ -81,15 +86,16 @@ td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candi
  
                      if (c.data_.size() > max_decompressed_data_size) {

                        return td::Status::Error("Compressed data is too big");

                      }

                      TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, c.root_hash_));

                      TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size,

                                                              k_called_from_validator_session, c.root_hash_));

                      return create_tl_object<ton_api::validatorSession_candidate>(c.src_, c.round_, c.root_hash_,

                                                                                   std::move(p.first), std::move(p.second));

                    }();

                  }));

      return res;

    }

    td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size,

    td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data, std::string called_from,

                                                        td::Bits256 root_hash) {

      vm::BagOfCells boc1, boc2;

      TRY_STATUS(boc1.deserialize(block));

    @@ -102,23 +108,21 @@ td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice c
  
        roots.push_back(boc2.get_root_cell(i));

      }

      auto t_compression_start = td::Time::now();

      TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));

      decompressed_size = data.size();

      td::BufferSlice compressed = td::lz4_compress(data);

      TRY_RESULT(compressed, vm::boc_compress(roots, vm::CompressionAlgorithm::ImprovedStructureLZ4));

      TRY_RESULT(algorithm_name, vm::boc_get_algorithm_name(compressed));

      LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size();

      VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark compress_candidate_data block_id=" << root_hash.to_hex()

      VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark serialize_candidate block_id=" << root_hash.to_hex()

                                        << " called_from=" << called_from

                                        << " time_sec=" << (td::Time::now() - t_compression_start)

                                        << " compression=" << "compressed"

                                        << " compression=" << "compressedV2_" << algorithm_name

                                        << " original_size=" << block.size() + collated_data.size()

                                        << " compressed_size=" << compressed.size();

      return compressed;

    }

    td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(td::Slice compressed,

                                                                                      bool improved_compression,

                                                                                      int decompressed_size,

                                                                                      int max_decompressed_size,

                                                                                      td::Bits256 root_hash) {

    td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(

        td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size,

        std::string called_from, td::Bits256 root_hash) {

      std::vector<td::Ref<vm::Cell>> roots;

      auto t_decompression_start = td::Time::now();

      if (!improved_compression) {

    @@ -127,15 +131,17 @@ td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_dat
  
          return td::Status::Error("decompressed size mismatch");

        }

        TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed));

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark decompress_candidate_data block_id=" << root_hash.to_hex()

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark deserialize_candidate block_id=" << root_hash.to_hex()

                                          << " called_from=" << called_from

                                          << " time_sec=" << (td::Time::now() - t_decompression_start)

                                          << " compression=" << "compressed"

                                          << " compressed_size=" << compressed.size();

                                          << " compression=" << "compressed" << " compressed_size=" << compressed.size();

      } else {

        TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size));

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark decompress_candidate_data block_id=" << root_hash.to_hex()

        TRY_RESULT(algorithm_name, vm::boc_get_algorithm_name(compressed));

        VLOG(VALIDATOR_SESSION_BENCHMARK) << "Broadcast_benchmark deserialize_candidate block_id=" << root_hash.to_hex()

                                          << " called_from=" << called_from

                                          << " time_sec=" << (td::Time::now() - t_decompression_start)

                                          << " compression=" << "compressedV2"

                                          << " compression=" << "compressedV2_" << algorithm_name

                                          << " compressed_size=" << compressed.size();

      }

      if (roots.empty()) {

validator-session/candidate-serializer.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,8 @@ @@
         along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
     */
     #pragma once
+    #include <string>
     #include "auto/tl/ton_api.h"
     #include "ton/ton-types.h"
@@ Expand All @@
                                                                                          bool compression_enabled,
                                                                                          int max_decompressed_data_size);
-    td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size,
+    td::Result<td::BufferSlice> compress_candidate_data(td::Slice block, td::Slice collated_data, std::string called_from,
                                                         td::Bits256 root_hash);
-    td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(td::Slice compressed,
-                                                                                      bool improved_compression,
-                                                                                      int decompressed_size,
-                                                                                      int max_decompressed_size,
-                                                                                      td::Bits256 root_hash);
+    td::Result<std::pair<td::BufferSlice, td::BufferSlice>> decompress_candidate_data(
+        td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size,
+        std::string called_from, td::Bits256 root_hash);
     }  // namespace ton::validatorsession

validator/collator-node/utils.cpp

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -23,19 +23,28 @@
  
    namespace ton::validator {

    constexpr const char* k_called_from_collator_node = "collator_node";

    tl_object_ptr<ton_api::collatorNode_Candidate> serialize_candidate(const BlockCandidate& block, bool compress) {

      if (!compress) {

        return create_tl_object<ton_api::collatorNode_candidate>(

        auto t_compression_start = td::Time::now();

        auto res = create_tl_object<ton_api::collatorNode_candidate>(

            PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(),

            block.collated_data.clone());

        VLOG(COLLATOR_NODE_BENCHMARK) << "Broadcast_benchmark serialize_candidate block_id=" << block.id.root_hash.to_hex()

                                      << " called_from=" << k_called_from_collator_node

                                      << " time_sec=" << (td::Time::now() - t_compression_start)

                                      << " compression=" << "none"

                                      << " original_size=" << block.data.size() + block.collated_data.size()

                                      << " compressed_size=" << block.data.size() + block.collated_data.size();

        return res;

      }

      size_t decompressed_size;

      td::BufferSlice compressed =

          validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size, block.id.root_hash)

              .move_as_ok();

      return create_tl_object<ton_api::collatorNode_compressedCandidate>(

      td::BufferSlice compressed = validatorsession::compress_candidate_data(

                                       block.data, block.collated_data, k_called_from_collator_node, block.id.root_hash)

                                       .move_as_ok();

      return create_tl_object<ton_api::collatorNode_compressedCandidateV2>(

          0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id),

          (int)decompressed_size, std::move(compressed));

          std::move(compressed));

    }

    td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collatorNode_Candidate> f,

    @@ -45,14 +54,21 @@ td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collator
  
          *f, td::overloaded(

                  [&](ton_api::collatorNode_candidate& c) {

                    res = [&]() -> td::Result<BlockCandidate> {

                      auto t_decompression_start = td::Time::now();

                      auto hash = td::sha256_bits256(c.collated_data_);

                      auto key = PublicKey{c.source_};

                      if (!key.is_ed25519()) {

                        return td::Status::Error("invalid pubkey");

                      }

                      auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()};

                      return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_),

                                            std::move(c.collated_data_)};

                      auto block_id = create_block_id(c.id_);

                      BlockCandidate res{e_key, block_id, hash, std::move(c.data_), std::move(c.collated_data_)};

                      VLOG(COLLATOR_NODE_BENCHMARK)

                          << "Broadcast_benchmark deserialize_candidate block_id=" << block_id.root_hash.to_hex()

                          << " called_from=" << k_called_from_collator_node

                          << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "none"

                          << " compressed_size=" << res.data.size() + res.collated_data.size();

                      return std::move(res);

                    }();

                  },

                  [&](ton_api::collatorNode_compressedCandidate& c) {

    @@ -63,9 +79,9 @@ td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collator
  
                      if (c.decompressed_size_ > max_decompressed_data_size) {

                        return td::Status::Error("decompressed size is too big");

                      }

                      TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_,

                                                                                max_decompressed_data_size,

                                                                                create_block_id(c.id_).root_hash));

                      TRY_RESULT(p, validatorsession::decompress_candidate_data(

                                        c.data_, false, c.decompressed_size_, max_decompressed_data_size,

                                        k_called_from_collator_node, create_block_id(c.id_).root_hash));

                      auto collated_data_hash = td::sha256_bits256(p.second);

                      auto key = PublicKey{c.source_};

                      if (!key.is_ed25519()) {

    @@ -79,7 +95,8 @@ td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collator
  
                  [&](ton_api::collatorNode_compressedCandidateV2& c) {

                    res = [&]() -> td::Result<BlockCandidate> {

                      TRY_RESULT(p, validatorsession::decompress_candidate_data(

                                        c.data_, true, 0, max_decompressed_data_size, create_block_id(c.id_).root_hash));

                                        c.data_, true, 0, max_decompressed_data_size, k_called_from_collator_node,

                                        create_block_id(c.id_).root_hash));

                      auto collated_data_hash = td::sha256_bits256(p.second);

                      auto key = PublicKey{c.source_};

                      if (!key.is_ed25519()) {

validator/collator-node/utils.hpp

-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,8 @@ @@
     namespace ton::validator {
+    constexpr int VERBOSITY_NAME(COLLATOR_NODE_BENCHMARK) = verbosity_WARNING;
     tl_object_ptr<ton_api::collatorNode_Candidate> serialize_candidate(const BlockCandidate& block, bool compress);
     td::Result<BlockCandidate> deserialize_candidate(tl_object_ptr<ton_api::collatorNode_Candidate> f,
                                                      int max_decompressed_data_size);
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Activate improved compression #1948

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Activate improved compression #1948

Are you sure you want to change the base?

Uh oh!

Activate improved compression #1948

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!