Ledger chunking (#1274)

Author: jumaffre

.azure-pipelines-templates/install.yml

@@ -17,15 +17,15 @@ steps:
       -b ${{ parameters.install_prefix }}/bin \
       -g $(pwd)/../../src/runtime_config/gov.lua \
       -v
-    cp ./workspace/start_network_0/0.ledger .
+    cp -r ./workspace/start_network_0/0.ledger .
     cp ./workspace/start_network_0/network_enc_pubk.pem .
     timeout --signal=SIGINT --kill-after=30s --preserve-status 30s \
       python ${{ parameters.install_prefix }}/bin/start_network.py \
       -p ../../build/liblogging \
       -b ${{ parameters.install_prefix }}/bin \
       -v \
       --recover \
-      --ledger 0.ledger \
+      --ledger-dir 0.ledger \
       --network-enc-pubk network_enc_pubk.pem \
       --common-dir ./workspace/start_network_common/
   displayName: Test installed CCF

CMakeLists.txt

@@ -313,7 +313,8 @@ if(BUILD_TESTS)
     target_link_libraries(encryptor_test PRIVATE secp256k1.host)
 
     add_unit_test(
-      historical_queries_test ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/historical_queries.cpp
+      historical_queries_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/historical_queries.cpp
     )
     target_link_libraries(historical_queries_test PRIVATE secp256k1.host)
 
@@ -461,9 +462,7 @@ if(BUILD_TESTS)
     kv_bench SRCS src/kv/test/kv_bench.cpp src/crypto/symmetric_key.cpp
                   src/enclave/thread_local.cpp
   )
-  add_picobench(
-    hash_bench SRCS src/ds/test/hash_bench.cpp
-  )
+  add_picobench(hash_bench SRCS src/ds/test/hash_bench.cpp)
 
   # Storing signed governance operations
   add_e2e_test(
@@ -483,6 +482,10 @@ if(BUILD_TESTS)
       # purpose to check that a recovery network is robust to a view change.
       --raft-election-timeout
       4000
+      # Smaller ledger chunks than default to make sure that a ledger made of
+      # multiple chunks can be recovered
+      --ledger-chunk-max-bytes
+      20KB
   )
 
   add_e2e_test(

doc/developers/ledger.rst

@@ -3,7 +3,7 @@ Ledger
 
 The ledger is the persistent distributed append-only record of the transactions that have been executed by the network. It is written by the primary when a transaction is committed and replicated to all backups which maintain their own duplicated copy.
 
-A node writes its ledger to a file as specified by the ``--ledger-file`` command line argument.
+A node writes its ledger to a directory as specified by the ``--ledger-dir`` command line argument.
 
 Ledger Encryption
 -----------------

doc/operators/recovery.rst

@@ -26,12 +26,12 @@ To initiate the first phase of the recovery procedure, one or several nodes shou
     --rpc-address <ccf-node-address>
     --public-rpc-address <ccf-node-public-address>
     [--domain domain]
-    --ledger-file /path/to/ledger/to/recover
+    --ledger-dir /path/to/ledger/dir/to/recover
     --node-cert-file /path/to/node_certificate
     recover
     --network-cert-file /path/to/network_certificate
 
-Each node will then immediately restore the public entries of its ledger (``--ledger-file``). Because deserialising the public entries present in the ledger may take some time, operators can query the progress of the public recovery by calling ``getSignedIndex`` which returns the version of the last signed recovered ledger entry. Once the public ledger is fully recovered, the recovered node automatically becomes part of the public network, allowing other nodes to join the network.
+Each node will then immediately restore the public entries of its ledger (``--ledger-dir``). Because deserialising the public entries present in the ledger may take some time, operators can query the progress of the public recovery by calling ``getSignedIndex`` which returns the version of the last signed recovered ledger entry. Once the public ledger is fully recovered, the recovered node automatically becomes part of the public network, allowing other nodes to join the network.
 
 .. note:: If more than one node were started in ``recover`` mode, the node with the highest signed index (as per the response to the ``getSignedIndex`` RPC) should be preferred to start the new network. Other nodes should be shutdown and new nodes restarted with the ``join`` option.
 
@@ -44,7 +44,7 @@ Similarly to the normal join protocol (see :ref:`operators/start_network:Adding
         participant Node 2
         participant Node 3
 
-        Operators->>+Node 2: cchost --rpc-address=ip2:port2 --ledger-file=ledger0 recover
+        Operators->>+Node 2: cchost --rpc-address=ip2:port2 --ledger-dir=./ledger recover
         Node 2-->>Operators: Network Certificate
         Note over Node 2: Reading Public Ledger...
 

doc/operators/start_network.rst

@@ -19,7 +19,7 @@ To create a new CCF network, the first node of the network should be invoked wit
     --node-address <ccf-node-to-node-address>
     --public-rpc-address <ccf-node-public-address>
     [--domain domain]
-    --ledger-file /path/to/ledger
+    --ledger-dir /path/to/ledger/dir
     --node-cert-file /path/to/node_certificate
     [--sig-max-tx number_of_transactions]
     [--sig-max-ms number_of_milliseconds]
@@ -81,7 +81,7 @@ To add a new node to an existing opening network, other nodes should be started
     --rpc-address <ccf-node-address>
     --node-address <ccf-node-to-node-address>
     --public-rpc-address <ccf-node-public-address>
-    --ledger-file /path/to/ledger
+    --ledger-dir /path/to/ledger/dir
     --node-cert-file /path/to/node_certificate
     join
     --network-cert-file /path/to/existing/network_certificate

doc/users/deploy_app.rst

@@ -41,9 +41,9 @@ The ``start_test_network.sh`` script can also be used to automatically recover a
 .. code-block:: bash
 
     $ cd CCF/build
-    $ cp ./workspace/test_network_0/0.ledger .
+    $ cp -r ./workspace/test_network_0/0.ledger .
     $ cp ./workspace/test_network_0/network_enc_pubk.pem .
-    $ ../start_test_network.sh -p liblogging.enclave.so.signed --recover --ledger 0.ledger --network-enc-pubk network_enc_pubk.pem --common-dir ./workspace/test_network_common/
+    $ ../start_test_network.sh -p liblogging.enclave.so.signed --recover --ledger-dir 0.ledger --network-enc-pubk network_enc_pubk.pem --common-dir ./workspace/test_network_common/
     [2020-05-14 14:50:19.746] Starting 3 CCF nodes...
     [2020-05-14 14:50:19.746] Recovering network from:
     [2020-05-14 14:50:19.746]  - Ledger: 0.ledger

src/consensus/ledger_enclave.h

@@ -28,70 +28,56 @@ namespace consensus
      * Put a single entry to be written the ledger, when primary.
      *
      * @param entry Serialised entry
+     * @param globally_committable True is entry is signature transaction
      */
-    void put_entry(const std::vector<uint8_t>& entry)
+    void put_entry(const std::vector<uint8_t>& entry, bool globally_committable)
     {
-      // write the message
-      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_append, to_host, entry);
+      put_entry(entry.data(), entry.size(), globally_committable);
     }
 
     /**
      * Put a single entry to be written the ledger, when primary.
      *
      * @param data Serialised entry start
      * @param size Serialised entry size
+     * @param globally_committable True is entry is signature transaction
      */
-    void put_entry(const uint8_t* data, size_t size)
+    void put_entry(const uint8_t* data, size_t size, bool globally_committable)
     {
       serializer::ByteRange byte_range = {data, size};
       // write the message
-      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_append, to_host, byte_range);
+      RINGBUFFER_WRITE_MESSAGE(
+        consensus::ledger_append, to_host, globally_committable, byte_range);
     }
 
     /**
-     * Record a single entry to the ledger, when backup.
+     * Skip a single entry, when backup.
+     *
+     * Does not write any entry to the legder.
      *
      * @param data Serialised entries
      * @param size Size of overall serialised entries
-     *
-     * @return Pair of boolean status (false if rejected), raw data as a vector
      */
-    std::pair<std::vector<uint8_t>, bool> record_entry(
-      const uint8_t*& data, size_t& size)
+    void skip_entry(const uint8_t*& data, size_t& size)
     {
       auto entry_len = serialized::read<uint32_t>(data, size);
-      std::vector<uint8_t> entry(data, data + entry_len);
-
-      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_append, to_host, entry);
-
       serialized::skip(data, size, entry_len);
-
-      return std::make_pair(std::move(entry), true);
     }
 
     /**
-     * Skip a single entry, when backup.
-     *
-     * Does not write any entry to the legder.
+     * Retrieve a single entry, advancing offset to the next entry.
      *
      * @param data Serialised entries
      * @param size Size of overall serialised entries
+     *
+     * @return Raw entry as a vector
      */
-    void skip_entry(const uint8_t*& data, size_t& size)
-    {
-      auto entry_len = serialized::read<uint32_t>(data, size);
-      serialized::skip(data, size, entry_len);
-    }
-
-    std::pair<std::vector<uint8_t>, bool> get_entry(
-      const uint8_t*& data, size_t& size)
+    std::vector<uint8_t> get_entry(const uint8_t*& data, size_t& size)
     {
       auto entry_len = serialized::read<uint32_t>(data, size);
       std::vector<uint8_t> entry(data, data + entry_len);
-
       serialized::skip(data, size, entry_len);
-
-      return std::make_pair(std::move(entry), true);
+      return entry;
     }
 
     /**
@@ -103,5 +89,15 @@ namespace consensus
     {
       RINGBUFFER_WRITE_MESSAGE(consensus::ledger_truncate, to_host, idx);
     }
+
+    /**
+     * Commit the ledger at a given index.
+     *
+     * @param idx Index to commit at
+     */
+    void commit(Index idx)
+    {
+      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_commit, to_host, idx);
+    }
   };
 }

src/consensus/ledger_enclave_types.h

@@ -30,6 +30,7 @@ namespace consensus
     /// Modify the local ledger. Enclave -> Host
     DEFINE_RINGBUFFER_MSG_TYPE(ledger_append),
     DEFINE_RINGBUFFER_MSG_TYPE(ledger_truncate),
+    DEFINE_RINGBUFFER_MSG_TYPE(ledger_commit),
     ///@}
   };
 }
@@ -46,6 +47,7 @@ DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   consensus::Index,
   consensus::LedgerRequestPurpose);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
-  consensus::ledger_append, std::vector<uint8_t>);
+  consensus::ledger_append, bool /* committable */, std::vector<uint8_t>);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   consensus::ledger_truncate, consensus::Index);
+DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(consensus::ledger_commit, consensus::Index);

src/consensus/pbft/libbyz/ledger_writer.h

@@ -2,12 +2,12 @@
 // Licensed under the MIT license.
 #pragma once
 
+#include "consensus/pbft/libbyz/ledger.h"
 #include "consensus/pbft/pbft_new_views.h"
 #include "consensus/pbft/pbft_pre_prepares.h"
 #include "consensus/pbft/pbft_requests.h"
 #include "consensus/pbft/pbft_types.h"
 #include "kv/kv_types.h"
-#include "ledger.h"
 #include "new_view.h"
 #include "node/signatures.h"
 #include "prepared_cert.h"

src/consensus/pbft/libbyz/test/replica_unit_tests.cpp

@@ -8,7 +8,6 @@
 #include "consensus/pbft/pbft_tables.h"
 #include "consensus/pbft/pbft_types.h"
 #include "ds/ccf_exception.h"
-#include "host/ledger.h"
 #include "kv/store.h"
 #include "kv/test/stub_consensus.h"
 #include "message.h"