Enable 2 round trip receipts (#999)

Author: ashamis

src/consensus/pbft/libbyz/nv_info.cpp

@@ -891,7 +891,7 @@ Pre_prepare* NV_info::fetch_request(Seqno n, Digest& d, View& prev_view)
     // Null request
     Req_queue empty;
     size_t requests_in_batch;
-    pp = new Pre_prepare(v, n, empty, requests_in_batch);
+    pp = new Pre_prepare(v, n, empty, requests_in_batch, 0);
     pp->set_digest();
     d = pp->digest();
     prev_view = v;

src/consensus/pbft/libbyz/pre_prepare.cpp

@@ -19,6 +19,7 @@ Pre_prepare::Pre_prepare(
   Seqno s,
   Req_queue& reqs,
   size_t& requests_in_batch,
+  uint64_t nonce_,
   Prepared_cert* prepared_cert) :
   Message(
     Pre_prepare_tag,
@@ -29,14 +30,21 @@ Pre_prepare::Pre_prepare(
       pbft::GlobalState::get_node().auth_size() + // Merkle root signature
       (pbft_max_signature_size + sizeof(uint64_t)) *
         pbft::GlobalState::get_node()
-          .num_of_replicas()) // signatures for the previous pre_prepare
+          .num_of_replicas()), // signatures for the previous pre_prepare
+  nonce(nonce_)
 {
   rep().view = v;
   rep().seqno = s;
   rep().replicated_state_merkle_root.fill(0);
   rep().contains_gov_req = false;
   rep().last_gov_req_updated = 0;
 
+  Digest dh;
+  Digest::Context context;
+  dh.update_last(context, (char*)&nonce_, sizeof(uint64_t));
+  dh.finalize(context);
+  rep().hashed_nonce = dh;
+
   START_CC(pp_digest_cycles);
   INCR_OP(pp_digest);
 
@@ -258,6 +266,7 @@ bool Pre_prepare::calculate_digest(Digest& d)
       rep().replicated_state_merkle_root.size());
     d.update_last(context, (char*)&rep().contains_gov_req, sizeof(uint64_t));
     d.update_last(context, (char*)&rep().last_gov_req_updated, sizeof(Seqno));
+    d.update_last(context, (char*)&rep().hashed_nonce, sizeof(uint64_t));
     d.update_last(context, (char*)&rep().ctx, sizeof(rep().ctx));
     d.update_last(context, (char*)&rep().rset_size, sizeof(rep().rset_size));
     d.update_last(context, (char*)&rep().n_big_reqs, sizeof(rep().n_big_reqs));

src/consensus/pbft/libbyz/pre_prepare.h

@@ -35,6 +35,7 @@ struct Pre_prepare_rep : public Message_rep
   View view;
   Seqno seqno;
   std::array<uint8_t, MERKLE_ROOT_SIZE> replicated_state_merkle_root;
+  Digest hashed_nonce;
   uint64_t contains_gov_req; // should be a bool, but need to use 8 bytes to
                              // maintain alignment
   Seqno last_gov_req_updated;
@@ -67,13 +68,14 @@ class Pre_prepare : public Message
   // Pre_prepare messages
   //
 public:
-  Pre_prepare(uint32_t msg_size = 0) : Message(msg_size) {}
+  Pre_prepare(uint32_t msg_size = 0) : Message(msg_size), nonce(0) {}
 
   Pre_prepare(
     View v,
     Seqno s,
     Req_queue& reqs,
     size_t& requests_in_batch,
+    uint64_t nonce,
     Prepared_cert* prepared_cert = nullptr);
   // Effects: Creates a new signed Pre_prepare message with view
   // number "v", sequence number "s", the requests in "reqs" (up to a
@@ -228,12 +230,17 @@ class Pre_prepare : public Message
   bool is_signed();
   // Effects: checks if there is a signature over the pre_prepare message
 
+  uint64_t get_nonce() const;
+  // Effects: returns the unhashed nonce
+
   static bool convert(Message* m1, Pre_prepare*& m2);
   // Effects: If "m1" has the right size and tag, casts "m1" to a
   // "Pre_prepare" pointer, returns the pointer in "m2" and returns
   // true. Otherwise, it returns false.
 
 private:
+  uint64_t nonce;
+
   Pre_prepare_rep& rep() const;
   // Effects: Casts contents to a Pre_prepare_rep&
 
@@ -328,3 +335,8 @@ inline bool Pre_prepare::did_exec_gov_req() const
 {
   return rep().contains_gov_req;
 }
+
+inline uint64_t Pre_prepare::get_nonce() const
+{
+  return nonce;
+}

src/consensus/pbft/libbyz/prepare.cpp

@@ -12,17 +12,23 @@
 #include "replica.h"
 
 Prepare::Prepare(
-  View v, Seqno s, Digest& d, Principal* dst, bool is_signed, int id) :
+  View v,
+  Seqno s,
+  Digest& d,
+  uint64_t nonce_,
+  Principal* dst,
+  bool is_signed,
+  int id) :
   Message(
     Prepare_tag,
     sizeof(Prepare_rep)
 #ifndef USE_PKEY
-      + ((dst) ? MAC_size : pbft::GlobalState::get_node().auth_size()))
-{
+      + ((dst) ? MAC_size : pbft::GlobalState::get_node().auth_size())),
 #else
-      + ((dst) ? MAC_size : pbft_max_signature_size)
-{
+      + ((dst) ? MAC_size : pbft_max_signature_size)),
 #endif
+  nonce(nonce_)
+{
   rep().extra = (dst) ? 1 : 0;
   rep().view = v;
   rep().seqno = s;
@@ -33,6 +39,12 @@ Prepare::Prepare(
     rep().id = pbft::GlobalState::get_node().id();
   }
 
+  Digest dh;
+  Digest::Context context;
+  dh.update_last(context, (char*)&nonce, sizeof(uint64_t));
+  dh.finalize(context);
+  rep().hashed_nonce = dh;
+
 #ifdef SIGN_BATCH
   rep().digest_sig_size = 0;
   rep().digest_padding.fill(0);
@@ -43,11 +55,13 @@ Prepare::Prepare(
       uint32_t magic = 0xba5eba11;
       NodeId id;
       Digest d;
+      Digest n;
 
-      signature(Digest d_, NodeId id_) : d(d_), id(id_) {}
+      signature(Digest d_, NodeId id_, Digest nonce) : d(d_), id(id_), n(nonce)
+      {}
     };
 
-    signature s(d, rep().id);
+    signature s(d, rep().id, rep().hashed_nonce);
 
     rep().digest_sig_size = pbft::GlobalState::get_node().gen_signature(
       reinterpret_cast<char*>(&s), sizeof(s), rep().batch_digest_signature);

src/consensus/pbft/libbyz/prepare.h

@@ -23,6 +23,7 @@ struct Prepare_rep : public Message_rep
   Seqno seqno;
   Digest digest;
   int id; // id of the replica that generated the message.
+  Digest hashed_nonce;
 #ifdef SIGN_BATCH
   size_t digest_sig_size;
   PbftSignature batch_digest_signature;
@@ -50,12 +51,13 @@ class Prepare : public Message
   // Prepare messages
   //
 public:
-  Prepare(uint32_t msg_size = 0) : Message(msg_size) {}
+  Prepare(uint32_t msg_size = 0) : Message(msg_size), nonce(0) {}
 
   Prepare(
     View v,
     Seqno s,
     Digest& d,
+    uint64_t nonce,
     Principal* dst = 0,
     bool is_signed = false,
     int id = -1);
@@ -99,12 +101,17 @@ class Prepare : public Message
   bool pre_verify();
   // Effects: Performs preliminary verification checks
 
+  uint64_t get_nonce() const;
+  // Effects: returns the unhashed nonce
+
   static bool convert(Message* m1, Prepare*& m2);
   // Effects: If "m1" has the right size and tag, casts "m1" to a
   // "Prepare" pointer, returns the pointer in "m2" and returns
   // true. Otherwise, it returns false.
 
 private:
+  uint64_t nonce;
+
   Prepare_rep& rep() const;
   // Effects: Casts contents to a Prepare_rep&
 };
@@ -152,3 +159,8 @@ inline bool Prepare::match(const Prepare* p) const
   PBFT_ASSERT(view() == p->view() && seqno() == p->seqno(), "Invalid argument");
   return digest() == p->digest();
 }
+
+inline uint64_t Prepare::get_nonce() const
+{
+  return nonce;
+}

src/consensus/pbft/libbyz/receive_message_base.h

@@ -48,6 +48,6 @@ class IMessageReceiveBase
   virtual void playback_pre_prepare(ccf::Store::Tx& tx) = 0;
   virtual void playback_request(ccf::Store::Tx& tx) = 0;
   virtual char* create_response_message(
-    int client_id, Request_id rid, uint32_t size) = 0;
+    int client_id, Request_id rid, uint32_t size, uint64_t nonce) = 0;
   virtual bool IsExecutionPending() = 0;
 };

src/consensus/pbft/libbyz/rep_info.cpp

@@ -34,10 +34,10 @@ void Rep_info::count_request()
 }
 
 char* Rep_info::new_reply(
-  int pid, Request_id rid, Seqno n, uint32_t message_size)
+  int pid, Request_id rid, Seqno n, uint64_t nonce, uint32_t message_size)
 {
   message_size += sizeof(Reply_rep) + MAC_size;
-  auto r = std::make_unique<Reply>(0, rid, n, 0, message_size);
+  auto r = std::make_unique<Reply>(0, rid, n, nonce, 0, message_size);
   PBFT_ASSERT(r != nullptr, "Out of memory");
   r->set_size(message_size);
   r->trim();

src/consensus/pbft/libbyz/rep_info.h

@@ -42,7 +42,8 @@ class Rep_info
   // how many individual requests have been processes since the
   // replica was initialized. Should be called once for each request.
 
-  char* new_reply(int pid, Request_id rid, Seqno n, uint32_t message_size);
+  char* new_reply(
+    int pid, Request_id rid, Seqno n, uint64_t nonce, uint32_t message_size);
   // Effects: Allocates a new reply for request rid from
   // principal pid executed at sequence number n and returns a buffer
   // to store the reply to the command. The buffer can store up to