fcmp++: hash the tree root in the PoW hash

Author: j-berman

src/blockchain_db/blockchain_db.h

@@ -1807,6 +1807,8 @@ class BlockchainDB
    */
   virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
 
+  virtual void advance_tree_one_block(const uint64_t block_idx) = 0;
+
   // TODO: description and make private
   virtual void grow_tree(const uint64_t block_id, std::vector<fcmp_pp::curve_trees::OutputContext> &&new_outputs) = 0;
 

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -819,13 +819,6 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
       throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
   }
 
-  // Grow the tree with outputs that are no longer locked once this block is in the chain
-  auto unlocked_outputs = this->get_outs_at_last_locked_block_id(m_height);
-  this->grow_tree(m_height, std::move(unlocked_outputs));
-
-  // Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked outputs table
-  this->del_locked_outs_at_block_id(m_height);
-
   int result = 0;
 
   MDB_val_set(key, m_height);
@@ -1405,6 +1398,31 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
   }
 }
 
+void BlockchainLMDB::advance_tree_one_block(const uint64_t blk_idx)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  if (blk_idx == 0)
+    throw0(DB_ERROR("Expected to advance to blk_idx > 0"));
+
+  if (blk_idx == 1)
+  {
+    // If we're advancing 1 block past genesis, make sure to initialize the tree for the genesis block.
+    // We grow the genesis block with empty outputs, since no outputs should be spendable once it's in the chain.
+    this->grow_tree(0, {});
+  }
+
+  // Now we can advance the tree 1 block
+  auto unlocked_outputs = this->get_outs_at_last_locked_block_id(blk_idx);
+
+  // Grow the tree with outputs that are spendable once blk_idx is in the chain (i.e. they can be included starting in blk_idx+1)
+  this->grow_tree(blk_idx, std::move(unlocked_outputs));
+
+  // Now that we've used the unlocked leaves to grow the tree, we delete them from the locked outputs table
+  this->del_locked_outs_at_block_id(blk_idx);
+}
+
 void BlockchainLMDB::grow_tree(const uint64_t blk_idx, std::vector<fcmp_pp::curve_trees::OutputContext> &&new_outputs)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -1426,13 +1444,15 @@ void BlockchainLMDB::grow_tree(const uint64_t blk_idx, std::vector<fcmp_pp::curv
     // Get the block idx corresponding to the current tree in the db
     const uint64_t tree_block_idx = this->get_tree_block_idx();
     if (blk_idx > (tree_block_idx + 1))
-      throw0(DB_ERROR("The chain has extended too far past the tree"));
+      throw0(DB_ERROR(("The chain has extended too far past the tree (blk_idx=" + std::to_string(blk_idx) + ", tree_block_idx=" + std::to_string(tree_block_idx) + ")").c_str()));
     if (tree_block_idx >= blk_idx)
     {
       LOG_PRINT_L1("Skip re-growing the tree at block " << blk_idx << ", waiting for the chain to catch up to the tree's block " << tree_block_idx);
       return;
     }
-    // blk_idx == (tree_block_idx + 1), we're growing the tree 1 block
+    // We're supposed to be growing the tree to the given blk_idx
+    if (blk_idx != (tree_block_idx + 1))
+      throw0(DB_ERROR(("Mismatch block idx to tree tip (blk_idx=" + std::to_string(blk_idx) + ", tree_block_idx=" + std::to_string(tree_block_idx) + ")").c_str()));
 
     // Get the prev block's tree edge (i.e. the current tree edge before growing)
     prev_blk_idx = blk_idx - 1;
@@ -7585,12 +7605,7 @@ void BlockchainLMDB::migrate_5_6()
           }
         }
 
-        // Get leaf tuples of outputs that unlock once i is in the chain, since i is their last locked block
-        auto unlocked_outputs = this->get_outs_at_last_locked_block_id(i);
-        this->grow_tree(i, std::move(unlocked_outputs));
-
-        // Now that we've used the unlocked leaves to grow the tree, we delete them from the locked outputs table
-        this->del_locked_outs_at_block_id(i);
+        this->advance_tree_one_block(i);
 
         LOGIF(el::Level::Info)
         {

src/blockchain_db/lmdb/db_lmdb.h

@@ -342,6 +342,8 @@ class BlockchainLMDB : public BlockchainDB
                             , const std::vector<std::pair<transaction, blobdata>>& txs
                             );
 
+  virtual void advance_tree_one_block(const uint64_t blk_idx);
+
   virtual void set_batch_transactions(bool batch_transactions);
   virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0);
   virtual void batch_commit();

src/blockchain_db/testdb.h

@@ -117,6 +117,7 @@ class BaseTestDB: public cryptonote::BlockchainDB {
   virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
   virtual void add_spent_key(const crypto::key_image& k_image) override {}
   virtual void remove_spent_key(const crypto::key_image& k_image) override {}
+  virtual void advance_tree_one_block(const uint64_t block_idx) override {}
   virtual void grow_tree(const uint64_t block_idx, std::vector<fcmp_pp::curve_trees::OutputContext> &&new_outputs) override {};
   virtual std::pair<uint64_t, fcmp_pp::curve_trees::PathBytes> get_last_path(const uint64_t block_idx) const override { return {0, fcmp_pp::curve_trees::PathBytes{}}; };
   virtual void trim_tree(const uint64_t new_n_leaf_tuples, const uint64_t trim_block_id) override {};

src/crypto/crypto.h

@@ -353,6 +353,9 @@ namespace crypto {
   inline std::ostream &operator <<(std::ostream &o, const crypto::view_tag &v) {
     epee::to_hex::formatted(o, epee::as_byte_span(v)); return o;
   }
+  inline std::ostream &operator <<(std::ostream &o, const crypto::ec_point &v) {
+    epee::to_hex::formatted(o, epee::as_byte_span(v)); return o;
+  }
 
   const extern crypto::public_key null_pkey;
   const extern crypto::secret_key null_skey;

src/cryptonote_basic/cryptonote_basic.h

@@ -478,8 +478,8 @@ namespace cryptonote
 
   public:
     block(): block_header(), hash_valid(false) {}
-    block(const block &b): block_header(b), hash_valid(false), miner_tx(b.miner_tx), tx_hashes(b.tx_hashes) { if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } }
-    block &operator=(const block &b) { block_header::operator=(b); hash_valid = false; miner_tx = b.miner_tx; tx_hashes = b.tx_hashes; if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } return *this; }
+    block(const block &b): block_header(b), hash_valid(false), miner_tx(b.miner_tx), tx_hashes(b.tx_hashes), fcmp_pp_tree_root(b.fcmp_pp_tree_root) { if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } }
+    block &operator=(const block &b) { block_header::operator=(b); hash_valid = false; miner_tx = b.miner_tx; tx_hashes = b.tx_hashes; fcmp_pp_tree_root = b.fcmp_pp_tree_root; if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } return *this; }
     void invalidate_hashes() { set_hash_valid(false); }
     bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
     void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
@@ -488,6 +488,8 @@ namespace cryptonote
     transaction miner_tx;
     std::vector<crypto::hash> tx_hashes;
 
+    crypto::ec_point fcmp_pp_tree_root;
+
     // hash cash
     mutable crypto::hash hash;
 
@@ -500,6 +502,8 @@ namespace cryptonote
       FIELD(tx_hashes)
       if (tx_hashes.size() > CRYPTONOTE_MAX_TX_PER_BLOCK)
         return false;
+      if (major_version >= HF_VERSION_FCMP_PLUS_PLUS)
+        FIELD(fcmp_pp_tree_root)
     END_SERIALIZE()
   };
 

src/cryptonote_basic/cryptonote_boost_serialization.h

@@ -202,6 +202,8 @@ namespace boost
     //------------------
     a & b.miner_tx;
     a & b.tx_hashes;
+    if (b.major_version >= HF_VERSION_FCMP_PLUS_PLUS)
+      a & b.fcmp_pp_tree_root;
   }
 
   template <class Archive>

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -99,6 +99,18 @@ namespace cryptonote
     const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5;
     return bp_clawback;
   }
+
+  static void get_tree_hash(const std::vector<crypto::hash>& tx_hashes, crypto::hash& h)
+  {
+    tree_hash(tx_hashes.data(), tx_hashes.size(), h);
+  }
+
+  static crypto::hash get_tree_hash(const std::vector<crypto::hash>& tx_hashes)
+  {
+    crypto::hash h = null_hash;
+    get_tree_hash(tx_hashes, h);
+    return h;
+  }
   //---------------------------------------------------------------
 }
 
@@ -1464,7 +1476,7 @@ namespace cryptonote
   blobdata get_block_hashing_blob(const block& b)
   {
     blobdata blob = t_serializable_object_to_blob(static_cast<block_header>(b));
-    crypto::hash tree_root_hash = get_tx_tree_hash(b);
+    crypto::hash tree_root_hash = get_tree_hash(b);
     blob.append(reinterpret_cast<const char*>(&tree_root_hash), sizeof(tree_root_hash));
     blob.append(tools::get_varint_data(b.tx_hashes.size()+1));
     return blob;
@@ -1607,29 +1619,23 @@ namespace cryptonote
     return t_serializable_object_to_blob(tx, b_blob);
   }
   //---------------------------------------------------------------
-  void get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes, crypto::hash& h)
-  {
-    tree_hash(tx_hashes.data(), tx_hashes.size(), h);
-  }
-  //---------------------------------------------------------------
-  crypto::hash get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes)
-  {
-    crypto::hash h = null_hash;
-    get_tx_tree_hash(tx_hashes, h);
-    return h;
-  }
-  //---------------------------------------------------------------
-  crypto::hash get_tx_tree_hash(const block& b)
+  crypto::hash get_tree_hash(const block& b)
   {
-    std::vector<crypto::hash> txs_ids;
-    txs_ids.reserve(1 + b.tx_hashes.size());
+    std::vector<crypto::hash> hashes;
+    hashes.reserve(1 + 1 + b.tx_hashes.size());
+    // 1. FCMP++ root
+    static_assert(sizeof(crypto::hash) == sizeof(crypto::ec_point), "expected sizeof hash == sizeof ec_point");
+    if (b.major_version >= HF_VERSION_FCMP_PLUS_PLUS)
+      hashes.push_back((crypto::hash&)b.fcmp_pp_tree_root);
+    // 2. Miner tx
     crypto::hash h = null_hash;
     size_t bl_sz = 0;
     CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(b.miner_tx, h, bl_sz), "Failed to calculate transaction hash");
-    txs_ids.push_back(h);
+    hashes.push_back(h);
+    // 3. All other txs
     for(auto& th: b.tx_hashes)
-      txs_ids.push_back(th);
-    return get_tx_tree_hash(txs_ids);
+      hashes.push_back(th);
+    return get_tree_hash(hashes);
   }
   //---------------------------------------------------------------
   bool is_valid_decomposed_amount(uint64_t amount)

src/cryptonote_basic/cryptonote_format_utils.h

@@ -260,9 +260,7 @@ namespace cryptonote
   bool block_to_blob(const block& b, blobdata& b_blob);
   blobdata tx_to_blob(const transaction& b);
   bool tx_to_blob(const transaction& b, blobdata& b_blob);
-  void get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes, crypto::hash& h);
-  crypto::hash get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes);
-  crypto::hash get_tx_tree_hash(const block& b);
+  crypto::hash get_tree_hash(const block& b);
   bool is_valid_decomposed_amount(uint64_t amount);
   void get_hash_stats(uint64_t &tx_hashes_calculated, uint64_t &tx_hashes_cached, uint64_t &block_hashes_calculated, uint64_t & block_hashes_cached);
 

src/cryptonote_core/blockchain.cpp

@@ -1529,6 +1529,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block,
     invalidate_block_template_cache();
   }
 
+  const uint64_t cur_n_blocks = m_db->height();
   if (from_block)
   {
     //build alternative subchain, front -> mainchain, back -> alternative head
@@ -1589,6 +1590,32 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block,
     b.minor_version = m_hardfork->get_ideal_version();
     b.prev_id = *from_block;
 
+    if (b.major_version >= HF_VERSION_FCMP_PLUS_PLUS)
+    {
+      if (alt_chain.size() > CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE)
+      {
+        // Since the daemon is not wasting cycles calculating alt-trees, it does
+        // not know the correct root to use for an alt chain longer than 10
+        // blocks. The daemon could theoretically build the altchain's tree in
+        // handle_alternative_block, but that seems like a pre-mature
+        // optimization. Who wants to mine altchains longer than 10 blocks
+        // anyway?
+        MERROR("altchain is too long, the daemon is not structured to calculate its FCMP++ tree root");
+        return false;
+      }
+
+      if (height > cur_n_blocks)
+      {
+        MERROR("altchain is longer than the mainchain, so we don't have the FCMP++ tree root saved for its block");
+        // Theoretically the daemon could calculate the tree 10 blocks ahead
+        // of the mainchain, which would allow us to mine on top of an altchain
+        // while growing the mainchain tree (which we need to do anyway), but it
+        // seems like pre-mature optimization to support mining on top
+        // of altchains longer than the mainchain anyway.
+        return false;
+      }
+    }
+
     // cheat and use the weight of the block we start from, virtually certain to be acceptable
     // and use 1.9 times rather than 2 times so we're even more sure
     if (parent_in_main)
@@ -1611,7 +1638,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block,
   }
   else
   {
-    height = m_db->height();
+    height = cur_n_blocks;
     b.major_version = m_hardfork->get_current_version();
     b.minor_version = m_hardfork->get_ideal_version();
     b.prev_id = get_tail_id();
@@ -1635,6 +1662,11 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block,
 
   CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead.");
 
+  if (b.major_version >= HF_VERSION_FCMP_PLUS_PLUS)
+  {
+    m_db->get_tree_root_at_blk_idx(height, b.fcmp_pp_tree_root);
+  }
+
   size_t txs_weight;
   uint64_t fee;
   if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, b.major_version))
@@ -1768,15 +1800,21 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
   return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, cumulative_weight, ex_nonce, seed_height, seed_hash);
 }
 //------------------------------------------------------------------
-bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog)
+bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::ec_point& fcmp_pp_tree_root, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog)
 {
   prev_id = m_db->top_block_hash(&height);
   ++height;
 
   major_version = m_hardfork->get_ideal_version(height);
 
+  const uint8_t cur_version = m_hardfork->get_current_version();
+
+  fcmp_pp_tree_root = crypto::ec_point{};
+  if (cur_version >= HF_VERSION_FCMP_PLUS_PLUS)
+    m_db->get_tree_root_at_blk_idx(height, fcmp_pp_tree_root);
+
   seed_hash = crypto::null_hash;
-  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  if (cur_version >= RX_BLOCK_VERSION)
   {
     uint64_t seed_height, next_height;
     crypto::rx_seedheights(height, &seed_height, &next_height);
@@ -4316,6 +4354,19 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash&
 
   TIME_MEASURE_START(t3);
 
+  // Make sure the block uses the correct FCMP++ tree root
+  if (hf_version >= HF_VERSION_FCMP_PLUS_PLUS)
+  {
+    crypto::ec_point fcmp_pp_tree_root;
+    m_db->get_tree_root_at_blk_idx(blockchain_height, fcmp_pp_tree_root);
+    if (bl.fcmp_pp_tree_root != fcmp_pp_tree_root)
+    {
+      MERROR_VER("Block with id: " << id << " used incorrect FCMP++ tree root");
+      bvc.m_verifivation_failed = true;
+      goto leave;
+    }
+  }
+
   // sanity check basic miner tx properties;
   if(!prevalidate_miner_transaction(bl, blockchain_height, hf_version))
   {
@@ -4630,14 +4681,36 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash&
     return false;
   }
 
+  // Assume we're adding block n, now we grow the FCMP++ tree and generate a
+  // new root that will be usable once block n+1 is in the chain. We keep the
+  // tree 1 block ahead of the chain so that miners can use the block's tree
+  // root to calculate the next block's PoW hash. Note that we're still growing
+  // the tree 1 block at a time in the sync process. We just grow the tree ahead
+  // of the "next" block after syncing "this" block. This approach works because
+  // outputs are not immediately spendable upon inclusion in the chain. There is
+  // room for improvement in parallelizing tree building, left for another day.
+  TIME_MEASURE_START(advance_tree);
+
+  static_assert(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > 0, "Expect a non-0 spendable age");
+  try { m_db->advance_tree_one_block(new_height); }
+  catch (const std::exception& e)
+  {
+    LOG_ERROR("Failed to advance tree at block with hash: " << id << ", what = " << e.what());
+    bvc.m_verifivation_failed = true;
+    return false;
+  }
+
+  TIME_MEASURE_FINISH(advance_tree);
+
   MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms");
   if(m_show_time_stats)
   {
     MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: "
         << cumulative_block_weight << " p/t: " << block_processing_time << " ("
         << target_calculating_time << "/" << longhash_calculating_time << "/"
         << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool
-        << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms");
+        << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock
+        << "/" << advance_tree << ")ms");
   }
 
   bvc.m_added_to_main_chain = true;
@@ -5909,9 +5982,13 @@ void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &s
   std::vector<tx_block_template_backlog_entry> tx_backlog;
   m_tx_pool.get_block_template_backlog(tx_backlog);
 
+  crypto::ec_point fcmp_pp_tree_root{};
+  if (m_hardfork->get_current_version() >= HF_VERSION_FCMP_PLUS_PLUS)
+    m_db->get_tree_root_at_blk_idx(height, fcmp_pp_tree_root);
+
   for (const auto& notifier : m_miner_notifiers)
   {
-    notifier(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog);
+    notifier(major_version, height, prev_id, fcmp_pp_tree_root, seed_hash, diff, median_weight, already_generated_coins, tx_backlog);
   }
 }