Merge #18021: Serialization improvements step 4 (undo.h)

3c94b00 Convert undo.h to new serialization framework (Pieter Wuille)
3cd8ab9 Make std::vector and prevector reuse the VectorFormatter logic (Pieter Wuille)
abf8624 Add custom vector-element formatter (Pieter Wuille)
37d800b Add a constant for the maximum vector allocation (5 Mbyte) (Pieter Wuille)

Pull request description:

  The next step of changes from #10785.

  This one adds:
  * A meta-formatter for vectors, which serializes the vector elements using another formatter
  * Switch the undo.h code to the new framework, using the above (where undo entries are serialized as a vector, each of which uses a modified serializer for the UTXOs).

ACKs for top commit:
  laanwj:
    code review ACK 3c94b00
  jonatack:
    Qualified ACK 3c94b00
  ryanofsky:
    Code review ACK 3c94b00. Changes since last review: renaming formatter classes, adding suggested static_assert, and removing temporary in VectorFormatter

Tree-SHA512: 44eebf51a303f6adbbc1ca2b9d043e8ae7fd37e06778e026590892f8d09f8253067862a68ba8ca5d733fd2f8e7c84edd255370f5a4b6560259427a65f94632df

Author: laanwj

src/serialize.h

@@ -25,6 +25,9 @@
 
 static const unsigned int MAX_SIZE = 0x02000000;
 
+/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
+static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
+
 /**
  * Dummy data type to identify deserializing constructors.
  *
@@ -593,6 +596,53 @@ class LimitedString
 template<typename I>
 BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
 
+/** Formatter to serialize/deserialize vector elements using another formatter
+ *
+ * Example:
+ *   struct X {
+ *     std::vector<uint64_t> v;
+ *     SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
+ *   };
+ * will define a struct that contains a vector of uint64_t, which is serialized
+ * as a vector of VarInt-encoded integers.
+ *
+ * V is not required to be an std::vector type. It works for any class that
+ * exposes a value_type, size, reserve, push_back, and const iterators.
+ */
+template<class Formatter>
+struct VectorFormatter
+{
+    template<typename Stream, typename V>
+    void Ser(Stream& s, const V& v)
+    {
+        WriteCompactSize(s, v.size());
+        for (const typename V::value_type& elem : v) {
+            s << Using<Formatter>(elem);
+        }
+    }
+
+    template<typename Stream, typename V>
+    void Unser(Stream& s, V& v)
+    {
+        v.clear();
+        size_t size = ReadCompactSize(s);
+        size_t allocated = 0;
+        while (allocated < size) {
+            // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
+            // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
+            // X MiB of data to make us allocate X+5 Mib.
+            static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
+            allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
+            v.reserve(allocated);
+            while (v.size() < allocated) {
+                typename V::value_type val;
+                s >> Using<Formatter>(val);
+                v.push_back(std::move(val));
+            }
+        }
+    };
+};
+
 /**
  * Forward declarations
  */
@@ -673,6 +723,20 @@ inline void Unserialize(Stream& is, T&& a)
     a.Unserialize(is);
 }
 
+/** Default formatter. Serializes objects as themselves.
+ *
+ * The vector/prevector serialization code passes this to VectorFormatter
+ * to enable reusing that logic. It shouldn't be needed elsewhere.
+ */
+struct DefaultFormatter
+{
+    template<typename Stream, typename T>
+    static void Ser(Stream& s, const T& t) { Serialize(s, t); }
+
+    template<typename Stream, typename T>
+    static void Unser(Stream& s, T& t) { Unserialize(s, t); }
+};
+
 
 
 
@@ -713,9 +777,7 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
 template<typename Stream, unsigned int N, typename T, typename V>
 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
 {
-    WriteCompactSize(os, v.size());
-    for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
-        ::Serialize(os, (*vi));
+    Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, unsigned int N, typename T>
@@ -744,19 +806,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
 template<typename Stream, unsigned int N, typename T, typename V>
 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
 {
-    v.clear();
-    unsigned int nSize = ReadCompactSize(is);
-    unsigned int i = 0;
-    unsigned int nMid = 0;
-    while (nMid < nSize)
-    {
-        nMid += 5000000 / sizeof(T);
-        if (nMid > nSize)
-            nMid = nSize;
-        v.resize_uninitialized(nMid);
-        for (; i < nMid; ++i)
-            Unserialize(is, v[i]);
-    }
+    Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, unsigned int N, typename T>
@@ -793,9 +843,7 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
 template<typename Stream, typename T, typename A, typename V>
 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
 {
-    WriteCompactSize(os, v.size());
-    for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
-        ::Serialize(os, (*vi));
+    Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, typename T, typename A>
@@ -824,19 +872,7 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
 template<typename Stream, typename T, typename A, typename V>
 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
 {
-    v.clear();
-    unsigned int nSize = ReadCompactSize(is);
-    unsigned int i = 0;
-    unsigned int nMid = 0;
-    while (nMid < nSize)
-    {
-        nMid += 5000000 / sizeof(T);
-        if (nMid > nSize)
-            nMid = nSize;
-        v.resize(nMid);
-        for (; i < nMid; i++)
-            Unserialize(is, v[i]);
-    }
+    Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, typename T, typename A>

src/undo.h

@@ -13,88 +13,50 @@
 #include <serialize.h>
 #include <version.h>
 
-/** Undo information for a CTxIn
+/** Formatter for undo information for a CTxIn
  *
  *  Contains the prevout's CTxOut being spent, and its metadata as well
  *  (coinbase or not, height). The serialization contains a dummy value of
  *  zero. This is compatible with older versions which expect to see
  *  the transaction version there.
  */
-class TxInUndoSerializer
+struct TxInUndoFormatter
 {
-    const Coin* txout;
-
-public:
     template<typename Stream>
-    void Serialize(Stream &s) const {
-        ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1u : 0u)));
-        if (txout->nHeight > 0) {
+    void Ser(Stream &s, const Coin& txout) {
+        ::Serialize(s, VARINT(txout.nHeight * 2 + (txout.fCoinBase ? 1u : 0u)));
+        if (txout.nHeight > 0) {
             // Required to maintain compatibility with older undo format.
             ::Serialize(s, (unsigned char)0);
         }
-        ::Serialize(s, Using<TxOutCompression>(REF(txout->out)));
+        ::Serialize(s, Using<TxOutCompression>(txout.out));
     }
 
-    explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {}
-};
-
-class TxInUndoDeserializer
-{
-    Coin* txout;
-
-public:
     template<typename Stream>
-    void Unserialize(Stream &s) {
+    void Unser(Stream &s, Coin& txout) {
         unsigned int nCode = 0;
         ::Unserialize(s, VARINT(nCode));
-        txout->nHeight = nCode / 2;
-        txout->fCoinBase = nCode & 1;
-        if (txout->nHeight > 0) {
+        txout.nHeight = nCode / 2;
+        txout.fCoinBase = nCode & 1;
+        if (txout.nHeight > 0) {
             // Old versions stored the version number for the last spend of
             // a transaction's outputs. Non-final spends were indicated with
             // height = 0.
             unsigned int nVersionDummy;
             ::Unserialize(s, VARINT(nVersionDummy));
         }
-        ::Unserialize(s, Using<TxOutCompression>(REF(txout->out)));
+        ::Unserialize(s, Using<TxOutCompression>(txout.out));
     }
-
-    explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {}
 };
 
-static const size_t MIN_TRANSACTION_INPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxIn(), PROTOCOL_VERSION);
-static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_INPUT_WEIGHT;
-
 /** Undo information for a CTransaction */
 class CTxUndo
 {
 public:
     // undo information for all txins
     std::vector<Coin> vprevout;
 
-    template <typename Stream>
-    void Serialize(Stream& s) const {
-        // TODO: avoid reimplementing vector serializer
-        uint64_t count = vprevout.size();
-        ::Serialize(s, COMPACTSIZE(REF(count)));
-        for (const auto& prevout : vprevout) {
-            ::Serialize(s, TxInUndoSerializer(&prevout));
-        }
-    }
-
-    template <typename Stream>
-    void Unserialize(Stream& s) {
-        // TODO: avoid reimplementing vector deserializer
-        uint64_t count = 0;
-        ::Unserialize(s, COMPACTSIZE(count));
-        if (count > MAX_INPUTS_PER_BLOCK) {
-            throw std::ios_base::failure("Too many input undo records");
-        }
-        vprevout.resize(count);
-        for (auto& prevout : vprevout) {
-            ::Unserialize(s, TxInUndoDeserializer(&prevout));
-        }
-    }
+    SERIALIZE_METHODS(CTxUndo, obj) { READWRITE(Using<VectorFormatter<TxInUndoFormatter>>(obj.vprevout)); }
 };
 
 /** Undo information for a CBlock */
@@ -103,12 +65,7 @@ class CBlockUndo
 public:
     std::vector<CTxUndo> vtxundo; // for all but the coinbase
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(vtxundo);
-    }
+    SERIALIZE_METHODS(CBlockUndo, obj) { READWRITE(obj.vtxundo); }
 };
 
 #endif // BITCOIN_UNDO_H