Merge pull request #541 from evoskuil/master

Updates to prevouts table/queries, impl set_filter_head(link).

Author: evoskuil

.gitignore

@@ -29,3 +29,4 @@ build
 
 *.vsidx
 *.lock
+/.vs

include/bitcoin/database/impl/query/confirm.ipp

@@ -338,31 +338,31 @@ error::error_t CLASS::spent_prevout(const point_link& link, index index,
 
 // protected
 TEMPLATE
-error::error_t CLASS::unspendable_prevout(const point_link& link,
-    uint32_t sequence, uint32_t version, const context& ctx) const NOEXCEPT
+error::error_t CLASS::unspendable_prevout(uint32_t sequence, bool coinbase,
+    const tx_link& prevout_tx, uint32_t version,
+    const context& ctx) const NOEXCEPT
 {
-    // TODO: If unconfirmed_spend is encountered, perform a search (free).
-    // It's not possible for a confirmed spend to be the wrong tx instance.
-    // This eliminates the hash lookup and to_strong(hash) iteration.
-
     // TODO: don't need to return tx link here, just the block (for strong/context).
     // MOOT: get_point_key(link) is redundant with spent_prevout().
     // to_strong has the only searches [tx.iterate, strong.find].
-    const auto strong_prevout = to_strong(get_point_key(link));
+    ////const auto strong_prevout = to_strong(get_point_key(link));
+    ////
+    ////// prevout is strong if present.
+    ////if (strong_prevout.block.is_terminal())
+    ////    return strong_prevout.tx.is_terminal() ?
+    ////        error::missing_previous_output : error::unconfirmed_spend;
 
-    // prevout is strong if present.
-    if (strong_prevout.block.is_terminal())
-        return strong_prevout.tx.is_terminal() ?
-            error::missing_previous_output : error::unconfirmed_spend;
+    // TODO: If unconfirmed_spend is encountered, perform a search (free).
+    // It's not possible for a confirmed spend to be the wrong tx instance.
+    // This eliminates the hash lookup and to_strong(hash) iteration.
+    const auto block = to_block(prevout_tx);
 
     context out{};
-    if (!get_context(out, strong_prevout.block))
+    if (!get_context(out, block))
         return error::integrity;
 
-    // All txs with same hash must be coinbase or not, so this query is redundant.
-    // TODO: Just use the cached value for the prevout, obtained in validation.
-    if (is_coinbase(strong_prevout.tx) &&
-        !transaction::is_coinbase_mature(out.height, ctx.height))
+    // All txs with same hash must be coinbase or not.
+    if (coinbase && !transaction::is_coinbase_mature(out.height, ctx.height))
         return error::coinbase_maturity;
 
     if (ctx.is_enabled(system::chain::flags::bip68_rule) &&
@@ -387,15 +387,10 @@ code CLASS::unspent_duplicates(const header_link& link,
     // [txs.find, {tx.iterate}, strong_tx.it]
     auto coinbases = to_strong_txs(get_tx_key(to_coinbase(link)));
 
-    // Found only this block's coinbase instance, no duplicates.
-    if (is_one(coinbases.size()))
+    // Current block is not set strong.
+    if (is_zero(coinbases.size()))
         return error::success;
 
-    // Remove self (will be not found if current block is not set_strong).
-    const auto self = std::find(coinbases.begin(), coinbases.end(), link);
-    if (self == coinbases.end() || coinbases.erase(self) == coinbases.end())
-        return error::integrity;
-
     // [point.first, is_spent_prevout()]
     const auto spent = [this](const auto& tx) NOEXCEPT
     {
@@ -502,16 +497,34 @@ spend_sets CLASS::to_spend_sets(const header_link& link) const NOEXCEPT
     if (txs.empty())
         return {};
 
-    spend_sets out{ txs.size() };
+    spend_sets sets{ txs.size() };
     const auto to_set = [this](const auto& tx) NOEXCEPT
     {
         return to_spend_set(tx);
     };
 
     // C++17 incomplete on GCC/CLang, so presently parallel only on MSVC++.
-    std_transform(bc::par_unseq, txs.begin(), txs.end(), out.begin(), to_set);
+    std_transform(bc::par_unseq, txs.begin(), txs.end(), sets.begin(), to_set);
+
+    const auto count = [](size_t total, const auto& set) NOEXCEPT
+    {
+        return system::ceilinged_add(total, set.spends.size());
+    };
+    const auto spends = std::accumulate(sets.begin(), sets.end(), zero, count);
+
+    table::prevout::record_get prevouts{};
+    prevouts.values.resize(spends);
+    store_.prevout.at(link, prevouts);
 
-    return out;
+    size_t index{};
+    for (auto& set: sets)
+        for (auto& spend: set.spends)
+        {
+            spend.coinbase = prevouts.coinbase(index);
+            spend.prevout_tx_fk = prevouts.output_tx_fk(index++);
+        }
+
+    return sets;
 }
 
 // split(3) 219 secs for 400k-410k; split(2) 255 and split(0) 456 (not shown).
@@ -526,8 +539,7 @@ code CLASS::block_confirmable(const header_link& link) const NOEXCEPT
     code ec{};
     if ((ec = unspent_duplicates(link, ctx)))
         return ec;
-    
-    // This is eliminated by caching, since each non-internal spend is cached.
+
     const auto sets = to_spend_sets(link);
     if (sets.empty())
         return ec;
@@ -536,27 +548,39 @@ code CLASS::block_confirmable(const header_link& link) const NOEXCEPT
 
     const auto is_unspendable = [this, &ctx, &fault](const auto& set) NOEXCEPT
     {
+        // TODO: prevout table optimized, evaluate.
         error::error_t ec{};
         for (const auto& spend: set.spends)
-            if ((ec = unspendable_prevout(spend.point_fk, spend.sequence,
-                set.version, ctx)))
+        {
+            if (spend.prevout_tx_fk == table::prevout::tx::terminal)
+                continue;
+
+            if ((ec = unspendable_prevout(spend.sequence, spend.coinbase,
+                    spend.prevout_tx_fk, set.version, ctx)))
             {
                 fault.store(ec);
                 return true;
             }
+        }
 
         return false;
     };
 
     const auto is_spent = [this, &fault](const auto& set) NOEXCEPT
     {
+        // TODO: point table optimize via consolidation with spend table.
         error::error_t ec{};
-        for (const auto& spend: set.spends)
+        for (const spend_set::spend& spend: set.spends)
+        {
+            if (spend.prevout_tx_fk == table::prevout::tx::terminal)
+                continue;
+
             if ((ec = spent_prevout(spend.point_fk, spend.point_index, set.tx)))
             {
                 fault.store(ec);
                 return true;
             }
+        }
 
         return false;
     };
@@ -708,7 +732,7 @@ bool CLASS::set_unstrong(const header_link& link) NOEXCEPT
 }
 
 TEMPLATE
-bool CLASS::set_prevouts(size_t height, const block& block) NOEXCEPT
+bool CLASS::set_prevouts(const header_link& link, const block& block) NOEXCEPT
 {
     // Empty or coinbase only implies no spends.
     if (block.transactions() <= one)
@@ -719,7 +743,7 @@ bool CLASS::set_prevouts(size_t height, const block& block) NOEXCEPT
 
     // Clean single allocation failure (e.g. disk full).
     const table::prevout::record_put_ref prevouts{ {}, block };
-    return store_.prevout.put(height, prevouts);
+    return store_.prevout.put(link, prevouts);
     // ========================================================================
 }
 

include/bitcoin/database/impl/query/optional.ipp

@@ -198,13 +198,38 @@ bool CLASS::get_filter_head(hash_digest& out,
     return true;
 }
 
+TEMPLATE
+bool CLASS::set_filter_head(const header_link& link) NOEXCEPT
+{
+    if (!neutrino_enabled())
+        return true;
+
+    // The filter body must have been previously stored under the block link.
+    filter bytes{};
+    if (!get_filter_body(bytes, link))
+        return false;
+
+    // If genesis then previous is null_hash otherwise get by confirmed height.
+    hash_digest previous{};
+    const auto height = get_height(link);
+    if (!is_zero(height))
+        if (!get_filter_head(previous, to_confirmed(sub1(height))))
+            return false;
+
+    // Use the previous head and current body to compute the current head.
+    return set_filter_head(link,
+        system::neutrino::compute_filter_header(previous, bytes));
+}
+
 TEMPLATE
 bool CLASS::set_filter_body(const header_link& link,
     const block& block) NOEXCEPT
 {
-    filter bytes{};
+    if (!neutrino_enabled())
+        return true;
 
-    // compute_filter is only false if prevouts are missing.
+    // Compute the current filter from the block and store under the link.
+    filter bytes{};
     return system::neutrino::compute_filter(bytes, block) &&
         set_filter_body(link, bytes);
 }

include/bitcoin/database/impl/query/translate.ipp

@@ -506,9 +506,11 @@ spend_set CLASS::to_spend_set(const tx_link& link) const NOEXCEPT
         // Translate query to public struct.
 #if defined(HAVE_CLANG)
         // emplace_back aggregate initialization requires clang 16.
-        set.spends.push_back({ get.point_fk, get.point_index, get.sequence });
+        set.spends.push_back({ get.point_fk, get.point_index, get.sequence,
+            table::prevout::tx::integer{}, bool{} });
 #else
-        set.spends.emplace_back(get.point_fk, get.point_index, get.sequence);
+        set.spends.emplace_back(get.point_fk, get.point_index, get.sequence,
+            table::prevout::tx::integer{}, bool{});
 #endif
     }
 

include/bitcoin/database/impl/query/validate.ipp

@@ -196,7 +196,6 @@ code CLASS::get_block_state(const header_link& link) const NOEXCEPT
     if (!store_.validated_bk.find(link, valid))
         return is_associated(link) ? error::unvalidated : error::unassociated;
 
-    // Fees only valid if block_confirmable.
     return to_block_code(valid.code);
 }
 
@@ -208,8 +207,9 @@ code CLASS::get_block_state(uint64_t& fees,
     if (!store_.validated_bk.find(link, valid))
         return is_associated(link) ? error::unvalidated : error::unassociated;
 
-    // Fees only valid if block_confirmable.
+    // TODO: Fees only valid if block_valid is the current state (iterate for valid).
     fees = valid.fees;
+
     return to_block_code(valid.code);
 }
 

include/bitcoin/database/query.hpp

@@ -67,9 +67,14 @@ struct spend_set
             return point_fk == table::spend::pt::terminal;
         }
 
+        // From tx input.
         table::spend::pt::integer point_fk{};
         table::spend::ix::integer point_index{};
         uint32_t sequence{};
+
+        // From prevouts table.
+        table::prevout::tx::integer prevout_tx_fk{};
+        bool coinbase{};
     };
 
     tx_link tx{};
@@ -507,7 +512,7 @@ class query
     /// Block association relies on strong (confirmed or pending).
     bool set_strong(const header_link& link) NOEXCEPT;
     bool set_unstrong(const header_link& link) NOEXCEPT;
-    bool set_prevouts(size_t height, const block& block) NOEXCEPT;
+    bool set_prevouts(const header_link& link, const block& block) NOEXCEPT;
     code block_confirmable(const header_link& link) const NOEXCEPT;
     ////code tx_confirmable(const tx_link& link, const context& ctx) const NOEXCEPT;
     code unspent_duplicates(const header_link& coinbase,
@@ -538,6 +543,7 @@ class query
     bool get_filter_head(hash_digest& out, const header_link& link) const NOEXCEPT;
     bool set_filter_body(const header_link& link, const block& block) NOEXCEPT;
     bool set_filter_body(const header_link& link, const filter& body) NOEXCEPT;
+    bool set_filter_head(const header_link& link) NOEXCEPT;
     bool set_filter_head(const header_link& link,
         const hash_digest& head) NOEXCEPT;
 
@@ -581,8 +587,8 @@ class query
         const tx_link& self=tx_link::terminal) const NOEXCEPT;
     error::error_t spent_prevout(const point_link& link, index index,
         const tx_link& self=tx_link::terminal) const NOEXCEPT;
-    error::error_t unspendable_prevout(const point_link& link,
-        uint32_t sequence, uint32_t version,
+    error::error_t unspendable_prevout(uint32_t sequence, bool coinbase,
+        const tx_link& prevout_tx, uint32_t version,
         const context& ctx) const NOEXCEPT;
     bool set_strong(const header_link& link, const tx_links& txs,
         bool positive) NOEXCEPT;

include/bitcoin/database/tables/caches/prevout.hpp

@@ -28,16 +28,18 @@ namespace libbitcoin {
 namespace database {
 namespace table {
 
-/// prevout is an array map index of previous outputs by block.
+/// prevout is an array map index of previous outputs by block link.
 /// The coinbase flag is merged into the tx field, reducing it's domain.
 /// Masking is from the right in order to accomodate non-integral domain.
 struct prevout
   : public array_map<schema::prevout>
 {
     using tx = linkage<schema::tx>;
+    using header = linkage<schema::block>;
     using array_map<schema::prevout>::arraymap;
     static constexpr size_t offset = sub1(to_bits(tx::size));
 
+    // This supports only a single record (not too useful).
     struct record
       : public schema::prevout
     {
@@ -61,14 +63,14 @@ struct prevout
         inline bool from_data(reader& source) NOEXCEPT
         {
             value = source.read_little_endian<tx::integer, tx::size>();
-            BC_ASSERT(!source || source.get_read_position() == minrow);
+            BC_ASSERT(!source || source.get_read_position() == count() * minrow);
             return source;
         }
 
         inline bool to_data(finalizer& sink) const NOEXCEPT
         {
             sink.write_little_endian<tx::integer, tx::size>(value);
-            BC_ASSERT(!sink || sink.get_write_position() == minrow);
+            BC_ASSERT(!sink || sink.get_write_position() == count() * minrow);
             return sink;
         }
 
@@ -140,6 +142,10 @@ struct prevout
 
         inline bool from_data(reader& source) NOEXCEPT
         {
+            // TODO: Can be optimized using unsafe_array_cast copy, as long as
+            // TODO: endianness lines up. Should have writer methods for native
+            // TODO: endianness so that both big and little are optimal. But
+            // TODO: this would prevent store portability across endianness.
             // Values must be set to read size (i.e. using knowledge of spends).
             std::for_each(values.begin(), values.end(), [&](auto& value) NOEXCEPT
             {