Merge pull request #931 from evoskuil/master

Update explore queries and integrated read test.

Author: evoskuil

console/executor_test_reader.cpp

@@ -36,11 +36,11 @@ void executor::read_test(const hash_digest&) const
 {
     logger("Wire size computation.");
     const auto start = fine_clock::now();
-    const auto last = metadata_.configured.node.maximum_height_();
+    const auto top = query_.get_top_associated();
     const auto concurrency = metadata_.configured.node.maximum_concurrency_();
 
     size_t size{};
-    for (auto height = zero; !cancel_ && height <= last; ++height)
+    for (auto height = zero; !cancel_ && height <= top; ++height)
     {
         const auto link = query_.to_candidate(height);
         if (link.is_terminal())
@@ -64,6 +64,10 @@ void executor::read_test(const hash_digest&) const
                 size % height % span.count());
         }
     }
+
+    const auto span = duration_cast<milliseconds>(fine_clock::now() - start);
+    logger(format("Wire size (%1%) at (%2%) in (%3%) ms.") %
+        size % top % span.count());
 }
 
 #if defined(UNDEFINED)

include/bitcoin/node/interfaces/explore.hpp

@@ -33,16 +33,17 @@ struct explore_methods
         method<"top", uint8_t, uint8_t>{ "version", "media" },
         method<"block", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>, optional<true>>{ "version", "media", "hash", "height", "witness" },
         method<"block_header", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "hash", "height" },
+        method<"block_header_context", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "hash", "height" },
+        method<"block_details", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "hash", "height" },
         method<"block_txs", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "hash", "height" },
-        method<"block_fees", uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "hash", "height" },
         method<"block_filter", uint8_t, uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "type", "hash", "height" },
         method<"block_filter_hash", uint8_t, uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "type", "hash", "height" },
         method<"block_filter_header", uint8_t, uint8_t, uint8_t, nullable<system::hash_cptr>, nullable<uint32_t>>{ "version", "media", "type", "hash", "height" },
         method<"block_tx", uint8_t, uint8_t, uint32_t, nullable<system::hash_cptr>, nullable<uint32_t>, optional<true>>{ "version", "media", "position", "hash", "height", "witness" },
 
         method<"tx", uint8_t, uint8_t, system::hash_cptr, optional<true>>{ "version", "media", "hash", "witness" },
         method<"tx_header", uint8_t, uint8_t, system::hash_cptr>{ "version", "media", "hash" },
-        method<"tx_fee", uint8_t, uint8_t, system::hash_cptr>{ "version", "media", "hash" },
+        method<"tx_details", uint8_t, uint8_t, system::hash_cptr>{ "version", "media", "hash" },
 
         method<"inputs", uint8_t, uint8_t, system::hash_cptr, optional<true>>{ "version", "media", "hash", "witness" },
         method<"input", uint8_t, uint8_t, system::hash_cptr, uint32_t, optional<true>>{ "version", "media", "hash", "index", "witness" },
@@ -73,32 +74,33 @@ struct explore_methods
 
     using block = at<1>;
     using block_header = at<2>;
-    using block_txs = at<3>;
-    using block_fees = at<4>;
-    using block_filter = at<5>;
-    using block_filter_hash = at<6>;
-    using block_filter_header = at<7>;
-    using block_tx = at<8>;
-
-    using tx = at<9>;
-    using tx_header = at<10>;
-    using tx_fee = at<11>;
-
-    using inputs = at<12>;
-    using input = at<13>;
-    using input_script = at<14>;
-    using input_witness = at<15>;
-
-    using outputs = at<16>;
-    using output = at<17>;
-    using output_script = at<18>;
-    using output_spender = at<19>;
-    using output_spenders = at<20>;
-
-    using address = at<21>;
-    using address_confirmed = at<22>;
-    using address_unconfirmed = at<23>;
-    using address_balance = at<24>;
+    using block_header_context = at<3>;
+    using block_details = at<4>;
+    using block_txs = at<5>;
+    using block_filter = at<6>;
+    using block_filter_hash = at<7>;
+    using block_filter_header = at<8>;
+    using block_tx = at<9>;
+
+    using tx = at<10>;
+    using tx_header = at<11>;
+    using tx_details = at<12>;
+
+    using inputs = at<13>;
+    using input = at<14>;
+    using input_script = at<15>;
+    using input_witness = at<16>;
+
+    using outputs = at<17>;
+    using output = at<18>;
+    using output_script = at<19>;
+    using output_spender = at<20>;
+    using output_spenders = at<21>;
+
+    using address = at<22>;
+    using address_confirmed = at<23>;
+    using address_unconfirmed = at<24>;
+    using address_balance = at<25>;
 };
 
 /// ?format=data|text|json (via query string).

include/bitcoin/node/protocols/protocol_explore.hpp

@@ -72,10 +72,13 @@ class BCN_API protocol_explore
     bool handle_get_block_header(const code& ec, interface::block_header,
         uint8_t version, uint8_t media, std::optional<system::hash_cptr> hash,
         std::optional<uint32_t> height) NOEXCEPT;
-    bool handle_get_block_txs(const code& ec, interface::block_txs,
+    bool handle_get_block_header_context(const code& ec, interface::block_header_context,
+        uint8_t version, uint8_t media, std::optional<system::hash_cptr> hash,
+        std::optional<uint32_t> height) NOEXCEPT;
+    bool handle_get_block_details(const code& ec, interface::block_details,
         uint8_t version, uint8_t media, std::optional<system::hash_cptr> hash,
         std::optional<uint32_t> height) NOEXCEPT;
-    bool handle_get_block_fees(const code& ec, interface::block_fees,
+    bool handle_get_block_txs(const code& ec, interface::block_txs,
         uint8_t version, uint8_t media, std::optional<system::hash_cptr> hash,
         std::optional<uint32_t> height) NOEXCEPT;
     bool handle_get_block_filter(const code& ec, interface::block_filter,
@@ -101,7 +104,7 @@ class BCN_API protocol_explore
     bool handle_get_tx_header(const code& ec, interface::tx_header,
         uint8_t version, uint8_t media,
         const system::hash_cptr& hash) NOEXCEPT;
-    bool handle_get_tx_fee(const code& ec, interface::tx_fee,
+    bool handle_get_tx_details(const code& ec, interface::tx_details,
         uint8_t version, uint8_t media,
         const system::hash_cptr& hash) NOEXCEPT;
 

src/parsers/explore_target.cpp

@@ -209,8 +209,8 @@ code explore_target(request_t& out, const std::string_view& path) NOEXCEPT
             const auto component = segments[segment++];
             if (component == "header")
                 method = "tx_header";
-            else if (component == "fee")
-                method = "tx_fee";
+            else if (component == "details")
+                method = "tx_details";
             else
                 return error::invalid_component;
         }
@@ -267,11 +267,24 @@ code explore_target(request_t& out, const std::string_view& path) NOEXCEPT
                 method = "block_tx";
             }
             else if (component == "header")
-                method = "block_header";
+            {
+                if (segment == segments.size())
+                {
+                    method = "block_header";
+                }
+                else
+                {
+                    const auto subcomponent = segments[segment++];
+                    if (subcomponent == "context")
+                        method = "block_header_context";
+                    else
+                        return error::invalid_subcomponent;
+                }
+            }
             else if (component == "txs")
                 method = "block_txs";
-            else if (component == "fees")
-                method = "block_fees";
+            else if (component == "details")
+                method = "block_details";
             else if (component == "filter")
             {
                 if (segment == segments.size())

src/protocols/protocol_explore.cpp

@@ -84,16 +84,17 @@ void protocol_explore::start() NOEXCEPT
 
     SUBSCRIBE_EXPLORE(handle_get_block, _1, _2, _3, _4, _5, _6, _7);
     SUBSCRIBE_EXPLORE(handle_get_block_header, _1, _2, _3, _4, _5, _6);
+    SUBSCRIBE_EXPLORE(handle_get_block_header_context, _1, _2, _3, _4, _5, _6);
+    SUBSCRIBE_EXPLORE(handle_get_block_details, _1, _2, _3, _4, _5, _6);
     SUBSCRIBE_EXPLORE(handle_get_block_txs, _1, _2, _3, _4, _5, _6);
-    SUBSCRIBE_EXPLORE(handle_get_block_fees, _1, _2, _3, _4, _5, _6);
     SUBSCRIBE_EXPLORE(handle_get_block_filter, _1, _2, _3, _4, _5, _6, _7);
     SUBSCRIBE_EXPLORE(handle_get_block_filter_hash, _1, _2, _3, _4, _5, _6, _7);
     SUBSCRIBE_EXPLORE(handle_get_block_filter_header, _1, _2, _3, _4, _5, _6, _7);
     SUBSCRIBE_EXPLORE(handle_get_block_tx, _1, _2, _3, _4, _5, _6, _7, _8);
 
     SUBSCRIBE_EXPLORE(handle_get_tx, _1, _2, _3, _4, _5, _6);
     SUBSCRIBE_EXPLORE(handle_get_tx_header, _1, _2, _3, _4, _5);
-    SUBSCRIBE_EXPLORE(handle_get_tx_fee, _1, _2, _3, _4, _5);
+    SUBSCRIBE_EXPLORE(handle_get_tx_details, _1, _2, _3, _4, _5);
 
     SUBSCRIBE_EXPLORE(handle_get_inputs, _1, _2, _3, _4, _5, _6);
     SUBSCRIBE_EXPLORE(handle_get_input, _1, _2, _3, _4, _5, _6, _7);
@@ -318,56 +319,81 @@ bool protocol_explore::handle_get_block_header(const code& ec,
     return true;
 }
 
-bool protocol_explore::handle_get_block_txs(const code& ec,
-    interface::block_txs, uint8_t, uint8_t media,
+bool protocol_explore::handle_get_block_header_context(const code& ec,
+    interface::block_header_context, uint8_t, uint8_t media,
     std::optional<hash_cptr> hash, std::optional<uint32_t> height) NOEXCEPT
 {
     if (stopped(ec))
         return false;
 
+    // states:
+    // block_valid
+    // block_confirmable
+    // block_unconfirmable
+    // get_header_state->unvalidated can be no header or no txs.
+    ////const auto state = query.get_header_state(link);
+    ////if (state == database::error::unvalidated)
+    ////{
+    ////    send_not_found();
+    ////    return true;
+    ////}
+
     const auto& query = archive();
-    if (const auto hashes = query.get_tx_keys(to_header(height, hash));
-        !hashes.empty())
+    const auto link = to_header(height, hash);
+    database::context context{};
+    if (query.get_context(context, link))
     {
-        const auto size = hashes.size() * hash_size;
         switch (media)
         {
             case data:
-            {
-                const auto data = pointer_cast<const uint8_t>(hashes.data());
-                send_chunk(to_chunk({ data, std::next(data, size) }));
+                send_chunk(to_little_endian_size(context.flags));
                 return true;
-            }
             case text:
-            {
-                const auto data = pointer_cast<const uint8_t>(hashes.data());
-                send_text(encode_base16({ data, std::next(data, size) }));
+                send_text(encode_base16(to_little_endian_size(context.flags)));
                 return true;
-            }
             case json:
-            {
-                array out(hashes.size());
-                std::ranges::transform(hashes, out.begin(),
-                    [](const auto& hash) { return encode_hash(hash); });
-                send_json(out, two * size);
+                send_json(context.flags, two * sizeof(context.flags));
                 return true;
-            }
         }
     }
 
     send_not_found();
     return true;
 }
 
-bool protocol_explore::handle_get_block_fees(const code& ec,
-    interface::block_fees, uint8_t, uint8_t media,
+bool protocol_explore::handle_get_block_details(const code& ec,
+    interface::block_details, uint8_t, uint8_t media,
     std::optional<hash_cptr> hash, std::optional<uint32_t> height) NOEXCEPT
 {
     if (stopped(ec))
         return false;
 
-    if (const auto fees = archive().get_block_fees(to_header(height, hash));
-        fees != max_uint64)
+    const auto& query = archive();
+    const auto link = to_header(height, hash);
+    const auto state = query.get_block_state(link);
+
+    // get_block_state->unassociated can be no header or no txs.
+    if (state == database::error::unassociated)
+    {
+        send_not_found();
+        return true;
+    }
+
+    // states:
+    // unvalidated
+    // block_valid
+    // block_confirmable
+    // block_unconfirmable
+
+    // both txs table (can get from details)
+    //const auto size = query.get_block_size(link);
+    //const auto count = query.get_tx_count(link);
+
+    // TODO:
+    // query (whole block and all prevouts, same as get_block_fees)
+    // fees, claim, reward, subsidy, weight, size, count.
+
+    if (const auto fees = query.get_block_fees(link); fees != max_uint64)
     {
         switch (media)
         {
@@ -387,6 +413,47 @@ bool protocol_explore::handle_get_block_fees(const code& ec,
     return true;
 }
 
+bool protocol_explore::handle_get_block_txs(const code& ec,
+    interface::block_txs, uint8_t, uint8_t media,
+    std::optional<hash_cptr> hash, std::optional<uint32_t> height) NOEXCEPT
+{
+    if (stopped(ec))
+        return false;
+
+    const auto& query = archive();
+    if (const auto hashes = query.get_tx_keys(to_header(height, hash));
+        !hashes.empty())
+    {
+        const auto size = hashes.size() * hash_size;
+        switch (media)
+        {
+            case data:
+            {
+                const auto data = pointer_cast<const uint8_t>(hashes.data());
+                send_chunk(to_chunk({ data, std::next(data, size) }));
+                return true;
+            }
+            case text:
+            {
+                const auto data = pointer_cast<const uint8_t>(hashes.data());
+                send_text(encode_base16({ data, std::next(data, size) }));
+                return true;
+            }
+            case json:
+            {
+                array out(hashes.size());
+                std::ranges::transform(hashes, out.begin(),
+                    [](const auto& hash) { return encode_hash(hash); });
+                send_json(out, two * size);
+                return true;
+            }
+        }
+    }
+
+    send_not_found();
+    return true;
+}
+
 bool protocol_explore::handle_get_block_filter(const code& ec,
     interface::block_filter, uint8_t, uint8_t media, uint8_t type,
     std::optional<hash_cptr> hash, std::optional<uint32_t> height) NOEXCEPT
@@ -595,12 +662,14 @@ bool protocol_explore::handle_get_tx_header(const code& ec,
     return true;
 }
 
-bool protocol_explore::handle_get_tx_fee(const code& ec, interface::tx_fee,
-    uint8_t, uint8_t media, const hash_cptr& hash) NOEXCEPT
+bool protocol_explore::handle_get_tx_details(const code& ec,
+    interface::tx_details, uint8_t, uint8_t media,
+    const hash_cptr& hash) NOEXCEPT
 {
     if (stopped(ec))
         return false;
 
+    // TODO: expand details to include tx.size and tx.weight.
     const auto& query = archive();
     if (const auto fee = query.get_tx_fee(query.to_tx(*hash));
         fee != max_uint64)