Merge bitcoin/bitcoin#17631: Expose block filters over REST

2b64fa3 Update REST docs with new accessors (Matt Corallo)
ef7c822 Expose block filters over REST. (Matt Corallo)

Pull request description:

  This adds a new rest endpoint:
  /rest/blockfilter/filtertype/requesttype/blockhash (eg
  /rest/blockfilter/basic/header/000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f.hex)
  which exposes either the filter "header" or the filter data itself.
  Most of the code is cribbed from the equivalent RPC.

  You can test it at http://bitcoin-rest.bitcoin.ninja/rest//blockfilter/basic/header/000000005b7a58a939b2636f61fa4ddd62258c5fed57667a35d23f2334c4f86d.hex

ACKs for top commit:
  dergoegge:
    ACK 2b64fa3 - Adding blockfilters to the REST interface is analogous to serving other public data such as transactions or blocks.

Tree-SHA512: d487bc694266375c94d6fcf2e9d788a8a42a3b94e8d3290e46335a64cbcde55084ce5ea6119b79a4065888d94d7c3ae25a59a901fa46e3711f0eb296add12696

Author: MarcoFalke

doc/REST-interface.md

@@ -52,6 +52,20 @@ With the /notxdetails/ option JSON response will only contain the transaction ha
 Given a block hash: returns <COUNT> amount of blockheaders in upward direction.
 Returns empty if the block doesn't exist or it isn't in the active chain.
 
+#### Blockfilter Headers
+`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>`
+
+Given a block hash: returns <COUNT> amount of blockfilter headers in upward
+direction for the filter type <FILTERTYPE>.
+Returns empty if the block doesn't exist or it isn't in the active chain.
+
+#### Blockfilters
+`GET /rest/blockfilter/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>`
+
+Given a block hash: returns the block filter of the given block of type
+<FILTERTYPE>.
+Responds with 404 if the block doesn't exist.
+
 #### Blockhash by height
 `GET /rest/blockhashbyheight/<HEIGHT>.<bin|hex|json>`
 

src/rest.cpp

@@ -3,10 +3,12 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <blockfilter.h>
 #include <chain.h>
 #include <chainparams.h>
 #include <core_io.h>
 #include <httpserver.h>
+#include <index/blockfilterindex.h>
 #include <index/txindex.h>
 #include <node/blockstorage.h>
 #include <node/context.h>
@@ -31,6 +33,7 @@
 #include <univalue.h>
 
 static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once
+static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000;
 
 enum class RetFormat {
     UNDEF,
@@ -191,8 +194,8 @@ static bool rest_headers(const std::any& context,
         return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>.");
 
     const auto parsed_count{ToIntegral<size_t>(path[0])};
-    if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) {
-        return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]);
+    if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
+        return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s",  MAX_REST_HEADERS_RESULTS, path[0]));
     }
 
     std::string hashStr = path[1];
@@ -255,7 +258,7 @@ static bool rest_headers(const std::any& context,
         return true;
     }
     default: {
-        return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)");
+        return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
     }
     }
 }
@@ -338,6 +341,210 @@ static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, co
     return rest_block(context, req, strURIPart, TxVerbosity::SHOW_TXID);
 }
 
+
+static bool rest_filter_header(const std::any& context, HTTPRequest* req, const std::string& strURIPart)
+{
+    if (!CheckWarmup(req))
+        return false;
+    std::string param;
+    const RetFormat rf = ParseDataFormat(param, strURIPart);
+
+    std::vector<std::string> uri_parts;
+    boost::split(uri_parts, param, boost::is_any_of("/"));
+    if (uri_parts.size() != 3) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<count>/<blockhash>");
+    }
+
+    uint256 block_hash;
+    if (!ParseHashStr(uri_parts[2], block_hash)) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]);
+    }
+
+    BlockFilterType filtertype;
+    if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
+    }
+
+    BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
+    if (!index) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
+    }
+
+    const auto parsed_count{ToIntegral<size_t>(uri_parts[1])};
+    if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
+        return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s",  MAX_REST_HEADERS_RESULTS, uri_parts[1]));
+    }
+
+    std::vector<const CBlockIndex *> headers;
+    headers.reserve(*parsed_count);
+    {
+        ChainstateManager* maybe_chainman = GetChainman(context, req);
+        if (!maybe_chainman) return false;
+        ChainstateManager& chainman = *maybe_chainman;
+        LOCK(cs_main);
+        CChain& active_chain = chainman.ActiveChain();
+        const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block_hash);
+        while (pindex != nullptr && active_chain.Contains(pindex)) {
+            headers.push_back(pindex);
+            if (headers.size() == *parsed_count)
+                break;
+            pindex = active_chain.Next(pindex);
+        }
+    }
+
+    bool index_ready = index->BlockUntilSyncedToCurrentChain();
+
+    std::vector<uint256> filter_headers;
+    filter_headers.reserve(*parsed_count);
+    for (const CBlockIndex *pindex : headers) {
+        uint256 filter_header;
+        if (!index->LookupFilterHeader(pindex, filter_header)) {
+            std::string errmsg = "Filter not found.";
+
+            if (!index_ready) {
+                errmsg += " Block filters are still in the process of being indexed.";
+            } else {
+                errmsg += " This error is unexpected and indicates index corruption.";
+            }
+
+            return RESTERR(req, HTTP_NOT_FOUND, errmsg);
+        }
+        filter_headers.push_back(filter_header);
+    }
+
+    switch (rf) {
+    case RetFormat::BINARY: {
+        CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
+        for (const uint256& header : filter_headers) {
+            ssHeader << header;
+        }
+
+        std::string binaryHeader = ssHeader.str();
+        req->WriteHeader("Content-Type", "application/octet-stream");
+        req->WriteReply(HTTP_OK, binaryHeader);
+        return true;
+    }
+    case RetFormat::HEX: {
+        CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
+        for (const uint256& header : filter_headers) {
+            ssHeader << header;
+        }
+
+        std::string strHex = HexStr(ssHeader) + "\n";
+        req->WriteHeader("Content-Type", "text/plain");
+        req->WriteReply(HTTP_OK, strHex);
+        return true;
+    }
+    case RetFormat::JSON: {
+        UniValue jsonHeaders(UniValue::VARR);
+        for (const uint256& header : filter_headers) {
+            jsonHeaders.push_back(header.GetHex());
+        }
+
+        std::string strJSON = jsonHeaders.write() + "\n";
+        req->WriteHeader("Content-Type", "application/json");
+        req->WriteReply(HTTP_OK, strJSON);
+        return true;
+    }
+    default: {
+        return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
+    }
+    }
+}
+
+static bool rest_block_filter(const std::any& context, HTTPRequest* req, const std::string& strURIPart)
+{
+    if (!CheckWarmup(req))
+        return false;
+    std::string param;
+    const RetFormat rf = ParseDataFormat(param, strURIPart);
+
+    //request is sent over URI scheme /rest/blockfilter/filtertype/blockhash
+    std::vector<std::string> uri_parts;
+    boost::split(uri_parts, param, boost::is_any_of("/"));
+    if (uri_parts.size() != 2) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter/<filtertype>/<blockhash>");
+    }
+
+    uint256 block_hash;
+    if (!ParseHashStr(uri_parts[1], block_hash)) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[1]);
+    }
+
+    BlockFilterType filtertype;
+    if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
+    }
+
+    BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
+    if (!index) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
+    }
+
+    const CBlockIndex* block_index;
+    bool block_was_connected;
+    {
+        ChainstateManager* maybe_chainman = GetChainman(context, req);
+        if (!maybe_chainman) return false;
+        ChainstateManager& chainman = *maybe_chainman;
+        LOCK(cs_main);
+        block_index = chainman.m_blockman.LookupBlockIndex(block_hash);
+        if (!block_index) {
+            return RESTERR(req, HTTP_NOT_FOUND, uri_parts[1] + " not found");
+        }
+        block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS);
+    }
+
+    bool index_ready = index->BlockUntilSyncedToCurrentChain();
+
+    BlockFilter filter;
+    if (!index->LookupFilter(block_index, filter)) {
+        std::string errmsg = "Filter not found.";
+
+        if (!block_was_connected) {
+            errmsg += " Block was not connected to active chain.";
+        } else if (!index_ready) {
+            errmsg += " Block filters are still in the process of being indexed.";
+        } else {
+            errmsg += " This error is unexpected and indicates index corruption.";
+        }
+
+        return RESTERR(req, HTTP_NOT_FOUND, errmsg);
+    }
+
+    switch (rf) {
+    case RetFormat::BINARY: {
+        CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
+        ssResp << filter;
+
+        std::string binaryResp = ssResp.str();
+        req->WriteHeader("Content-Type", "application/octet-stream");
+        req->WriteReply(HTTP_OK, binaryResp);
+        return true;
+    }
+    case RetFormat::HEX: {
+        CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
+        ssResp << filter;
+
+        std::string strHex = HexStr(ssResp) + "\n";
+        req->WriteHeader("Content-Type", "text/plain");
+        req->WriteReply(HTTP_OK, strHex);
+        return true;
+    }
+    case RetFormat::JSON: {
+        UniValue ret(UniValue::VOBJ);
+        ret.pushKV("filter", HexStr(filter.GetEncodedFilter()));
+        std::string strJSON = ret.write() + "\n";
+        req->WriteHeader("Content-Type", "application/json");
+        req->WriteReply(HTTP_OK, strJSON);
+        return true;
+    }
+    default: {
+        return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
+    }
+    }
+}
+
 // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp
 RPCHelpMan getblockchaininfo();
 
@@ -718,6 +925,8 @@ static const struct {
       {"/rest/tx/", rest_tx},
       {"/rest/block/notxdetails/", rest_block_notxdetails},
       {"/rest/block/", rest_block_extended},
+      {"/rest/blockfilter/", rest_block_filter},
+      {"/rest/blockfilterheaders/", rest_filter_header},
       {"/rest/chaininfo", rest_chaininfo},
       {"/rest/mempool/info", rest_mempool_info},
       {"/rest/mempool/contents", rest_mempool_contents},

test/functional/interface_rest.py

@@ -41,7 +41,7 @@ class RESTTest (BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 2
-        self.extra_args = [["-rest"], []]
+        self.extra_args = [["-rest", "-blockfilterindex=1"], []]
         self.supports_cli = False
 
     def skip_test_if_missing_module(self):
@@ -272,11 +272,14 @@ def run_test(self):
         self.generate(self.nodes[1], 5)
         json_obj = self.test_rest_request(f"/headers/5/{bb_hash}")
         assert_equal(len(json_obj), 5)  # now we should have 5 header objects
+        json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")
+        assert_equal(len(json_obj), 5)  # now we should have 5 filter header objects
+        self.test_rest_request(f"/blockfilter/basic/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ)
 
         # Test number parsing
         for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']:
             assert_equal(
-                bytes(f'Header count out of range: {num}\r\n', 'ascii'),
+                bytes(f'Header count out of acceptable range (1-2000): {num}\r\n', 'ascii'),
                 self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400),
             )