Merge pull request #6193

6e71efa [REST] remove json input for getutxos, limit to query max. 15 outpoints (Jonas Schnelli)
64b8027 rest.cpp: strip whitespace (Jonas Schnelli)

Author: laanwj

doc/REST-interface.md

@@ -47,15 +47,15 @@ Only supports JSON as output format.
 * chainwork : (string) total amount of work in active chain, in hexadecimal
 
 ####Query UTXO set
-`GET /rest/getutxos.<bin|hex|json>`
+`GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>`
 
 The getutxo command allows querying of the UTXO set given a set of outpoints.
 See BIP64 for input and output serialisation:
 https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki
 
 Example:
 ```
-$ curl --data '{"checkmempool":true,"outpoints":[{"txid":"b2cdfd7b89def827ff8af7cd9bff7627ff72e5e8b0f71210f92ea7a4000c5d75","n":0}]}' localhost:18332/rest/getutxos.json 2>/dev/null | json_pp
+$ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff7627ff72e5e8b0f71210f92ea7a4000c5d75-0.json 2>/dev/null | json_pp
 {
    "chaintipHash" : "00000000fb01a7f3745a717f8caebee056c484e6e0bfe4a9591c235bb70506fb",
    "chainHeight" : 325347,

qa/rpc-tests/rest.py

@@ -55,78 +55,78 @@ def setup_network(self, split=False):
         connect_nodes_bi(self.nodes,0,2)
         self.is_network_split=False
         self.sync_all()
-        
+
     def run_test(self):
         url = urlparse.urlparse(self.nodes[0].url)
         print "Mining blocks..."
-        
+
         self.nodes[0].generate(1)
         self.sync_all()
         self.nodes[2].generate(100)
         self.sync_all()
-        
+
         assert_equal(self.nodes[0].getbalance(), 50)
-        
+
         txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
         self.sync_all()
         self.nodes[2].generate(1)
         self.sync_all()
         bb_hash = self.nodes[0].getbestblockhash()
-        
+
         assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) #balance now should be 0.1 on node 1
-        
+
         # load the latest 0.1 tx over the REST API
         json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json")
         json_obj = json.loads(json_string)
         vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then)
-        # get n of 0.1 outpoint 
+        # get n of 0.1 outpoint
         n = 0
         for vout in json_obj['vout']:
             if vout['value'] == 0.1:
                 n = vout['n']
-        
-        
+
+
         ######################################
         # GETUTXOS: query a unspent outpoint #
         ######################################
-        json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}'
-        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request)
+        json_request = '/checkmempool/'+txid+'-'+str(n)
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
-        
+
         #check chainTip response
         assert_equal(json_obj['chaintipHash'], bb_hash)
-        
+
         #make sure there is one utxo
         assert_equal(len(json_obj['utxos']), 1)
         assert_equal(json_obj['utxos'][0]['value'], 0.1)
-        
-        
+
+
         ################################################
         # GETUTXOS: now query a already spent outpoint #
         ################################################
-        json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+vintx+'","n":0}]}'
-        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request)
+        json_request = '/checkmempool/'+vintx+'-0'
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
-        
+
         #check chainTip response
         assert_equal(json_obj['chaintipHash'], bb_hash)
 
         #make sure there is no utox in the response because this oupoint has been spent
         assert_equal(len(json_obj['utxos']), 0)
-        
+
         #check bitmap
         assert_equal(json_obj['bitmap'], "0")
-        
-        
+
+
         ##################################################
         # GETUTXOS: now check both with the same request #
         ##################################################
-        json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'},{"txid":"'+vintx+'","n":0}]}'
-        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request)
+        json_request = '/checkmempool/'+txid+'-'+str(n)+'/'+vintx+'-0'
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
         assert_equal(len(json_obj['utxos']), 1)
         assert_equal(json_obj['bitmap'], "10")
-        
+
         #test binary response
         bb_hash = self.nodes[0].getbestblockhash()
 
@@ -135,19 +135,18 @@ def run_test(self):
         binaryRequest += pack("i", n);
         binaryRequest += binascii.unhexlify(vintx);
         binaryRequest += pack("i", 0);
-        
+
         bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest)
-        
         output = StringIO.StringIO()
         output.write(bin_response)
         output.seek(0)
         chainHeight = unpack("i", output.read(4))[0]
         hashFromBinResponse = hex(deser_uint256(output))[2:].zfill(65).rstrip("L")
-        
+
         assert_equal(bb_hash, hashFromBinResponse) #check if getutxo's chaintip during calculation was fine
         assert_equal(chainHeight, 102) #chain height must be 102
-        
-        
+
+
         ############################
         # GETUTXOS: mempool checks #
         ############################
@@ -157,55 +156,56 @@ def run_test(self):
         json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json")
         json_obj = json.loads(json_string)
         vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then)
-        # get n of 0.1 outpoint 
+        # get n of 0.1 outpoint
         n = 0
         for vout in json_obj['vout']:
             if vout['value'] == 0.1:
                 n = vout['n']
-        
-        json_request = '{"checkmempool":false,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}'
-        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request)
+
+        json_request = '/'+txid+'-'+str(n)
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
         assert_equal(len(json_obj['utxos']), 0) #there should be a outpoint because it has just added to the mempool
-        
-        json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}'
-        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request)
+
+        json_request = '/checkmempool/'+txid+'-'+str(n)
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
         assert_equal(len(json_obj['utxos']), 1) #there should be a outpoint because it has just added to the mempool
-        
+
         #do some invalid requests
         json_request = '{"checkmempool'
         response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)
         assert_equal(response.status, 500) #must be a 500 because we send a invalid json request
-        
+
         json_request = '{"checkmempool'
         response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True)
         assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request
-        
+
+        response = http_get_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True)
+        assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request
+
         #test limits
-        json_request = '{"checkmempool":true,"outpoints":['
-        for x in range(0, 200):
-            json_request += '{"txid":"'+txid+'","n":'+str(n)+'},'
-        json_request = json_request.rstrip(",")
-        json_request+="]}";
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)
+        json_request = '/checkmempool/'
+        for x in range(0, 20):
+            json_request += txid+'-'+str(n)+'/'
+        json_request = json_request.rstrip("/")
+        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
         assert_equal(response.status, 500) #must be a 500 because we exceeding the limits
-        
-        json_request = '{"checkmempool":true,"outpoints":['
-        for x in range(0, 90):
-            json_request += '{"txid":"'+txid+'","n":'+str(n)+'},'
-        json_request = json_request.rstrip(",")
-        json_request+="]}";
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)
+
+        json_request = '/checkmempool/'
+        for x in range(0, 15):
+            json_request += txid+'-'+str(n)+'/'
+        json_request = json_request.rstrip("/");
+        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
         assert_equal(response.status, 200) #must be a 500 because we exceeding the limits
 
         self.nodes[0].generate(1) #generate block to not affect upcomming tests
         self.sync_all()
-        
+
         ################
         # /rest/block/ #
         ################
-        
+
         # check binary format
         response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True)
         assert_equal(response.status, 200)
@@ -249,7 +249,7 @@ def run_test(self):
         hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True)
         assert_equal(hex_string.status, 200)
         assert_greater_than(int(response.getheader('content-length')), 10)
-        
+
 
 
         # check block tx details
@@ -279,7 +279,7 @@ def run_test(self):
 
         #test rest bestblock
         bb_hash = self.nodes[0].getbestblockhash()
-        
+
         json_string = http_get_call(url.hostname, url.port, '/rest/chaininfo.json')
         json_obj = json.loads(json_string)
         assert_equal(json_obj['bestblockhash'], bb_hash)

src/rest.cpp

@@ -19,7 +19,7 @@
 using namespace std;
 using namespace json_spirit;
 
-static const int MAX_GETUTXOS_OUTPOINTS = 100; //allow a max of 100 outpoints to be queried at once
+static const int MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once
 
 enum RetFormat {
     RF_UNDEF,
@@ -262,12 +262,12 @@ static bool rest_chaininfo(AcceptedConnection* conn,
 {
     vector<string> params;
     const RetFormat rf = ParseDataFormat(params, strURIPart);
-    
+
     switch (rf) {
     case RF_JSON: {
         Array rpcParams;
         Value chainInfoObject = getblockchaininfo(rpcParams, false);
-        
+
         string strJSON = write_string(chainInfoObject, false) + "\n";
         conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush;
         return true;
@@ -276,7 +276,7 @@ static bool rest_chaininfo(AcceptedConnection* conn,
         throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)");
     }
     }
-    
+
     // not reached
     return true; // continue to process further HTTP reqs on this cxn
 }
@@ -342,18 +342,53 @@ static bool rest_getutxos(AcceptedConnection* conn,
     vector<string> params;
     enum RetFormat rf = ParseDataFormat(params, strURIPart);
 
+    vector<string> uriParts;
+    if (params.size() > 0 && params[0].length() > 1)
+    {
+        std::string strUriParams = params[0].substr(1);
+        boost::split(uriParts, strUriParams, boost::is_any_of("/"));
+    }
+
     // throw exception in case of a empty request
-    if (strRequest.length() == 0)
+    if (strRequest.length() == 0 && uriParts.size() == 0)
         throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request");
 
+    bool fInputParsed = false;
     bool fCheckMemPool = false;
     vector<COutPoint> vOutPoints;
 
     // parse/deserialize input
     // input-format = output-format, rest/getutxos/bin requires binary input, gives binary output, ...
-    
+
+    if (uriParts.size() > 0)
+    {
+
+        //inputs is sent over URI scheme (/rest/getutxos/checkmempool/txid1-n/txid2-n/...)
+        if (uriParts.size() > 0 && uriParts[0] == "checkmempool")
+            fCheckMemPool = true;
+
+        for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++)
+        {
+            uint256 txid;
+            int32_t nOutput;
+            std::string strTxid = uriParts[i].substr(0, uriParts[i].find("-"));
+            std::string strOutput = uriParts[i].substr(uriParts[i].find("-")+1);
+
+            if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid))
+                throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error");
+
+            txid.SetHex(strTxid);
+            vOutPoints.push_back(COutPoint(txid, (uint32_t)nOutput));
+        }
+
+        if (vOutPoints.size() > 0)
+            fInputParsed = true;
+        else
+            throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request");
+    }
+
     string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting
-    
+
     switch (rf) {
     case RF_HEX: {
         // convert hex to bin, continue then with bin part
@@ -363,11 +398,17 @@ static bool rest_getutxos(AcceptedConnection* conn,
 
     case RF_BINARY: {
         try {
-            //deserialize
-            CDataStream oss(SER_NETWORK, PROTOCOL_VERSION);
-            oss << strRequestMutable;
-            oss >> fCheckMemPool;
-            oss >> vOutPoints;
+            //deserialize only if user sent a request
+            if (strRequestMutable.size() > 0)
+            {
+                if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA
+                    throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed");
+
+                CDataStream oss(SER_NETWORK, PROTOCOL_VERSION);
+                oss << strRequestMutable;
+                oss >> fCheckMemPool;
+                oss >> vOutPoints;
+            }
         } catch (const std::ios_base::failure& e) {
             // abort in case of unreadable binary data
             throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error");
@@ -376,33 +417,8 @@ static bool rest_getutxos(AcceptedConnection* conn,
     }
 
     case RF_JSON: {
-        try {
-            // parse json request
-            Value valRequest;
-            if (!read_string(strRequest, valRequest))
-                throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error");
-
-            Object jsonObject = valRequest.get_obj();
-            const Value& checkMempoolValue = find_value(jsonObject, "checkmempool");
-
-            if (!checkMempoolValue.is_null()) {
-                fCheckMemPool = checkMempoolValue.get_bool();
-            }
-            const Value& outpointsValue = find_value(jsonObject, "outpoints");
-            if (!outpointsValue.is_null()) {
-                Array outPoints = outpointsValue.get_array();
-                BOOST_FOREACH (const Value& outPoint, outPoints) {
-                    Object outpointObject = outPoint.get_obj();
-                    uint256 txid = ParseHashO(outpointObject, "txid");
-                    Value nValue = find_value(outpointObject, "n");
-                    int nOutput = nValue.get_int();
-                    vOutPoints.push_back(COutPoint(txid, nOutput));
-                }
-            }
-        } catch (...) {
-            // return HTTP 500 if there was a json parsing error
-            throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error");
-        }
+        if (!fInputParsed)
+            throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request");
         break;
     }
     default: {