Skip to content

Commit a9c56b6

Browse files
author
MarcoFalke
committed
Merge #13918: rpc: Replace median fee rate with feerate percentiles in getblockstats
4b7091a Replace median fee rate with feerate percentiles (Marcin Jachymiak) Pull request description: Currently, the `medianfeerate` statistic is calculated from the feerate of the middle transaction of a list of transactions sorted by feerate. This PR instead uses the value of the 50th percentile weight unit in the block, and also calculates the feerate at the 10th, 25th, 75th, and 90th percentiles. This more accurately corresponds with what is generally meant by median feerate. Tree-SHA512: 59255e243df90d7afbe69839408c58c9723884b8ab82c66dc24a769e89c6d539db1905374a3f025ff28272fb25a0b90e92d8101103e39a6d9c0d60423a596714
2 parents ef86f26 + 4b7091a commit a9c56b6

File tree

5 files changed

+162
-12
lines changed

5 files changed

+162
-12
lines changed

‎src/rpc/blockchain.cpp

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1640,6 +1640,35 @@ static T CalculateTruncatedMedian(std::vector<T>& scores)
16401640
}
16411641
}
16421642

1643+
void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight)
1644+
{
1645+
if (scores.empty()) {
1646+
return;
1647+
}
1648+
1649+
std::sort(scores.begin(), scores.end());
1650+
1651+
// 10th, 25th, 50th, 75th, and 90th percentile weight units.
1652+
const double weights[NUM_GETBLOCKSTATS_PERCENTILES] = {
1653+
total_weight / 10.0, total_weight / 4.0, total_weight / 2.0, (total_weight * 3.0) / 4.0, (total_weight * 9.0) / 10.0
1654+
};
1655+
1656+
int64_t next_percentile_index = 0;
1657+
int64_t cumulative_weight = 0;
1658+
for (const auto& element : scores) {
1659+
cumulative_weight += element.second;
1660+
while (next_percentile_index < NUM_GETBLOCKSTATS_PERCENTILES && cumulative_weight >= weights[next_percentile_index]) {
1661+
result[next_percentile_index] = element.first;
1662+
++next_percentile_index;
1663+
}
1664+
}
1665+
1666+
// Fill any remaining percentiles with the last value.
1667+
for (int64_t i = next_percentile_index; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) {
1668+
result[i] = scores.back().first;
1669+
}
1670+
}
1671+
16431672
template<typename T>
16441673
static inline bool SetHasKeys(const std::set<T>& set) {return false;}
16451674
template<typename T, typename Tk, typename... Args>
@@ -1673,13 +1702,19 @@ static UniValue getblockstats(const JSONRPCRequest& request)
16731702
" \"avgfeerate\": xxxxx, (numeric) Average feerate (in satoshis per virtual byte)\n"
16741703
" \"avgtxsize\": xxxxx, (numeric) Average transaction size\n"
16751704
" \"blockhash\": xxxxx, (string) The block hash (to check for potential reorgs)\n"
1705+
" \"feerate_percentiles\": [ (array of numeric) Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in satoshis per virtual byte)\n"
1706+
" \"10th_percentile_feerate\", (numeric) The 10th percentile feerate\n"
1707+
" \"25th_percentile_feerate\", (numeric) The 25th percentile feerate\n"
1708+
" \"50th_percentile_feerate\", (numeric) The 50th percentile feerate\n"
1709+
" \"75th_percentile_feerate\", (numeric) The 75th percentile feerate\n"
1710+
" \"90th_percentile_feerate\", (numeric) The 90th percentile feerate\n"
1711+
" ],\n"
16761712
" \"height\": xxxxx, (numeric) The height of the block\n"
16771713
" \"ins\": xxxxx, (numeric) The number of inputs (excluding coinbase)\n"
16781714
" \"maxfee\": xxxxx, (numeric) Maximum fee in the block\n"
16791715
" \"maxfeerate\": xxxxx, (numeric) Maximum feerate (in satoshis per virtual byte)\n"
16801716
" \"maxtxsize\": xxxxx, (numeric) Maximum transaction size\n"
16811717
" \"medianfee\": xxxxx, (numeric) Truncated median fee in the block\n"
1682-
" \"medianfeerate\": xxxxx, (numeric) Truncated median feerate (in satoshis per virtual byte)\n"
16831718
" \"mediantime\": xxxxx, (numeric) The block median time past\n"
16841719
" \"mediantxsize\": xxxxx, (numeric) Truncated median transaction size\n"
16851720
" \"minfee\": xxxxx, (numeric) Minimum fee in the block\n"
@@ -1747,13 +1782,13 @@ static UniValue getblockstats(const JSONRPCRequest& request)
17471782
const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default)
17481783
const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0;
17491784
const bool do_medianfee = do_all || stats.count("medianfee") != 0;
1750-
const bool do_medianfeerate = do_all || stats.count("medianfeerate") != 0;
1751-
const bool loop_inputs = do_all || do_medianfee || do_medianfeerate ||
1785+
const bool do_feerate_percentiles = do_all || stats.count("feerate_percentiles") != 0;
1786+
const bool loop_inputs = do_all || do_medianfee || do_feerate_percentiles ||
17521787
SetHasKeys(stats, "utxo_size_inc", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate");
17531788
const bool loop_outputs = do_all || loop_inputs || stats.count("total_out");
17541789
const bool do_calculate_size = do_mediantxsize ||
17551790
SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "swtotal_size");
1756-
const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight", "avgfeerate", "swtotal_weight", "avgfeerate", "medianfeerate", "minfeerate", "maxfeerate");
1791+
const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight", "avgfeerate", "swtotal_weight", "avgfeerate", "feerate_percentiles", "minfeerate", "maxfeerate");
17571792
const bool do_calculate_sw = do_all || SetHasKeys(stats, "swtxs", "swtotal_size", "swtotal_weight");
17581793

17591794
CAmount maxfee = 0;
@@ -1773,7 +1808,7 @@ static UniValue getblockstats(const JSONRPCRequest& request)
17731808
int64_t total_weight = 0;
17741809
int64_t utxo_size_inc = 0;
17751810
std::vector<CAmount> fee_array;
1776-
std::vector<CAmount> feerate_array;
1811+
std::vector<std::pair<CAmount, int64_t>> feerate_array;
17771812
std::vector<int64_t> txsize_array;
17781813

17791814
for (const auto& tx : block.vtx) {
@@ -1848,26 +1883,34 @@ static UniValue getblockstats(const JSONRPCRequest& request)
18481883

18491884
// New feerate uses satoshis per virtual byte instead of per serialized byte
18501885
CAmount feerate = weight ? (txfee * WITNESS_SCALE_FACTOR) / weight : 0;
1851-
if (do_medianfeerate) {
1852-
feerate_array.push_back(feerate);
1886+
if (do_feerate_percentiles) {
1887+
feerate_array.emplace_back(std::make_pair(feerate, weight));
18531888
}
18541889
maxfeerate = std::max(maxfeerate, feerate);
18551890
minfeerate = std::min(minfeerate, feerate);
18561891
}
18571892
}
18581893

1894+
CAmount feerate_percentiles[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 };
1895+
CalculatePercentilesByWeight(feerate_percentiles, feerate_array, total_weight);
1896+
1897+
UniValue feerates_res(UniValue::VARR);
1898+
for (int64_t i = 0; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) {
1899+
feerates_res.push_back(feerate_percentiles[i]);
1900+
}
1901+
18591902
UniValue ret_all(UniValue::VOBJ);
18601903
ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0);
18611904
ret_all.pushKV("avgfeerate", total_weight ? (totalfee * WITNESS_SCALE_FACTOR) / total_weight : 0); // Unit: sat/vbyte
18621905
ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0);
18631906
ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex());
1907+
ret_all.pushKV("feerate_percentiles", feerates_res);
18641908
ret_all.pushKV("height", (int64_t)pindex->nHeight);
18651909
ret_all.pushKV("ins", inputs);
18661910
ret_all.pushKV("maxfee", maxfee);
18671911
ret_all.pushKV("maxfeerate", maxfeerate);
18681912
ret_all.pushKV("maxtxsize", maxtxsize);
18691913
ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array));
1870-
ret_all.pushKV("medianfeerate", CalculateTruncatedMedian(feerate_array));
18711914
ret_all.pushKV("mediantime", pindex->GetMedianTimePast());
18721915
ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array));
18731916
ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee);

‎src/rpc/blockchain.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,16 @@
55
#ifndef BITCOIN_RPC_BLOCKCHAIN_H
66
#define BITCOIN_RPC_BLOCKCHAIN_H
77

8+
#include <vector>
9+
#include <stdint.h>
10+
#include <amount.h>
11+
812
class CBlock;
913
class CBlockIndex;
1014
class UniValue;
1115

16+
static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5;
17+
1218
/**
1319
* Get the difficulty of the net wrt to the given block index, or the chain tip if
1420
* not provided.
@@ -33,4 +39,7 @@ UniValue mempoolToJSON(bool fVerbose = false);
3339
/** Block header to JSON */
3440
UniValue blockheaderToJSON(const CBlockIndex* blockindex);
3541

42+
/** Used by getblockstats to get feerates at different percentiles by weight */
43+
void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight);
44+
3645
#endif

‎src/test/rpc_tests.cpp

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
#include <univalue.h>
1818

19+
#include <rpc/blockchain.h>
20+
1921
UniValue CallRPC(std::string args)
2022
{
2123
std::vector<std::string> vArgs;
@@ -336,4 +338,82 @@ BOOST_AUTO_TEST_CASE(rpc_convert_values_generatetoaddress)
336338
BOOST_CHECK_EQUAL(result[2].get_int(), 9);
337339
}
338340

341+
BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight)
342+
{
343+
int64_t total_weight = 200;
344+
std::vector<std::pair<CAmount, int64_t>> feerates;
345+
CAmount result[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 };
346+
347+
for (int64_t i = 0; i < 100; i++) {
348+
feerates.emplace_back(std::make_pair(1 ,1));
349+
}
350+
351+
for (int64_t i = 0; i < 100; i++) {
352+
feerates.emplace_back(std::make_pair(2 ,1));
353+
}
354+
355+
CalculatePercentilesByWeight(result, feerates, total_weight);
356+
BOOST_CHECK_EQUAL(result[0], 1);
357+
BOOST_CHECK_EQUAL(result[1], 1);
358+
BOOST_CHECK_EQUAL(result[2], 1);
359+
BOOST_CHECK_EQUAL(result[3], 2);
360+
BOOST_CHECK_EQUAL(result[4], 2);
361+
362+
// Test with more pairs, and two pairs overlapping 2 percentiles.
363+
total_weight = 100;
364+
CAmount result2[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 };
365+
feerates.clear();
366+
367+
feerates.emplace_back(std::make_pair(1, 9));
368+
feerates.emplace_back(std::make_pair(2 , 16)); //10th + 25th percentile
369+
feerates.emplace_back(std::make_pair(4 ,50)); //50th + 75th percentile
370+
feerates.emplace_back(std::make_pair(5 ,10));
371+
feerates.emplace_back(std::make_pair(9 ,15)); // 90th percentile
372+
373+
CalculatePercentilesByWeight(result2, feerates, total_weight);
374+
375+
BOOST_CHECK_EQUAL(result2[0], 2);
376+
BOOST_CHECK_EQUAL(result2[1], 2);
377+
BOOST_CHECK_EQUAL(result2[2], 4);
378+
BOOST_CHECK_EQUAL(result2[3], 4);
379+
BOOST_CHECK_EQUAL(result2[4], 9);
380+
381+
// Same test as above, but one of the percentile-overlapping pairs is split in 2.
382+
total_weight = 100;
383+
CAmount result3[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 };
384+
feerates.clear();
385+
386+
feerates.emplace_back(std::make_pair(1, 9));
387+
feerates.emplace_back(std::make_pair(2 , 11)); // 10th percentile
388+
feerates.emplace_back(std::make_pair(2 , 5)); // 25th percentile
389+
feerates.emplace_back(std::make_pair(4 ,50)); //50th + 75th percentile
390+
feerates.emplace_back(std::make_pair(5 ,10));
391+
feerates.emplace_back(std::make_pair(9 ,15)); // 90th percentile
392+
393+
CalculatePercentilesByWeight(result3, feerates, total_weight);
394+
395+
BOOST_CHECK_EQUAL(result3[0], 2);
396+
BOOST_CHECK_EQUAL(result3[1], 2);
397+
BOOST_CHECK_EQUAL(result3[2], 4);
398+
BOOST_CHECK_EQUAL(result3[3], 4);
399+
BOOST_CHECK_EQUAL(result3[4], 9);
400+
401+
// Test with one transaction spanning all percentiles.
402+
total_weight = 104;
403+
CAmount result4[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 };
404+
feerates.clear();
405+
406+
feerates.emplace_back(std::make_pair(1, 100));
407+
feerates.emplace_back(std::make_pair(2, 1));
408+
feerates.emplace_back(std::make_pair(3, 1));
409+
feerates.emplace_back(std::make_pair(3, 1));
410+
feerates.emplace_back(std::make_pair(999999, 1));
411+
412+
CalculatePercentilesByWeight(result4, feerates, total_weight);
413+
414+
for (int64_t i = 0; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) {
415+
BOOST_CHECK_EQUAL(result4[i], 1);
416+
}
417+
}
418+
339419
BOOST_AUTO_TEST_SUITE_END()

‎test/functional/data/rpc_getblockstats.json

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,13 +112,19 @@
112112
"avgfeerate": 0,
113113
"avgtxsize": 0,
114114
"blockhash": "1d7fe80f19d28b8e712af0399ac84006db753441f3033111b3a8d610afab364f",
115+
"feerate_percentiles": [
116+
0,
117+
0,
118+
0,
119+
0,
120+
0
121+
],
115122
"height": 101,
116123
"ins": 0,
117124
"maxfee": 0,
118125
"maxfeerate": 0,
119126
"maxtxsize": 0,
120127
"medianfee": 0,
121-
"medianfeerate": 0,
122128
"mediantime": 1525107242,
123129
"mediantxsize": 0,
124130
"minfee": 0,
@@ -144,12 +150,18 @@
144150
"avgtxsize": 187,
145151
"blockhash": "4e21a43675d7a41cb6b944e068c5bcd0a677baf658d9ebe021ae2d2f99397ccc",
146152
"height": 102,
153+
"feerate_percentiles": [
154+
20,
155+
20,
156+
20,
157+
20,
158+
20
159+
],
147160
"ins": 1,
148161
"maxfee": 3760,
149162
"maxfeerate": 20,
150163
"maxtxsize": 187,
151164
"medianfee": 3760,
152-
"medianfeerate": 20,
153165
"mediantime": 1525107242,
154166
"mediantxsize": 187,
155167
"minfee": 3760,
@@ -174,13 +186,19 @@
174186
"avgfeerate": 109,
175187
"avgtxsize": 228,
176188
"blockhash": "22d9b8b9c2a37c81515f3fc84f7241f6c07dbcea85ef16b00bcc33ae400a030f",
189+
"feerate_percentiles": [
190+
20,
191+
20,
192+
20,
193+
300,
194+
300
195+
],
177196
"height": 103,
178197
"ins": 3,
179198
"maxfee": 49800,
180199
"maxfeerate": 300,
181200
"maxtxsize": 248,
182201
"medianfee": 3760,
183-
"medianfeerate": 20,
184202
"mediantime": 1525107243,
185203
"mediantxsize": 248,
186204
"minfee": 3320,

‎test/functional/rpc_getblockstats.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class GetblockstatsTest(BitcoinTestFramework):
2727
'maxfee',
2828
'maxfeerate',
2929
'medianfee',
30-
'medianfeerate',
30+
'feerate_percentiles',
3131
'minfee',
3232
'minfeerate',
3333
'totalfee',

0 commit comments

Comments
 (0)