diff --git a/README.md b/README.md index 6dc88422..104f681a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ GPLv3. See the included `LICENSE.txt` file or [visit gnu.org and read the licens ### Highlights: - *Fast:* Written in 100% modern `C++20` using multi-threaded and asynchronous programming techniques. -- *A drop-in replacement for ElectronX/ElectrumX:* Fulcrum is 100% protocol-level compatible with the [Electrum Cash 1.5.3 protocol](https://electrum-cash-protocol.readthedocs.io/en/latest/). Existing server admins should feel right at home with this software since installation and management of it is nearly identical to ElectronX/ElectrumX server. +- *A drop-in replacement for ElectrumX:* Fulcrum is 100% protocol-level compatible with the [Electrum Cash 1.6 protocol](https://electrum-cash-protocol.readthedocs.io/en/latest/). Existing server admins should feel right at home with this software since installation and management of it is nearly identical to an ElectrumX server. - *Cross-platform:* While this codebase was mainly developed and tested on MacOS, Windows and Linux, it should theoretically work on any modern OS (such as *BSD) that has Qt5 or Qt6 Networking available. - ***NEW!*** *Triple-coin:* Supports BCH, BTC and LTC. @@ -27,7 +27,8 @@ GPLv3. See the included `LICENSE.txt` file or [visit gnu.org and read the licens - *For running*: - A supported bitcoin full node with its JSON-RPC service enabled, preferably running on the same machine. - *For **BCH***: Bitcoin Cash Node, Bitcoin Cash Unlimited, Flowee, and bchd have all been tested extensively and are known to work well with this software. - - *For **BTC***: Bitcoin Core v0.17.0 or later. Bitcoin Knots is also rumored to work but is untested by the developer. + - *For **BTC***: Bitcoin Core v0.17.0 or later. Bitcoin Knots is also known to work perfectly well. + - Note: Bitcoin Core and/or Bitcoin Knots >= v28.0.0 are recommended for full Electrum Protocol v1.6 support. - *For **LTC***: Litecoin Core v0.17.0 or later. No other full nodes are supported by this software for LTC. - If using Litcoin Core v0.21.2 or above, your daemon is serializing data using mweb extensions. While Fulcrum understands this serialization format, your Electrum-LTC clients may not. You can run `litecoind` with `-rpcserialversion=1` to have your daemon return transactions in pre-mweb format which is understood by most Electrum-LTC clients. - The node must have txindex enabled e.g. `txindex=1`. @@ -160,7 +161,7 @@ Execute the binary, with `-h` to see the built-in help, e.g. `./Fulcrum -h`. You It is recommended you specify a data dir (`-D` via CLI or `datadir=` via config file) on an SSD drive for best results. Synching against `testnet` should take you about 10-20 minutes (more on slower machines), and mainnet can take anywhere from 4 hours to 20+ hours, depending on machine and drive speed. I have not tried synching against mainnet on an HDD and it will probably take ***days*** if you are lucky. -As long as the server is still synchronizing, all public-facing ports will not yet be bound for listening and as such an attempt to connect to one of the RPC ports will fail with a socket error such as e.g. "Connection refused". Once the server finishes synching it will behave like an ElectronX/ElectrumX server and it can receive requests from Electron Cash (or Electrum if on BTC). +As long as the server is still synchronizing, all public-facing ports will not yet be bound for listening and as such an attempt to connect to one of the RPC ports will fail with a socket error such as e.g. "Connection refused". Once the server finishes synching it will behave like an ElectrumX server and it can receive requests from Electron Cash (or Electrum if on BTC). You may also wish to read the [Fulcrum manpage](https://github.com/cculianu/Fulcrum/blob/master/doc/unix-man-page.md). diff --git a/doc/electrum-cash-protocol b/doc/electrum-cash-protocol index 5a5ff8b1..797c322e 160000 --- a/doc/electrum-cash-protocol +++ b/doc/electrum-cash-protocol @@ -1 +1 @@ -Subproject commit 5a5ff8b17a0540d07f48b05aeca588626b676ac1 +Subproject commit 797c322e22c3236b475005a073bf7c53dd10accf diff --git a/src/BitcoinD.cpp b/src/BitcoinD.cpp index 66b6758a..640fb71b 100644 --- a/src/BitcoinD.cpp +++ b/src/BitcoinD.cpp @@ -18,6 +18,7 @@ // #include "BitcoinD.h" +#include "Util.h" #include "ZmqSubNotifier.h" #include "bitcoin/rpc/protocol.h" @@ -58,6 +59,8 @@ BitcoinDMgr::~BitcoinDMgr() { cleanup(); } void BitcoinDMgr::startup() { Log() << objectName() << ": starting " << nClients << " " << Util::Pluralize("bitcoin RPC client", nClients) << " ..."; + // ensure the cached results table is cleared on reconnect + conns += connect(this, &BitcoinDMgr::gotFirstGoodConnection, this, &BitcoinDMgr::clearCachedResultsTable); // As soon as a good BitcoinD is up, try and grab the network info (version, subversion, etc). This must // happen early because the values in this info object determine which workarounds we may or may not apply to // RPC args. @@ -120,6 +123,7 @@ void BitcoinDMgr::on_started() { ThreadObjectMixin::on_started(); callOnTimerSoon(kRequestTimerPolltimeMS, kRequestTimeoutTimer, [this]{ requestTimeoutChecker(); return true; }); + callOnTimerSoon(kExpireOldCachedResultsPolltimeMS, kExpireOldCachedResultsTimer, [this] { expireOldCachedResults(); return true; }); } void BitcoinDMgr::on_finished() @@ -223,7 +227,7 @@ namespace { version = Version::BitcoinDCompact(val); isBchd = true; } else { - isCore = subversion.startsWith("/Satoshi:"); + isCore = subversion.startsWith("/Satoshi:"); // this matches Bitcoin Knots as well isBU = subversion.startsWith("/BCH Unlimited:"); isBCHN = subversion.startsWith("/Bitcoin Cash Node:"); isLTC = subversion.startsWith("/LitecoinCore:"); @@ -294,15 +298,30 @@ void BitcoinDMgr::refreshBitcoinDNetworkInfo() return true; return false; }; - bitcoinDInfo.isZeroArgEstimateFee = !res.isCore && !res.isLTC && isZeroArgEstimateFee(bitcoinDInfo.version, bitcoinDInfo.subversion); + // Set up RpcSupportInfo + auto & rsi = bitcoinDInfo.rpcSupportInfo; + rsi.isZeroArgEstimateFee = !res.isCore && !res.isLTC && isZeroArgEstimateFee(bitcoinDInfo.version, bitcoinDInfo.subversion); + rsi.hasEstimateSmartFee = (res.isCore || res.isLTC) && bitcoinDInfo.version >= Version{0, 15, 0}; + rsi.isTwoArgEstimateSmartFee = (res.isCore && bitcoinDInfo.version >= Version{0, 16, 0}) || (res.isLTC && bitcoinDInfo.version >= Version{0, 15, 0}); // Implementations known to lack `getzmqnotifications`: // - bchd (all versions) // - BU before version 1.9.1.0 - bitcoinDInfo.lacksGetZmqNotifications - = lacksGetZmqNotifications - = res.isBchd || (res.isBU && res.version < Version{1, 9, 1}); + rsi.lacksGetZmqNotifications = lacksGetZmqNotifications = res.isBchd || (res.isBU && res.version < Version{1, 9, 1}); // clear hasDSProofRPC until proven to have it via a query - bitcoinDInfo.hasDSProofRPC = false; + rsi.hasDSProofRPC = false; + // Bitcoin Core 25.0+ requires specifying `maxburnamount` in the `sendrawtransaction` RPC + // (which, due to bitcoin core weirdness in how it encodes the version int, maps to Version{0, 25, 0} + rsi.sendRawTransactionRequiresMaxBurnAmount = res.isCore && bitcoinDInfo.version >= Version{0, 25, 0}; + // The `submitpackage` RPC is only present in a form usable by us on Bitcoin Core >= 27.0.0, but it was + // almost worthless on 27.0.0 (according to ElectrumX devs), and so we require 28.0.0. + // See: https://github.com/spesmilo/electrum-protocol/pull/6/files#r2459860616 + rsi.hasSubmitPackageRPC = res.isCore && bitcoinDInfo.version >= Version{0, 28, 0}; + if (res.isCore && !rsi.hasSubmitPackageRPC) // warn admin about lack of submitpackage support + Warning() << "*** Compatibility Warning *** The BTC full node backing this " APPNAME " instance" + " lacks a known-good `submitpackage` RPC, which is needed for full Electrum protocol" + " v1.6 support. Disabling the `blockchain.transaction.broadcast_package` method out of" + " an abundance of caution. Consider upgrading your full node to either Bitcoin Core or" + " Bitcoin Knots version 28.0.0 or above."; } // end lock scope // be sure to announce whether remote bitcoind is bitcoin core (this determines whether we use segwit or not) BTC::Coin coin = BTC::Coin::BCH; // default BCH if unknown (not segwit) @@ -471,7 +490,7 @@ void BitcoinDMgr::refreshBitcoinDZmqNotifications() << msg.errorMessage(); setZmqNotifications({}); // clear current, if any std::unique_lock g(bitcoinDInfoLock); - bitcoinDInfo.lacksGetZmqNotifications = true; // flag that we think remote lacks this RPC + bitcoinDInfo.rpcSupportInfo.lacksGetZmqNotifications = true; // flag that we think remote lacks this RPC }, // failure [this](const RPC::Message::Id &, const QString &reason) { @@ -536,18 +555,6 @@ BitcoinDInfo BitcoinDMgr::getBitcoinDInfo() const void BitcoinDMgr::requestBitcoinDInfoRefresh() { refreshBitcoinDNetworkInfo(); } -bool BitcoinDMgr::isZeroArgEstimateFee() const -{ - std::shared_lock g(bitcoinDInfoLock); - return bitcoinDInfo.isZeroArgEstimateFee; -} - -bool BitcoinDMgr::isCoreLike() const -{ - std::shared_lock g(bitcoinDInfoLock); - return bitcoinDInfo.isCore || bitcoinDInfo.isLTC; -} - Version BitcoinDMgr::getBitcoinDVersion() const { std::shared_lock g(bitcoinDInfoLock); @@ -580,25 +587,104 @@ void BitcoinDMgr::setZmqNotifications(const BitcoinDZmqNotifications &zmqs) if (changed) emit zmqNotificationsChanged(zmqs); } -bool BitcoinDMgr::hasDSProofRPC() const +BitcoinDInfo::RpcSupportInfo BitcoinDMgr::getRpcSupportInfo() const { std::shared_lock g(bitcoinDInfoLock); - return bitcoinDInfo.hasDSProofRPC; + return bitcoinDInfo.rpcSupportInfo; } void BitcoinDMgr::setHasDSProofRPC(bool b) { std::unique_lock g(bitcoinDInfoLock); - bitcoinDInfo.hasDSProofRPC = b; + bitcoinDInfo.rpcSupportInfo.hasDSProofRPC = b; +} + +auto BitcoinDMgr::getCachedResult(const QString &method) const -> std::optional +{ + std::shared_lock g(cachedResultsLock); + if (auto it = cachedResultsTable.find(method); it != cachedResultsTable.end()) + return it.value(); + return std::nullopt; +} + +void BitcoinDMgr::updateCachedResult(const QString &method, qint64 maxAge, QVariant var) +{ + const auto now = Util::getTime(); + std::unique_lock g(cachedResultsLock); + auto & r = cachedResultsTable[method]; + r.result = std::move(var); + r.maxAge = maxAge; + r.timeStamp = now; +} + +void BitcoinDMgr::clearCachedResultsTable() +{ + decltype(cachedResultsTable)::size_type size{}; + { + std::unique_lock g(cachedResultsLock); + size = cachedResultsTable.size(); + cachedResultsTable.clear(); + cachedResultsTable.squeeze(); + } + DebugM("cachedResultsTable: Cleared ", size, Util::Pluralize(" entry", size)); +} + +void BitcoinDMgr::expireOldCachedResults() +{ + decltype(cachedResultsTable)::size_type origSize{}, deletions{}; + { + std::unique_lock g(cachedResultsLock); + origSize = cachedResultsTable.size(); + for (auto it = cachedResultsTable.cbegin(); it != cachedResultsTable.cend(); /* */) { + if (it.value().ageMSec() > it.value().maxAge) { + it = cachedResultsTable.erase(it); + ++deletions; + } else + ++it; + } + if (deletions) cachedResultsTable.squeeze(); + } + if (deletions) + DebugM("cachedResultsTable: Deleted ", deletions, "/", origSize, " expired", Util::Pluralize(" entry", deletions)); } /// This is safe to call from any thread. Internally it dispatches messages to this obejct's thread. /// Does not throw. Results/Error/Fail functions are called in the context of the `sender` thread. void BitcoinDMgr::submitRequest(QObject *sender, const RPC::Message::Id &rid, const QString & method, const QVariantList & params, - const ResultsF & resf, const ErrorF & errf, const FailF & failf, int timeout) + const ResultsF & resf, const ErrorF & errf, const FailF & failf, int timeout, + std::optional cachedResultOkIfNotOlderThan) { using namespace BitcoinDMgrHelper; constexpr bool debugDeletes = false; // set this to true to print debug messages tracking all the below object deletions (tested: no leaks!) + + // handle caching of responses (such as getmempoolinfo) + std::optional optResf; + if (cachedResultOkIfNotOlderThan.has_value()) { + if (!params.empty()) [[unlikely]] { + // Detect calling code mis-use of this function and print an error message. + Error() << "INTERNAL ERROR: Caller to BitcoinDMgr::" << __func__ << " specified a value for" + << "`cachedResultOkIfNotOlderThan`, but also specified a non-empty params list. This usage is not" + << " supported; will proceed without caching. FIXME!"; + } else { + bool missing = true; + if (auto optCached = getCachedResult(method)) { + missing = false; + if (const qint64 age = optCached->ageMSec(); age >= 0 && age <= *cachedResultOkIfNotOlderThan) { + // cached value is good, give it to sender in their event loop + DebugM("Cached `", method, "` is good, returning cached result (age: ", QString::number(age / 1e3, 'f', 3), " sec)"); + if (resf) + Util::AsyncOnObject(sender, [m = RPC::Message::makeResponse(rid, optCached->result), resf]{ resf(m); }); + return; // return early -- no work to do! + } + } + DebugM("Cached `", method, "` is ", missing ? "missing" : "old", ", proceeding to send request to bitcoind ..."); + optResf = [this, method, resf, maxAge = *cachedResultOkIfNotOlderThan](const RPC::Message &response) { + updateCachedResult(method, maxAge, response.result()); + if (resf) resf(response); + }; + } + } + // A note about ownership: this context object is "owned" by the connections below to ->sender *only*. // It will be auto-deleted when the shared_ptr refct held by the lambdas drops to 0. This is guaranteed // to happen either as a result of a successful request reply, or due to bitcoind failure, or if the sender @@ -616,7 +702,7 @@ void BitcoinDMgr::submitRequest(QObject *sender, const RPC::Message::Id &rid, co context->setObjectName(QStringLiteral("context for '%1' request id: %2").arg(sender ? sender->objectName() : QString{}, rid.toString())); // result handler (runs in sender thread), captures context and keeps it alive as long as signal/slot connection is alive - connect(context.get(), &ReqCtxObj::results, sender, [context, resf, sender/*, method, params, timeout*/](const RPC::Message &response) { + connect(context.get(), &ReqCtxObj::results, sender, [context, resf = optResf.value_or(resf), sender/*, method, params, timeout*/](const RPC::Message &response) { // Debug code for troubleshooting the extent of bitcoind backlogs in servicing requests /* const auto now = Util::getTime(); @@ -982,16 +1068,28 @@ QVariantMap BitcoinDInfo::toVariantMap() const QVariantMap ret; ret["version"] = version.toString(true); ret["subversion"] = subversion; - ret["warnings"] = warnings; ret["relayfee"] = relayFee; - ret["isZeroArgEstimateFee"] = isZeroArgEstimateFee; - ret["isBchd"] = isBchd; + ret["warnings"] = warnings; + ret["isZeroArgEstimateFee"] = rpcSupportInfo.isZeroArgEstimateFee; + ret["hasEstimateSmartFee"] = rpcSupportInfo.hasEstimateSmartFee; + ret["isTwoArgEstimateSmartFee"] = rpcSupportInfo.isTwoArgEstimateSmartFee; + ret["lacksGetZmqNotifications"] = rpcSupportInfo.lacksGetZmqNotifications; + ret["hasDSProofRPC"] = rpcSupportInfo.hasDSProofRPC; + ret["sendRawTransactionRequiresMaxBurnAmount"] = rpcSupportInfo.sendRawTransactionRequiresMaxBurnAmount; + ret["hasSubmitPackageRPC"] = rpcSupportInfo.hasSubmitPackageRPC; ret["isCore"] = isCore; - ret["lacksGetZmqNotifications"] = lacksGetZmqNotifications; - ret["hasDSProofRPC"] = hasDSProofRPC; + ret["isLTC"] = isLTC; + ret["isBU"] = isBU; + ret["isFlowee"] = isFlowee; + ret["isBchd"] = isBchd; QVariantList zmqs; for (auto it = zmqNotifications.begin(); it != zmqNotifications.end(); ++it) zmqs.push_back(QVariantList{it.key(), it.value()}); ret["zmqNotifications"] = zmqs; return ret; } + +qint64 BitcoinDMgr::CachedResult::ageMSec() const +{ + return std::max(Util::getTime() - timeStamp, 0); +} diff --git a/src/BitcoinD.h b/src/BitcoinD.h index bb32c13e..16378782 100644 --- a/src/BitcoinD.h +++ b/src/BitcoinD.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -48,14 +49,21 @@ struct BitcoinDInfo { QString subversion; ///< subversion string from daemon e.g.: /Bitcoin Cash Node bla bla;EB32 ..../ double relayFee = 0.0; ///< from 'relayfee' in the getnetworkinfo response; minimum fee/kb to relay a tx, usually: 0.00001000 QString warnings = ""; ///< from 'warnings' in the getnetworkinfo response (usually is empty string, but may not always be) - bool isBchd = false; ///< true if remote bitcoind subversion is: /bchd:... - bool isZeroArgEstimateFee = false; ///< true if remote bitcoind expects 0 argument "estimatefee" RPC. + struct RpcSupportInfo { + bool isZeroArgEstimateFee = false; ///< true if remote bitcoind expects 0 argument "estimatefee" RPC. + bool hasEstimateSmartFee = false; ///< true if remote bitcoind supports estimatesmartfee. (Bitcoin Core >= 0.15.0, Litecoin >= 0.15.0) + bool isTwoArgEstimateSmartFee = false; ///< if true, estimatesmartfee has an optional second parameter "mode" (Bitcoin Core >= 0.16.0, Litecoin >= 0.15.0) + bool lacksGetZmqNotifications = false; ///< true if bchd or BU < 1.9.1.0, or if we got an RPC error the last time we queried + bool hasDSProofRPC = false; ///< true if the RPC query to `getdsprooflist` didn't return an error. + bool sendRawTransactionRequiresMaxBurnAmount = false; ///< true if the `sendrawtransaction` RPC requires 2 extra args. Bitcoin Core >= 25.0.0 only. + bool hasSubmitPackageRPC = false; ///< true if the `submitpackage` RPC method exists and is what we expect. Bitcoin Core >= 28.0.0 only. + }; + RpcSupportInfo rpcSupportInfo; bool isCore = false; ///< true if we are actually connected to /Satoshi.. node (Bitcoin Core) bool isLTC = false; ///< true if we are actually connected to /LitecoinCore.. node (Litecoin) bool isBU = false; ///< true if subversion string starts with "/BCH Unlimited:" bool isFlowee = false; ///< true if subversion string starts with "/Flowee" - bool lacksGetZmqNotifications = false; ///< true if bchd or BU < 1.9.1.0, or if we got an RPC error the last time we queried - bool hasDSProofRPC = false; ///< true if the RPC query to `getdsprooflist` didn't return an error. + bool isBchd = false; ///< true if remote bitcoind subversion is: /bchd:... /// The below field is populated from bitcoind RPC `getzmqnotifications` (if supported and if we are compiled to /// use libzmq). Note that entires in here are auto-transformed by BitcoinDMgr such that: @@ -107,7 +115,7 @@ class BitcoinDMgr : public Mgr, public IdMixin, public ThreadObjectMixin, public /// NOTE2: calling this method while this BitcoinDManager is stopped or about to be stopped is not supported. void submitRequest(QObject *sender, const RPC::Message::Id &id, const QString & method, const QVariantList & params, const ResultsF & = ResultsF(), const ErrorF & = ErrorF(), const FailF & = FailF(), - int timeout = kDefaultTimeoutMS); + int timeout = kDefaultTimeoutMS, std::optional cachedResultOkIfNotOlderThan = std::nullopt); /// Thread-safe. Returns a copy of the BitcoinDInfo object. This object is refreshed each time we /// reconnect to BitcoinD. This is called by ServerBase in various places. @@ -122,20 +130,14 @@ class BitcoinDMgr : public Mgr, public IdMixin, public ThreadObjectMixin, public /// in the db. See also: Storage::genesisHash(). BlockHash getBitcoinDGenesisHash() const; - /// Thread-safe. Convenient method to avoid an extra copy. Returns getBitcoinDInfo().isZeroArgEstimateFee - bool isZeroArgEstimateFee() const; - - /// Thread-safe. Convenient method to avoid an extra copy. Returns true iff getBitcoinDInfo().isCore || getBitcoinDInfo().isLTC. - bool isCoreLike() const; - /// Thread-safe. Convenient method to avoid an extra copy. Returns getBitcoinDInfo().version Version getBitcoinDVersion() const; /// Thread-safe. Convenient method to avoid an extra copy. Returns getBitcoinDInfo().zmqNotifications BitcoinDZmqNotifications getZmqNotifications() const; - /// Thread-safe. Convenient method to avoid an extra copy. Returns getBitcoinDInfo().hasDSProofRPC - bool hasDSProofRPC() const; + /// Thread-safe. Convenient method to avoid an extra copy. Returns getBitcoinDInfo().rpcSupportInfo + BitcoinDInfo::RpcSupportInfo getRpcSupportInfo() const; signals: void gotFirstGoodConnection(quint64 bitcoindId); // emitted whenever the first bitcoind after a "down" state (or after startup) gets its first good status (after successful authentication) @@ -237,6 +239,32 @@ protected slots: /// dsproof rpc setter -- called internally by probeBitcoinDHasDSProofRPC void setHasDSProofRPC(bool); + + /// Mechanism to cache the last-known response from bitcoind's RPC calls (used for RPC calls such as getmempoolinfo that don't need to be super-up-to-date) + struct CachedResult { + qint64 timeStamp{}; ///< The time in msec that this was last cached (as returned by Util::getTime()) + qint64 maxAge{}; ///< The maximum age that is permitted for this cached entry, after which time it may be deleted + QVariant result; ///< The last good result from a bitcoind RPC call + CachedResult() noexcept = default; + /// Returns the "age" of this cached value in msec + qint64 ageMSec() const; + }; + + mutable std::shared_mutex cachedResultsLock; + QHash cachedResultsTable; ///< guarded-by cachedResultsLock + + /// Thread-safe. Returns a copy of the CachedResult object for method, if found, or std::nullopt otherwise. + /// Note that all cached results are cleared each time we reconnect to BitcoinD. + std::optional getCachedResult(const QString &method) const; + /// Thread-safe, caches a result, updating the internal timestamp to current. + void updateCachedResult(const QString &method, qint64 maxAge, QVariant result); + /// Thread-safe, clears the cached results table + void clearCachedResultsTable(); + + static constexpr auto kExpireOldCachedResultsTimer = "+ExpireZombieCachedResults"; + static constexpr auto kExpireOldCachedResultsPolltimeMS = 60'000; + /// Thread-safe, called from a timer. Deletes old entries from table. + void expireOldCachedResults(); }; class BitcoinD : public RPC::HttpConnection, public ThreadObjectMixin /* NB: also inherits TimersByNameMixin via AbstractConnection base */ diff --git a/src/Common.h b/src/Common.h index a829b425..150a5288 100644 --- a/src/Common.h +++ b/src/Common.h @@ -38,7 +38,7 @@ #endif #include -#include +#include #ifdef __clang__ // turn off the dreaded "warning: class padded with xx bytes, etc" since we aren't writing wire protocols using structs.. diff --git a/src/Controller.cpp b/src/Controller.cpp index 047a4ac0..4aa11d5e 100644 --- a/src/Controller.cpp +++ b/src/Controller.cpp @@ -529,7 +529,7 @@ DownloadBlocksTask::DownloadBlocksTask(unsigned from, unsigned to, unsigned stri allowSegWit(ctl_->isSegWitCoin()), allowMimble(ctl_->isMimbleWimbleCoin()), allowCashTokens(ctl_->isBCHCoin()), rpaStartHeight(rpaHeight) { - FatalAssert( (to >= from) && (ctl_) && (stride > 0), "Invalid params to DonloadBlocksTask c'tor, FIXME!"); + FatalAssert(to >= from && ctl_ && stride > 0, "Invalid params to DonloadBlocksTask c'tor, FIXME!"); if (stride > 1 || expectedCt > 1) { // tolerate slow request responses (up to 10 mins) if downloading multiple blocks // fixes issue #116 @@ -1381,7 +1381,7 @@ void Controller::process(bool beSilentIfUpToDate) // ... nothing.. } else if (sm->state == State::SynchMempoolFinished) { // ... - if (bitcoindmgr->hasDSProofRPC()) + if (bitcoindmgr->getRpcSupportInfo().hasDSProofRPC) sm->state = State::SynchDSPs; // remote bitcoind has dsproof rpc, proceed to synch dsps else { emit synchedMempool(); diff --git a/src/PeerMgr.cpp b/src/PeerMgr.cpp index 0224d316..00a961fd 100644 --- a/src/PeerMgr.cpp +++ b/src/PeerMgr.cpp @@ -67,7 +67,7 @@ QVariantMap PeerMgr::makeFeaturesDict(PeerClient *c) const { const bool isBCH = coin == BTC::Coin::BCH; return Server::makeFeaturesDictForConnection(c, _genesisHash, *options, srvmgr->hasDSProofRPC(), isBCH, - storage->getConfiguredRpaStartHeight()); + storage->getConfiguredRpaStartHeight(), srvmgr->hasSubmitPackageRPC()); } QString PeerMgr::publicHostNameForConnection(PeerClient *c) const diff --git a/src/PeerMgr.h b/src/PeerMgr.h index 93a26a63..b3652155 100644 --- a/src/PeerMgr.h +++ b/src/PeerMgr.h @@ -33,9 +33,6 @@ #include #include -#include -#include - struct Options; struct PeerInfo; class PeerClient; diff --git a/src/RPC.h b/src/RPC.h index b9418e6c..598fa1c5 100644 --- a/src/RPC.h +++ b/src/RPC.h @@ -32,6 +32,7 @@ #include #include +#include #include #include #include // for std::pair, std::move @@ -81,7 +82,7 @@ namespace RPC { /// If allowsNotifications is false, notifications for this method will be silently ignored. bool allowsRequests = true, allowsNotifications = false; using PosParamRange = std::pair; - static constexpr unsigned NO_POS_PARAM_LIMIT = UINT_MAX; ///< use this for PosParamsRange.second to specify no limit. + static constexpr unsigned NO_POS_PARAM_LIMIT = std::numeric_limits::max(); ///< use this for PosParamsRange.second to specify no limit. /// If this optional !has_value, then positional arguments (list for "params") are rejected. /// Otherwise, specify an unsigned int range where .first is the minimum and .second is the maximum number /// of positional parameters accepted. If .second is NO_POS_PARAM_LIMIT, then any number of parameters from diff --git a/src/ServerMisc.cpp b/src/ServerMisc.cpp index a102b374..521d0927 100644 --- a/src/ServerMisc.cpp +++ b/src/ServerMisc.cpp @@ -4,7 +4,7 @@ namespace ServerMisc { const Version MinProtocolVersion(1,4,0); - const Version MaxProtocolVersion(1,5,3); + const Version MaxProtocolVersion(1,6,0); const Version MinTokenAwareProtocolVersion(1,5,0); const QString AppVersion(VERSION); const QString AppSubVersion = QString("%1 %2").arg(APPNAME, VERSION); diff --git a/src/Servers.cpp b/src/Servers.cpp index 68426cb3..7f7f83cd 100644 --- a/src/Servers.cpp +++ b/src/Servers.cpp @@ -381,6 +381,11 @@ namespace { } return ret; } + + bool IsMetaTypeStringLike(const QVariant & var) { + return Compat::IsMetaType(var, QMetaType::Type::QString) || Compat::IsMetaType(var, QMetaType::Type::QByteArray); + } + } // namespace ServerBase::ServerBase(SrvMgr *sm, @@ -416,7 +421,7 @@ QVariant ServerBase::stats() const map["isSubscribedToHeaders"] = bool(client->headerSubConnection); map["nSubscriptions"] = client->nShSubs.load(); map["nTxSent"] = client->info.nTxSent; - map["nTxBytesSent"] = client->info.nTxBytesSent; + map["nTxBytesSent"] = qulonglong(client->info.nTxBytesSent); map["nTxBroadcastErrors"] = client->info.nTxBroadcastErrors; // data from the per-ip structure map["perIPData"] = [client]{ @@ -435,7 +440,7 @@ QVariant ServerBase::stats() const map.remove("lastSocketError"); map.remove("nUnansweredRequests"); map.remove("nRequestsSent"); - clientList.append(QVariantMap({{name, map}})); + clientList.push_back(QVariantMap({{name, map}})); } m["clients"] = clientList; return QVariantMap{{prettyName(), m}}; @@ -730,8 +735,8 @@ void ServerBase::onPeerError(IdMixin::Id clientId, const QString &what) // and finally log it Debug() << "onPeerError, client " << clientId << " error: " << what.left(num); } - if (Client *c = getClient(clientId); c) { - if (const auto diff = ++c->info.errCt - c->info.nRequestsRcv; diff >= kMaxErrorCount) { + if (Client *c = getClient(clientId)) { + if (const auto diff = qlonglong(++c->info.errCt) - qlonglong(c->info.nRequestsRcv); diff >= kMaxErrorCount) { Warning() << "Excessive errors (" << diff << ") for: " << c->prettyName() << ", disconnecting"; killClient(c); return; @@ -764,56 +769,93 @@ namespace { ServerBase::RPCError::~RPCError() {} ServerBase::RPCErrorWithDisconnect::~RPCErrorWithDisconnect() {} -void ServerBase::generic_do_async(Client *c, RPC::BatchId batchId, const RPC::Message::Id &reqId, - const std::function &work, int priority) -{ - if (LIKELY(work)) { - struct ResErr { - QVariant results; - bool error = false, doDisconnect = false; - QString errMsg; - int errCode = 0; - }; +template +requires + // WorkFunc must not return void + (not std::is_same_v>) + // and CompletionFunc must either be nullptr or must accept 1 arg, the result of invoking WorkFunc + && (std::is_same_v || requires (CompletionFunc c, WorkFunc w) { c(w()); }) +void ServerBase::generic_do_async(Client *c, RPC::BatchId batchId, const RPC::Message::Id &reqId, WorkFunc && work, + CompletionFunc && completion, int priority) +{ + // Defensive programming checks in case `work` or `completion` are std::functions that contain no target (are null) + if constexpr (std::is_constructible_v) + if (!work) [[unlikely]] { + Error() << "INTERNAL ERROR: work Func must be valid! FIXME!"; + return; + } + if constexpr (std::is_constructible_v && !std::is_same_v) + if (!completion) [[unlikely]] { + Error() << "INTERNAL ERROR: completion Func must be valid! FIXME!"; + return; + } - auto reserr = std::make_shared(); ///< shared with lambda for both work and completion. this is how they communicate. - - (asyncThreadPool ? asyncThreadPool : ::AppThreadPool())->submitWork( - c, // <--- all work done in client context, so if client is deleted, completion not called - // runs in worker thread, must not access anything other than reserr and work - [reserr,work]{ - try { - QVariant result = work(); - reserr->results.swap( result ); // constant-time copy - } catch (const RPCError & e) { - reserr->error = true; - reserr->doDisconnect = e.disconnect; - reserr->errMsg = e.what(); - reserr->errCode = e.code; - } - }, - // completion: runs in client thread (only called if client not already deleted) - [c, batchId, reqId, reserr] { - if (reserr->error) { - emit c->sendError(reserr->doDisconnect, reserr->errCode, reserr->errMsg, batchId, reqId); - return; - } - // no error, send results to client - emit c->sendResult(batchId, reqId, reserr->results); - }, - // default fail function just sends json rpc error "internal error: " - defaultTPFailFunc(c, batchId, reqId), - // lower is sooner, higher is later. Default 0. - priority - ); - } else - Error() << "INTERNAL ERROR: work must be valid! FIXME!"; + struct SharedState { + using ResultsT = std::invoke_result_t; + std::optional results; // we make this a std::optional to not require ResultsT to be default-constructible + bool error = false, doDisconnect = false; + QString errMsg; + int errCode = 0; + WorkFunc work; + CompletionFunc completion; + SharedState(WorkFunc && w, CompletionFunc && c) : work(std::move(w)), completion(std::move(c)) {} + }; + + // Shared with both work and completion lambdas that we pass to submitWork. This is how they communicate. + auto state = std::make_shared(std::move(work), std::move(completion)); + + (asyncThreadPool ? asyncThreadPool : ::AppThreadPool())->submitWork( + c, // <--- all work done in client context, so if client is deleted, completion not called + // runs in worker thread, must not access anything other than `state` + [state] { + try { + state->results.emplace(state->work()); + } catch (const RPCError & e) { + state->error = true; + state->doDisconnect = e.disconnect; + state->errMsg = e.what(); + state->errCode = e.code; + } catch (const std::exception &e) { + state->error = true; + state->doDisconnect = false; + state->errMsg = QString("internal error: %1").arg(e.what()); + state->errCode = RPC::ErrorCodes::Code_InternalError; + } + }, + // completion: runs in client thread (only called if client still alive) + [c, batchId, reqId, state] { + if (state->error) { + emit c->sendError(state->doDisconnect, state->errCode, state->errMsg, batchId, reqId); + return; + } + // no error, send results to client or invoke completion if specified + try { + ThrowInternalErrorIf(not state->results, "Defensive programming check failed"); + if constexpr (std::is_same_v) + // no completion defined, default behavior: send results to client + emit c->sendResult(batchId, reqId, *state->results); + else + // completion defined, invoke it + state->completion(*state->results); + } catch (const std::exception &e) { + // Uh oh, this should never happen! + Error() << "Caught unexepected exception in generic_do_async: " << e.what(); + emit c->sendError(false, RPC::ErrorCodes::Code_InternalError, e.what(), batchId, reqId); + } + }, + // default fail function just sends json rpc error "internal error: " + defaultTPFailFunc(c, batchId, reqId), + // lower is sooner, higher is later. Default 0. + priority + ); } void ServerBase::generic_async_to_bitcoind(Client *c, const RPC::BatchId batchId, const RPC::Message::Id & reqId, const QString &method, const QVariantList & params, const BitcoinDSuccessFunc & successFunc, - const BitcoinDErrorFunc & errorFunc) + const BitcoinDErrorFunc & errorFunc, + std::optional useCacheIfNotOlderThan) { if (UNLIKELY(QThread::currentThread() != c->thread())) { // Paranoia, in case I or a future programmer forgets this rule. @@ -890,7 +932,11 @@ void ServerBase::generic_async_to_bitcoind(Client *c, const RPC::BatchId batchId } }, // use default function on failure, sends json rpc error "internal error: " - defaultBDFailFunc(c, batchId, reqId) + defaultBDFailFunc(c, batchId, reqId), + // timeout + BitcoinDMgr::kDefaultTimeoutMS, + // optional response caching control + useCacheIfNotOlderThan ); } @@ -1068,7 +1114,8 @@ void Server::rpc_server_donation_address(Client *c, const RPC::BatchId batchId, emit c->sendResult(batchId, m.id, transformDefaultDonationAddressToBTCOrBCHOrLTC(*options, isNonBCH(), isLTC())); } /* static */ -QVariantMap Server::makeFeaturesDictForConnection(AbstractConnection *c, const QByteArray &genesisHash, const Options &opts, bool dsproof, bool hasCashTokens, int rpaStartingHeight) +QVariantMap Server::makeFeaturesDictForConnection(AbstractConnection *c, const QByteArray &genesisHash, const Options &opts, + bool dsproof, bool hasCashTokens, int rpaStartingHeight, bool hasBroadcastPackage) { QVariantMap r; if (!c) { @@ -1076,6 +1123,7 @@ QVariantMap Server::makeFeaturesDictForConnection(AbstractConnection *c, const Q Error() << __func__ << ": called with a nullptr for AbstractConnection FIXME!"; return r; } + assert(c->thread() == QThread::currentThread()); r["pruning"] = QVariant(); // null r["genesis_hash"] = QString(Util::ToHexFast(genesisHash)); r["server_version"] = ServerMisc::AppSubVersion; @@ -1094,6 +1142,8 @@ QVariantMap Server::makeFeaturesDictForConnection(AbstractConnection *c, const Q {"history_block_limit", opts.rpa.historyBlockLimit}, {"max_history", opts.rpa.maxHistory} }; + if (hasBroadcastPackage) + r["broadcast_package"] = true; QVariantMap hmap, hmapTor; if (opts.publicTcp.has_value()) @@ -1140,10 +1190,12 @@ QVariantMap Server::makeFeaturesDictForConnection(AbstractConnection *c, const Q void Server::rpc_server_features(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { const bool isBCH = coin == BTC::Coin::BCH; + const auto rsi = bitcoindmgr->getRpcSupportInfo(); emit c->sendResult(batchId, m.id, - makeFeaturesDictForConnection(c, storage->genesisHash(), *options, bitcoindmgr->hasDSProofRPC(), + makeFeaturesDictForConnection(c, storage->genesisHash(), *options, rsi.hasDSProofRPC, /* cashTokens = */ isBCH, - /* rpaStartHeight = */ storage->getConfiguredRpaStartHeight())); + /* rpaStartHeight = */ storage->getConfiguredRpaStartHeight(), + rsi.hasSubmitPackageRPC)); } void Server::rpc_server_peers_subscribe(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { @@ -1180,7 +1232,11 @@ void Server::rpc_server_version(Client *c, const RPC::BatchId batchId, const RPC if (l.size() == 1) // missing second arg, protocolVersion, default to our minimal protocol version "1.4" l.push_back(ServerMisc::MinProtocolVersion.toString()); - assert(l.size() == 2); + assert(l.size() >= 2); + if (l.size() > 2 && Debug::isEnabled()) { + const auto extraArgsStr = Json::toUtf8(l.mid(2), true).left(80); + Debug() << "Client " << c->id << " sent extra server.version args (ignored): " << extraArgsStr; + } if (c->info.alreadySentVersion) throw RPCError(QString("%1 already sent").arg(m.method)); @@ -1285,7 +1341,7 @@ void Server::rpc_blockchain_block_headers(Client *c, const RPC::BatchId batchId, throw RPCError("Invalid count"); const auto tip = storage->latestTip().first; if (tip < 0) throw InternalError("chain height is negative"); - static constexpr unsigned MAX_COUNT = 2016; ///< TODO: make this cofigurable. this is the current electrumx limit, for now. + static constexpr unsigned MAX_COUNT = 2016u; ///< TODO: make this cofigurable. this is the current electrumx limit, for now. count = std::min(std::min(unsigned(tip+1) - height, count), MAX_COUNT); ok = true; const unsigned cp_height = l.size() > 2 ? l.back().toUInt(&ok) : 0; @@ -1296,26 +1352,47 @@ void Server::rpc_blockchain_block_headers(Client *c, const RPC::BatchId batchId, throw RPCError(QString("header height + (count - 1) %1 must be <= cp_height %2 which must be <= chain height %3") .arg(height + (count - 1)).arg(cp_height).arg(tip)); } - generic_do_async(c, batchId, m.id, [height, count, cp_height, this] { + const bool headersAsList = c->info.protocolVersion >= Version{1, 6, 0}; // in protocol 1.6.0 we return a list, not a concatenated string + generic_do_async(c, batchId, m.id, [height, count, cp_height, headersAsList, this] { // EX doesn't seem to return error here if invalid height/no results, so we will do same. const auto hdrs = storage->headersFromHeight(height, std::min(count, MAX_COUNT)); - const size_t nHdrs = hdrs.size(), hdrSz = size_t(BTC::GetBlockHeaderSize()), hdrHexSz = hdrSz*2; - QByteArray hexHeaders(int(nHdrs * hdrHexSz), Qt::Uninitialized); - for (size_t i = 0, offset = 0; i < nHdrs; ++i, offset += hdrHexSz) { + const size_t nHdrs = hdrs.size(); + constexpr size_t hdrSz = BTC::GetBlockHeaderSize(), hdrHexSz = hdrSz*2u; + QVariantMap resp{ + {"count", quint32(nHdrs)}, + {"max", MAX_COUNT} + }; + auto getHeaderAndCheckSize = [&](size_t i) -> const QByteArray & { const auto & hdr = hdrs[i]; - if (UNLIKELY(hdr.size() != int(hdrSz))) { // ensure header looks the right size + if (hdr.size() != QByteArray::size_type(hdrSz)) [[unlikely]] { // ensure header looks the right size // this should never happen. Error() << "Header size from db height " << i + height << " is not " << hdrSz << " bytes! Database corruption likely! FIXME!"; throw RPCError("Server header store invalid", RPC::Code_InternalError); } - // fast, in-place conversion to hex - Util::ToHexFastInPlace(hdr, hexHeaders.data() + offset, hdrHexSz); - } - QVariantMap resp{ - {"hex" , QString(hexHeaders)}, // we cast to QString to prevent null for empty string "" - {"count", unsigned(hdrs.size())}, - {"max", MAX_COUNT} + return hdr; }; + if (headersAsList) { + // Protocol version 1.6.0 and above, return a list of hex headers + QVariantList headers; + headers.reserve(nHdrs); + for (size_t i = 0; i < nHdrs; ++i) { + const auto & hdr = getHeaderAndCheckSize(i); + headers.push_back(Util::ToHexFast(hdr)); + } + resp["headers"] = headers; + } else { + // Protocol version < 1.6.0, return a concatenated string of header hex + QByteArray hexHeaders(QByteArray::size_type(nHdrs * hdrHexSz), Qt::Uninitialized); + for (size_t i = 0, offset = 0; i < nHdrs; ++i, offset += hdrHexSz) { + const auto & hdr = getHeaderAndCheckSize(i); + // fast, in-place conversion to hex + Util::ToHexFastInPlace(hdr, hexHeaders.data() + offset, hdrHexSz); + } + if (hexHeaders.isEmpty()) + resp["hex"] = QString(""); // we cast to QString to prevent JSON null for empty string "" + else + resp["hex"] = hexHeaders; // use QByteArray if non-empty to save cycles + } if (count && cp_height) { // Note: it's possible for a reorg to happen and the chain height to be shortened in parellel in such // a way that lastHeight > chainHeight or cp_height > chainHeight, thus making this merkle branch query @@ -1336,16 +1413,30 @@ void Server::rpc_blockchain_estimatefee(Client *c, const RPC::BatchId batchId, c bool ok; int n = l.front().toInt(&ok); if (!ok || n < 0) - throw RPCError(QString("%1 parameter should be a single non-negative integer").arg(m.method)); + throw RPCError(QString("%1 first parameter should be a non-negative integer").arg(m.method)); + std::optional mode; + if (l.size() >= 2) { + // A second optional "mode" argument (string) + const auto &var = l.at(1); + if (!IsMetaTypeStringLike(var)) + throw RPCError(QString("%1 second parameter should be a string").arg(m.method)); + mode = var.toString(); + } QVariantList params; + const auto rsi = bitcoindmgr->getRpcSupportInfo(); // Flowee, BU, early ABC, bchd have a 1-arg estimate fee, newer ABC & BCHN -> 0 arg - if (!bitcoindmgr->isZeroArgEstimateFee()) + if (!rsi.isZeroArgEstimateFee) params.push_back(unsigned(n)); - if ((bitcoindmgr->isCoreLike()) - && bitcoindmgr->getBitcoinDVersion() >= Version{0,17,0}) { + if (rsi.hasEstimateSmartFee) { // Bitcoin Core removed the "estimatefee" RPC method entirely in version 0.17.0, in favor of "estimatesmartfee" + // (available starting in 0.15.0) + if (rsi.isTwoArgEstimateSmartFee && mode) { + // Core >= 0.16.0 supports a second optional "mode" argument (string), we force it to upper-case since + // very old Core versions wanted uppercase only here. + params.push_back(mode->toUpper()); + } generic_async_to_bitcoind(c, batchId, m.id, "estimatesmartfee", params, [](const RPC::Message &response){ // We don't validate what bitcoind returns. Sometimes if it has not enough information, it may // return no "feerate" but instead return an "errors" entry in the dict. This is fine. @@ -1418,7 +1509,7 @@ void Server::rpc_blockchain_header_get(Client *c, const RPC::BatchId batchId, co const QVariantList l(m.paramsList()); assert(!l.isEmpty()); // We support the first arg as either a 64-character hash or a numeric height - if (const auto var = l[0]; Compat::IsMetaType(var, QMetaType::Type::QString) || Compat::IsMetaType(var, QMetaType::Type::QByteArray)) { + if (const auto var = l[0]; IsMetaTypeStringLike(var)) { const BlockHash blockHash = parseFirstHashParamCommon(m, "Invalid block hash"); // first we need to figure out the height of this block hash -- query bitcoind generic_async_to_bitcoind(c, batchId, m.id, "getblockheader", {var, true}, @@ -1908,6 +1999,15 @@ void Server::LogFilter::Broadcast::onNewBlock() success.reset(); } +namespace { + void appendMaxBurnAmountForBTCToParams(QVariantList ¶ms) { + // bitcoin core 25.0+ requires specifying maxburnamount in sendrawtransaction call + // which also requires first sending maxfeerate, set to 0.1 BTC (default in Core) + params.push_back(0.1); + // set maxburnrate to max BTC supply to preserve pre-25.0 functionality + params.push_back(21'000'000); + } +} // namespace void Server::rpc_blockchain_transaction_broadcast(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { const QVariantList l = m.paramsList(); @@ -1919,26 +2019,21 @@ void Server::rpc_blockchain_transaction_broadcast(Client *c, const RPC::BatchId const QByteArray txkey = (!rawtxhex.isEmpty() ? BTC::HashOnce(Util::ParseHexFast(rawtxhex)).left(16) : QByteArrayLiteral("xx")); // no need to validate hex here -- bitcoind does validation for us! QVariantList params { rawtxhex } ; - if (isBTC() - && bitcoindmgr->getBitcoinDVersion() >= Version{0,25,0}) { + if (bitcoindmgr->getRpcSupportInfo().sendRawTransactionRequiresMaxBurnAmount) // bitcoin core 25.0+ requires specifying maxburnamount in sendrawtransaction call - // which also requires first sending maxfeerate, set to 0.1btc by default in core - params.append(0.1); - // set maxburnrate to max btc supply to preserve pre-25.0 functionality - params.append(21000000); - } + appendMaxBurnAmountForBTCToParams(params); generic_async_to_bitcoind(c, batchId, m.id, "sendrawtransaction", params, // print to log, echo bitcoind's reply to client - [size=rawtxhex.length()/2, c, this, txkey](const RPC::Message & reply){ + [size=size_t(rawtxhex.length()/2), c, this, txkey](const RPC::Message & reply){ QVariant ret = reply.result(); ++c->info.nTxSent; - c->info.nTxBytesSent += unsigned(size); - emit broadcastTxSuccess(unsigned(size)); + c->info.nTxBytesSent += size; + emit broadcastTxSuccess(1u, size); QByteArray logLine; { QTextStream ts{&logLine, QIODevice::WriteOnly}; ts << "Broadcast tx for client " << c->id; - if (!options->anonLogs) ts << ", size: " << size << " bytes, response: " << ret.toString(); + if (!options->anonLogs) ts << ", size: " << qulonglong(size) << " bytes, response: " << ret.toString(); } logFilter->broadcast(true, logLine, txkey); // Next, check if client is old and has the phishing exploit: @@ -2003,6 +2098,138 @@ void Server::rpc_blockchain_transaction_broadcast(Client *c, const RPC::BatchId ); // <-- do nothing right now, return without replying. Will respond when daemon calls us back in callbacks above. } +void Server::rpc_blockchain_transaction_broadcast_package(Client *c, const RPC::BatchId batchId, const RPC::Message &m) +{ + if (not bitcoindmgr->getRpcSupportInfo().hasSubmitPackageRPC) + // This is not supported on anything but Bitcoin Core 28.0.0 + throw RPCError("blockchain.transaction.broadcast_package is only available on Bitcoin Core and/or Bitcoin Knots" + " v28.0.0 or above", + RPC::ErrorCodes::Code_MethodNotFound); + + struct AsyncWorkResult { + QByteArray broadcast_key; + size_t packageSizeBytes = 0u, packageNTx = 0u; + QVariantList params; // array of raw tx hex strings, as was specified by incoming RPC param + bool verbose = false; // as was specified by incoming RPC param + }; + // We call generic_do_async to do some validation work that is non-trivial in a threadpool thread, and if that + // checks out, we proceed to call into bitcond with the `submitpackage` request. + generic_do_async(c, batchId, m.id, + // Work -- this runs in a threadpool thread (careful not to touch any outside objects from this lambda!!). + [paramsIn = m.paramsList(), maxBuffer = options->maxBuffer.load()]{ + AsyncWorkResult ret; + const QVariantList txnsIn = paramsIn[0].toList(); + QVariantList txns; + txns.reserve(txnsIn.size()); + const size_t maxPackageBytes = std::max(1000 * 1000, maxBuffer); // limit package size to max 1MB or maxBuffer, whichever is larger + size_t & packageSizeBytes = ret.packageSizeBytes, & packageNTx = ret.packageNTx; + bool oversized = false; + // -- Note we need the broadcast_key for broadcast fail filtering -- the key is a single sha256 of the last + // -- txn's hex + packageNTx + packageSizeBytes, but just the first 16 bytes of this hash are taken. + // -- Keeping the key small is essential to save cycles. + QByteArray & broadcast_key = ret.broadcast_key; + auto validateAndPushTxn = [&](const QVariant &vartx) { + if (!IsMetaTypeStringLike(vartx)) return false; + const QByteArray strhextx = vartx.toString().toUtf8().trimmed(); + const size_t hexSize = strhextx.size(); + if (oversized || (oversized = hexSize > kMaxTxHex) || strhextx.isEmpty()) return false; // bad txn size + const size_t txnSize = hexSize / 2; + if (!txnSize || hexSize % 2) return false; // bad size + if (oversized || (oversized = (packageSizeBytes += txnSize) > maxPackageBytes)) return false; // total size is oversized + txns.push_back(broadcast_key = strhextx); // shallow copy + ++packageNTx; + return true; + }; + // validate txn arg + if (txnsIn.isEmpty() || !std::all_of(txnsIn.begin(), txnsIn.end(), validateAndPushTxn)) + throw RPCError(QString("Invalid raw_txs argument; ") + + QString(!oversized ? "expected non-empty list of hex-encoded strings" + : "oversized")); + // validate optional verbose arg, if present + bool & verbose = ret.verbose; + if (paramsIn.size() >= 2) { // parse optional verbose arg + const auto [verbArg, verbArgOk] = parseBoolSemiLooselyButNotTooLoosely(paramsIn[1]); + if (!verbArgOk) + throw RPCError("Invalid verbose argument; expected boolean"); + verbose = verbArg; + } + // Calculate broadcast_key = Hash(lastTxHex + packageNTx + packageSizeBytes)[:16] + broadcast_key.reserve(broadcast_key.size() + sizeof(packageNTx) + sizeof(packageSizeBytes)); + broadcast_key.append(reinterpret_cast(&packageNTx), QByteArray::size_type(sizeof(packageNTx))); + broadcast_key.append(reinterpret_cast(&packageSizeBytes), QByteArray::size_type(sizeof(packageSizeBytes))); + broadcast_key = BTC::HashOnce(broadcast_key).left(16); + + QVariantList & params = ret.params; + params.reserve(3); + params.push_back(txns); + appendMaxBurnAmountForBTCToParams(params); + return ret; + }, + // Completion function after above async work completes -- this runs in this object's thread, only if `c` is still alive! + [c, batchId, mId = m.id, this](const AsyncWorkResult & res){ + // Forward RPC request to bitcoind via generic_async_to_bitcoind + const auto & [broadcast_key, packageSizeBytes, packageNTx, params, verbose] = res; + generic_async_to_bitcoind(c, batchId, mId, "submitpackage", params, + // print to log, echo bitcoind's reply to client iff verbose==true, otherwise prepare reply based on bitcond reply + [size = packageSizeBytes, ntx = packageNTx, c, this, broadcast_key, verbose](const RPC::Message &reply) { + const QVariantMap result = reply.result().toMap(); + const QString packageMsg = result.value("package_msg").toString(); + const bool success = 0 == packageMsg.compare("success", Qt::CaseInsensitive); + if (success) { + c->info.nTxSent += ntx; + c->info.nTxBytesSent += size; + emit broadcastTxSuccess(ntx, size); + } else + ++c->info.nTxBroadcastErrors; + QByteArray logLine; + { + QTextStream ts{&logLine, QIODevice::WriteOnly}; + ts << "Broadcast package for client " << c->id; + if (!options->anonLogs) + ts << ", size: " << size << " bytes, nTx: " << ntx << ", response: " << packageMsg; + } + logFilter->broadcast(true, logLine, broadcast_key); + if (verbose) + return QVariant{result}; // verbose mode, just return the bitcoind result object verbatim + // otherwise, non-verbose mode extract results from the "tx-results" dictionary + const QVariantMap txResultsMap = result.value("tx-results").toMap(); + QVariantMap ret = {{"success", success}}; + QVariantList errors; + bool warnIsBad = txResultsMap.isEmpty(); + for (const QVariant &item : txResultsMap) { + const QVariantMap m = item.toMap(); + if (m.isEmpty() || !Compat::IsMetaType(item, QMetaType::QVariantMap)) [[unlikely]] + warnIsBad = true; + else if (auto it = m.find("error"); it != m.end()) { + const QVariantMap err = {{"txid", m.value("txid", "???")}, {"error", it.value().toString()}}; + errors.push_back(err); + } + } + if (warnIsBad) [[unlikely]] { + Warning() << "Unexpected result format from bitcoind `submitpackage` RPC. Report this issue to" + " the " APPNAME " developers!"; + DebugM("Result from bitcoind was: ", Json::toUtf8(reply.result(), true)); + } + if (!errors.isEmpty()) + ret["errors"] = errors; + return QVariant{ret}; + }, + // error func, throw an RPCError that's formatted in a particular way + [c, this, broadcast_key](const RPC::Message &errResponse) { + ++c->info.nTxBroadcastErrors; + const auto errorMessage = errResponse.errorMessage(); + { + // This "logFilter" mechanism was added in Fulcrum 1.2.5 to suppress repeated broadcast fail spam + QByteArray logLine; + QTextStream{&logLine, QIODevice::WriteOnly} << "Broadcast package fail for client " << c->id + << ": " << errorMessage.left(120); + logFilter->broadcast(false, logLine, broadcast_key); + } + throw RPCError(QString("the transaction was rejected by network rules.\n\n%1\n").arg(errorMessage), + RPC::Code_App_BadRequest); + }); + }); +} void Server::rpc_blockchain_transaction_get(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { const QVariantList l = m.paramsList(); @@ -2204,7 +2431,7 @@ void Server::rpc_blockchain_transaction_unsubscribe(Client *c, const RPC::BatchI // DSPROOF void Server::rpc_blockchain_transaction_dsproof_get(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { - if (isNonBCH() || !bitcoindmgr->hasDSProofRPC()) + if (isNonBCH() || not bitcoindmgr->getRpcSupportInfo().hasDSProofRPC) throw RPCError("This server lacks dsproof support", RPC::ErrorCodes::Code_MethodNotFound); const auto dspid_or_txid = parseFirstHashParamCommon(m, "Invalid dsp hash or tx hash"); generic_do_async(c, batchId, m.id, [this, dspid_or_txid] { @@ -2220,7 +2447,7 @@ void Server::rpc_blockchain_transaction_dsproof_get(Client *c, const RPC::BatchI } void Server::rpc_blockchain_transaction_dsproof_list(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { - if (isNonBCH() || !bitcoindmgr->hasDSProofRPC()) + if (isNonBCH() || not bitcoindmgr->getRpcSupportInfo().hasDSProofRPC) throw RPCError("This server lacks dsproof support", RPC::ErrorCodes::Code_MethodNotFound); generic_do_async(c, batchId, m.id, [this] { DSProof::TxHashSet allDescendants; @@ -2234,20 +2461,20 @@ void Server::rpc_blockchain_transaction_dsproof_list(Client *c, const RPC::Batch QVariantList ret; ret.reserve(allDescendants.size()); for (const auto &txid : allDescendants) - ret.append(Util::ToHexFast(txid)); + ret.push_back(Util::ToHexFast(txid)); return ret; }); } void Server::rpc_blockchain_transaction_dsproof_subscribe(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { - if (isNonBCH() || !bitcoindmgr->hasDSProofRPC()) + if (isNonBCH() || not bitcoindmgr->getRpcSupportInfo().hasDSProofRPC) throw RPCError("This server lacks dsproof support", RPC::ErrorCodes::Code_MethodNotFound); const auto txid = parseFirstHashParamCommon(m, "Invalid tx hash"); impl_generic_subscribe(storage->dspSubs(), c, batchId, m, txid); } void Server::rpc_blockchain_transaction_dsproof_unsubscribe(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { - if (isNonBCH() || !bitcoindmgr->hasDSProofRPC()) + if (isNonBCH() || not bitcoindmgr->getRpcSupportInfo().hasDSProofRPC) throw RPCError("This server lacks dsproof support", RPC::ErrorCodes::Code_MethodNotFound); const auto txid = parseFirstHashParamCommon(m, "Invalid tx hash"); impl_generic_unsubscribe(storage->dspSubs(), c, batchId, m, txid); @@ -2391,6 +2618,25 @@ void Server::rpc_mempool_get_fee_histogram(Client *c, const RPC::BatchId batchId } emit c->sendResult(batchId, m.id, result); } + +void Server::rpc_mempool_get_info(Client *c, const RPC::BatchId batchId, const RPC::Message &m) +{ + constexpr int kMempoolInfoStaleThreshMS = 250; // if older than 250 msecs, refresh (this rate-limits spammers) + generic_async_to_bitcoind(c, batchId, m.id, "getmempoolinfo", QVariantList{}, + [](const RPC::Message &response){ + // to preserve privacy, only grab the following keys, omitting size, bytes, etc + constexpr const char * desiredKeys[] = { "mempoolminfee", "minrelaytxfee", "incrementalrelayfee", + "unbroadcastcount", "fullrbf" }; + QVariantMap m = response.result().toMap(), ret; + for (const auto & key : desiredKeys) + if (m.contains(key)) ret[key] = m.value(key); + return ret; + }, + BitcoinDErrorFunc{}, + kMempoolInfoStaleThreshMS + ); +} + void Server::rpc_daemon_passthrough(Client *c, const RPC::BatchId batchId, const RPC::Message &m) { Options::Subnet subnet; @@ -2403,7 +2649,7 @@ void Server::rpc_daemon_passthrough(Client *c, const RPC::BatchId batchId, const QVariantMap pm; QVariant method, params; if (!m.isParamsMap() || (pm = m.paramsMap()).size() < 1 - || (!Compat::IsMetaType(method = pm.value("method"), QMetaType::QString) && !Compat::IsMetaType(method, QMetaType::QByteArray)) + || !IsMetaTypeStringLike(method = pm.value("method")) || (!Compat::IsMetaType(params = pm.value("params"), QMetaType::QVariantList) && !params.isNull())) throw RPCError("Expected a dictionary of the form: { \"method\": \"string\", \"params\": [...] }"); const QString methodStr = method.toString(); @@ -2422,6 +2668,7 @@ void Server::rpc_daemon_passthrough(Client *c, const RPC::BatchId batchId, const // --- Server::StaticData Definitions --- #define HEY_COMPILER_PUT_STATIC_HERE(x) decltype(x) x #define PR RPC::Method::PosParamRange +#define NO_LIMIT RPC::Method::NO_POS_PARAM_LIMIT #define MP(x) static_cast(&Server :: x) // wrapper to cast from narrow method pointer to ServerBase::Member_t HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::dispatchTable); HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::methodMap); @@ -2434,7 +2681,7 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){ { {"server.features", true, false, PR{0,0}, }, MP(rpc_server_features) }, { {"server.peers.subscribe", true, false, PR{0,0}, }, MP(rpc_server_peers_subscribe) }, { {"server.ping", true, false, PR{0,0}, }, MP(rpc_server_ping) }, - { {"server.version", true, false, PR{0,2}, }, MP(rpc_server_version) }, + { {"server.version", true, false, PR{0,NO_LIMIT}, }, MP(rpc_server_version) }, { {"blockchain.address.get_balance", true, false, PR{1,2}, }, MP(rpc_blockchain_address_get_balance) }, { {"blockchain.address.get_first_use", true, false, PR{1,1}, }, MP(rpc_blockchain_address_get_first_use) }, @@ -2447,7 +2694,7 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){ { {"blockchain.block.header", true, false, PR{1,2}, }, MP(rpc_blockchain_block_header) }, { {"blockchain.block.headers", true, false, PR{2,3}, }, MP(rpc_blockchain_block_headers) }, - { {"blockchain.estimatefee", true, false, PR{1,1}, }, MP(rpc_blockchain_estimatefee) }, + { {"blockchain.estimatefee", true, false, PR{1,2}, }, MP(rpc_blockchain_estimatefee) }, { {"blockchain.headers.get_tip", true, false, PR{0,0}, }, MP(rpc_blockchain_headers_get_tip) }, { {"blockchain.headers.subscribe", true, false, PR{0,0}, }, MP(rpc_blockchain_headers_subscribe) }, { {"blockchain.headers.unsubscribe", true, false, PR{0,0}, }, MP(rpc_blockchain_headers_unsubscribe) }, @@ -2463,6 +2710,7 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){ { {"blockchain.scripthash.unsubscribe", true, false, PR{1,1}, }, MP(rpc_blockchain_scripthash_unsubscribe) }, { {"blockchain.transaction.broadcast", true, false, PR{1,1}, }, MP(rpc_blockchain_transaction_broadcast) }, + { {"blockchain.transaction.broadcast_package", true, false, PR{1,2}, }, MP(rpc_blockchain_transaction_broadcast_package) }, { {"blockchain.transaction.get", true, false, PR{1,2}, }, MP(rpc_blockchain_transaction_get) }, { {"blockchain.transaction.get_confirmed_blockhash", true, false, PR{1,2}, }, MP(rpc_blockchain_transaction_get_confirmed_blockhash) }, { {"blockchain.transaction.get_height", true, false, PR{1,1}, }, MP(rpc_blockchain_transaction_get_height) }, @@ -2487,8 +2735,10 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){ { {"daemon.passthrough", true, false, PR{0,0}, RPC::KeySet{{"method"}}, true /* allow unknown kwargs, since "params" is optional */ }, MP(rpc_daemon_passthrough) }, { {"mempool.get_fee_histogram", true, false, PR{0,0}, }, MP(rpc_mempool_get_fee_histogram) }, + { {"mempool.get_info", true, false, PR{0,0}, }, MP(rpc_mempool_get_info) }, }; #undef MP +#undef NO_LIMIT #undef PR #undef HEY_COMPILER_PUT_STATIC_HERE namespace { diff --git a/src/Servers.h b/src/Servers.h index a568d770..c905041b 100644 --- a/src/Servers.h +++ b/src/Servers.h @@ -33,6 +33,8 @@ #include #include +#include +#include // for std::nullptr_t #include // for shared_ptr #include #include @@ -269,7 +271,6 @@ protected slots: ~RPCErrorWithDisconnect() override; }; - using AsyncWorkFunc = std::function; struct DontAutoSendReply_t {}; static constexpr DontAutoSendReply_t DontAutoSendReply{}; using BitcoinDSuccessFuncResult = std::variant; @@ -280,14 +281,25 @@ protected slots: /// the work for later and handles sending the response (returned from work) to the client as well as sending /// any errors to the client. The `work` functor may throw RPCError, in which case code and message will be /// sent instead. Note that all other exceptions also end up sent to the client as "internal error: MESSAGE". - void generic_do_async(Client *client, RPC::BatchId, const RPC::Message::Id &reqId, const AsyncWorkFunc & work, int priority = 0); + /// The optional `CompletionFunc` is used if you want to override the default behavior of sending results to + /// the client to do additional processing (see rpc_blockchain_transaction_broadcast_package for example). + template + requires + // WorkFunc must not return void + (not std::is_same_v>) + // and CompletionFunc must either be nullptr or must accept 1 arg, the result of invoking WorkFunc + && (std::is_same_v || requires (CompletionFunc c, WorkFunc w) { c(w()); }) + void generic_do_async(Client *client, RPC::BatchId, const RPC::Message::Id &reqId, WorkFunc && work, + CompletionFunc && = {}, int priority = 0); + void generic_async_to_bitcoind(Client *client, RPC::BatchId batchId, ///< if running in batch context, will be !batchId.isNull() const RPC::Message::Id & reqId, ///< the original client request id const QString &method, ///< bitcoind method to invoke const QVariantList ¶ms, ///< params for bitcoind method const BitcoinDSuccessFunc & successFunc, - const BitcoinDErrorFunc & errorFunc = BitcoinDErrorFunc()); + const BitcoinDErrorFunc & errorFunc = BitcoinDErrorFunc(), + std::optional useCacheIfNotOlderThan = std::nullopt); /// Subclasses may set this pointer if they wish the generic_do_async function above to use a private/custom /// threadpool. Otherwise the app-global ::AppThreadPool() will be used for generic_do_async(). @@ -350,7 +362,8 @@ class Server : public ServerBase /// NOTE: Be sure to only ever call this function from the same thread as the AbstractConnection (first arg) instance! static QVariantMap makeFeaturesDictForConnection(AbstractConnection *, const QByteArray &genesisHash, const Options & options, bool hasDSProofRPC, bool hasCashTokens, - int rpaStartingHeight /* <=-1 means no RPA */); + int rpaStartingHeight /* <=-1 means no RPA */, + bool hasBroadcastPackage); virtual QString prettyName() const override; @@ -362,9 +375,8 @@ class Server : public ServerBase /// Used to notify clients that are subscribed to headers that a new header has arrived. void newHeader(unsigned height, const QByteArray &header); - /// Emitted for the SrvMgr to update its counters of the number of tx's successfully broadcast. The argument - /// is a size in bytes. - void broadcastTxSuccess(unsigned); + /// Emitted for the SrvMgr to update its counters of the number of tx's successfully broadcast, and total byte size. + void broadcastTxSuccess(size_t nTx, size_t nBytes); /// Emitted when the SubsMgr throws LimitReached inside rpc_scripthash_subscribe. Conneced to SrvMgr which /// will iterate through all perIPData instances and kick the ip address with the most subs. @@ -408,6 +420,7 @@ class Server : public ServerBase void rpc_blockchain_scripthash_unsubscribe(Client *, RPC::BatchId, const RPC::Message &); // fully implemented // transaction void rpc_blockchain_transaction_broadcast(Client *, RPC::BatchId, const RPC::Message &); // fully implemented + void rpc_blockchain_transaction_broadcast_package(Client *, RPC::BatchId, const RPC::Message &); // protocol v1.6.0 void rpc_blockchain_transaction_get(Client *, RPC::BatchId, const RPC::Message &); // fully implemented void rpc_blockchain_transaction_get_confirmed_blockhash(Client *, RPC::BatchId, const RPC::Message &); // protocol v1.5.2 void rpc_blockchain_transaction_get_height(Client *, RPC::BatchId, const RPC::Message &); // fully implemented @@ -431,6 +444,7 @@ class Server : public ServerBase void rpc_blockchain_utxo_get_info(Client *, RPC::BatchId, const RPC::Message &); // fully implemented // mempool void rpc_mempool_get_fee_histogram(Client *, RPC::BatchId, const RPC::Message &); // fully implemented + void rpc_mempool_get_info(Client *, RPC::BatchId, const RPC::Message &); // protocol v1.6.0 // daemon void rpc_daemon_passthrough(Client *, RPC::BatchId, const RPC::Message &); // protocol v1.5.2 @@ -628,8 +642,8 @@ class Client : public RPC::ElectrumConnection ~Client() override; struct Info { - int errCt = 0; ///< this gets incremented for each peerError. If errCt - nRequests >= 10, then we disconnect the client. - int nRequestsRcv = 0; ///< the number of request messages that were non-errors that the client sent us + unsigned errCt = 0; ///< this gets incremented for each peerError. If errCt - nRequestsRcv >= 10, then we disconnect the client. + unsigned nRequestsRcv = 0; ///< the number of request messages that were non-errors that the client sent us // server.version info the client sent us QString userAgent = "Unknown"; //< the exact useragent string as the client sent us. @@ -637,7 +651,8 @@ class Client : public RPC::ElectrumConnection inline Version uaVersion() const { return Version(userAgent); } Version protocolVersion = {1,4,0}; ///< defaults to 1,4,0 if client says nothing. bool alreadySentVersion = false; - unsigned nTxSent = 0, nTxBroadcastErrors = 0, nTxBytesSent = 0; + unsigned nTxSent = 0u, nTxBroadcastErrors = 0u; + size_t nTxBytesSent = 0u; }; Info info; diff --git a/src/SrvMgr.cpp b/src/SrvMgr.cpp index f30e35f1..21908d18 100644 --- a/src/SrvMgr.cpp +++ b/src/SrvMgr.cpp @@ -178,7 +178,10 @@ void SrvMgr::startServers() // same situation here as above -- servers kick the client in question immediately connect(this, &SrvMgr::clientIsBanned, srv, qOverload(&ServerBase::killClient)); // tally tx broadcasts (lifetime) - connect(srv, &Server::broadcastTxSuccess, this, [this](unsigned bytes){ ++numTxBroadcasts; txBroadcastBytesTotal += bytes; }); + connect(srv, &Server::broadcastTxSuccess, this, [this](size_t ntx, size_t bytes){ + numTxBroadcasts += ntx; + txBroadcastBytesTotal += bytes; + }); // kicking connect(this, &SrvMgr::kickById, srv, qOverload(&ServerBase::killClient)); @@ -601,4 +604,5 @@ void SrvMgr::globalSubsLimitReached() } /// Thread-Safe. Returns whether bitcoind currently probes as having the dsproof RPC. -bool SrvMgr::hasDSProofRPC() const { return bitcoindmgr && bitcoindmgr->hasDSProofRPC(); } +bool SrvMgr::hasDSProofRPC() const { return bitcoindmgr && bitcoindmgr->getRpcSupportInfo().hasDSProofRPC; } +bool SrvMgr::hasSubmitPackageRPC() const { return bitcoindmgr && bitcoindmgr->getRpcSupportInfo().hasSubmitPackageRPC; } diff --git a/src/SrvMgr.h b/src/SrvMgr.h index 49b2390d..d5632e01 100644 --- a/src/SrvMgr.h +++ b/src/SrvMgr.h @@ -83,8 +83,11 @@ class SrvMgr : public Mgr, public TimersByNameMixin std::shared_ptr findExistingPerIPData(const QHostAddress &address) { return perIPData.getOrCreate(address, false); } /// Thread-Safe. Returns whether bitcoind currently probes as having the dsproof RPC. - /// This just forwards the call to BitcoinDMgr::hasDSProofRPC(). + /// This just forwards the call to BitcoinDMgr::getRpcSupportInfo.hasDSProofRPC. bool hasDSProofRPC() const; + /// Thread-Safe. Returns whether bitcoind currently probes as having the submitpackage RPC. + /// This just forwards the call to BitcoinDMgr::getRpcSupportInfo.hasSubmitPackageRPC. + bool hasSubmitPackageRPC() const; signals: /// Notifies all blockchain.headers.subscribe'd clients for the entire server about a new header. diff --git a/src/Storage.cpp b/src/Storage.cpp index ffd944ce..3e512fa6 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -2899,7 +2899,7 @@ void Storage::loadCheckTxHash2TxNumMgr() // NOTE: this must be called *after* loadCheckTxNumsFileAndBlkInfo(), because it needs a valid p->txNumNext void Storage::loadCheckUTXOsInDB() { - FatalAssert(!!p->db.utxoset, __func__, ": Utxo set db is not open"); + FatalAssert(p->db.utxoset, __func__, ": Utxo set db is not open"); if (options->doSlowDbChecks) { Log() << "CheckDB: Verifying utxo set (this may take some time) ..."; @@ -3031,7 +3031,7 @@ void Storage::loadCheckUTXOsInDB() // NOTE: this must be called *after* loadCheckTxNumsFileAndBlkInfo(), because it needs a valid p->txNumNext void Storage::loadCheckShunspentInDB() { - FatalAssert(!!p->db.shunspent, __func__, ": Shunspent db is not open"); + FatalAssert(p->db.shunspent, __func__, ": Shunspent db is not open"); if (options->doSlowDbChecks < 2) // this is so slow it requires -C -C be specified return; @@ -3119,7 +3119,7 @@ void Storage::loadCheckShunspentInDB() void Storage::loadCheckRpaDB() { - FatalAssert(!!p->db.rpa, __func__, ": RPA db is not open"); + FatalAssert(p->db.rpa, __func__, ": RPA db is not open"); const bool doSlowChecks = options->doSlowDbChecks; const bool doNeededCheck = isRpaNeedsFullCheck(); @@ -3359,7 +3359,7 @@ void Storage::clampRpaEntries_nolock(rocksdb::WriteBatch *batch, BlockHeight fro void Storage::loadCheckEarliestUndo() { - FatalAssert(p->db.undo && p->db, __func__, ": Undo column family is not open"); + FatalAssert(p->db.undo && p->db, __func__, ": Undo column family is not open"); const Tic t0; unsigned ctr = 0; diff --git a/src/ThreadPool.cpp b/src/ThreadPool.cpp index 7d7db7ce..479256d2 100644 --- a/src/ThreadPool.cpp +++ b/src/ThreadPool.cpp @@ -21,6 +21,8 @@ #include +#include + namespace { constexpr bool debugPrt = false; } @@ -87,11 +89,11 @@ void ThreadPool::submitWork(QObject *context, const VoidFunc & work, const VoidF Warning() << "A ThreadPool job failed with the error message: " << msg; }; const FailFunc & failFuncToUse (fail ? fail : defaultFail); - Job *job = new Job(context, this, work, completion, failFuncToUse); - QObject::connect(job, &QObject::destroyed, this, [this](QObject *){ --extant;}, Qt::DirectConnection); + std::unique_ptr job(new Job(context, this, work, completion, failFuncToUse)); // must use `new` because private c'tor + QObject::connect(job.get(), &QObject::destroyed, this, [this](QObject *){ --extant;}, Qt::DirectConnection); if (const auto njobs = ++extant; njobs > extantLimit) { ++noverflows; - delete job; // will decrement extant on delete + job.reset(); // will decrement extant on delete const auto msg = QString("Job limit exceeded (%1)").arg(njobs); failFuncToUse(msg); return; @@ -105,17 +107,17 @@ void ThreadPool::submitWork(QObject *context, const VoidFunc & work, const VoidF const auto num = ++ctr; job->setObjectName(QStringLiteral("Job %1 for '%2'").arg(num).arg( context ? context->objectName() : QStringLiteral(""))); if constexpr (debugPrt) { - QObject::connect(job, &Job::started, this, [n=job->objectName()]{ + QObject::connect(job.get(), &Job::started, this, [n=job->objectName()]{ Debug() << n << " -- started"; }, Qt::DirectConnection); - QObject::connect(job, &Job::completed, this, [n=job->objectName()]{ + QObject::connect(job.get(), &Job::completed, this, [n=job->objectName()]{ Debug() << n << " -- completed"; }, Qt::DirectConnection); - QObject::connect(job, &Job::failed, this, [n=job->objectName()](const QString &msg){ + QObject::connect(job.get(), &Job::failed, this, [n=job->objectName()](const QString &msg){ Debug() << n << " -- failed: " << msg; }, Qt::DirectConnection); } - pool->start(job, priority); + pool->start(job.release(), priority); } bool ThreadPool::shutdownWaitForJobs(int timeout_ms) diff --git a/src/Util.cpp b/src/Util.cpp index 703db32d..57658b6c 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -813,7 +813,6 @@ QString Log::colorize(const QString &str, Color c) { } template <> Log & Log::operator<<(const Color &c) { setColor(c); return *this; } -template <> Log & Log::operator<<(const std::string &t) { s << t.c_str(); return *this; } Debug::~Debug() { diff --git a/src/Util.h b/src/Util.h index 1224b633..764d0feb 100644 --- a/src/Util.h +++ b/src/Util.h @@ -41,6 +41,7 @@ #include #include #include +#include /* used in macros */ #include #include #include @@ -69,6 +70,14 @@ class QHostAddress; #define LIKELY(bool_expr) EXPECT(int(bool(bool_expr)), 1) #define UNLIKELY(bool_expr) EXPECT(int(bool(bool_expr)), 0) +// Allow for QTextStream to work on std::string and std::string_view +template +requires std::constructible_from +QTextStream & operator<<(QTextStream & ts, const StringOrSV &s) { + const std::string_view sv{s}; + return ts << QString::fromUtf8(QByteArray::fromRawData(sv.data(), sv.size())); +} + /// Super class of Debug, Warning, Error classes. Can be instantiated for regular log messages. class Log { @@ -98,7 +107,7 @@ class Log /// Used by the DebugM macros, etc. Unpacks all of its args using operator<< for each arg. template - Log & operator()(Args&& ...args) { ((*this) << ... << args); return *this; } + Log & operator()(Args&& ...args) { return ((*this) << ... << args); } protected: static QString colorString(Color c); @@ -108,14 +117,11 @@ class Log int level = 0; Color color = Normal; QString str = ""; - QTextStream s = QTextStream(&str, QIODevice::WriteOnly); + QTextStream s{&str, QIODevice::WriteOnly|QIODevice::Text}; }; - // specialization to set the color. template <> Log & Log::operator<<(const Color &); -// specialization for std::string -template <> Log & Log::operator<<(const std::string &t); /** \brief Stream-like class to print a debug message to the app's logging facility Example: @@ -260,10 +266,22 @@ class Fatal : public Log #define ErrorM(...) (Error()(__VA_ARGS__)) #define FatalM(...) (Fatal()(__VA_ARGS__)) -#define FatalAssert(b,...) \ - do { \ - if (!(b)) \ - FatalM("ASSERTION FAILED: \"", #b, "\" - ", __VA_ARGS__); \ +#define FatalAssert(b, ...) \ + do { \ + if (not static_cast(b)) [[unlikely]] { \ + const auto sl = std::source_location::current(); \ + FatalM("ASSERTION FAILED: \"", #b, "\" - ", sl.file_name(), ":", sl.line(), \ + " - " __VA_OPT__(,) __VA_ARGS__); \ + } \ + } while (0) + +#define ThrowInternalErrorIf(b, ...) \ + do { \ + if (static_cast(b)) [[unlikely]] { \ + const auto sl = std::source_location::current(); \ + throw InternalError(Util::StringifyMany( \ + "INTERNAL ERROR: \"", #b, "\" - ", sl.file_name(), ":", sl.line(), " - " __VA_OPT__(,) __VA_ARGS__)); \ + } \ } while (0) namespace Util { @@ -1151,6 +1169,14 @@ namespace Util { return std::construct_at(obj, std::forward(args)...); } + template + QString StringifyMany(Args && ...args) { + QString s; + if constexpr (sizeof...(Args)) + (QTextStream(&s, QIODevice::WriteOnly|QIODevice::Text) << ... << args); + return s; + } + } // end namespace Util /// Kind of like Go's "defer" statement. Call a lambda (for clean-up code) at scope end.