Merge branch 'rbf_opts-28+knots' into bloom_default-28+knots

Author: luke-jr

configure.ac

@@ -927,6 +927,21 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]],
  [ AC_MSG_RESULT([no])]
 )
 
+AC_MSG_CHECKING(for compatible sysinfo call)
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/sysinfo.h>]],
+ [[
+  struct sysinfo info;
+  int rv = sysinfo(&info);
+  unsigned long test = info.freeram + info.bufferram + info.mem_unit;
+ ]])],
+ [
+  AC_MSG_RESULT(yes);
+  AC_DEFINE(HAVE_LINUX_SYSINFO, 1, [Define this symbol if you have a Linux-compatible sysinfo call])
+ ],[
+  AC_MSG_RESULT(no)
+ ]
+)
+
 dnl Check for posix_fallocate
 AC_MSG_CHECKING([for posix_fallocate])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[

src/bitcoin-cli.cpp

@@ -22,6 +22,10 @@
 #include <malloc.h>
 #endif
 
+#ifdef HAVE_LINUX_SYSINFO
+#include <sys/sysinfo.h>
+#endif
+
 #include <cstdlib>
 #include <locale>
 #include <stdexcept>
@@ -110,3 +114,39 @@ int64_t GetStartupTime()
 {
     return nStartupTime;
 }
+
+size_t g_low_memory_threshold = 10 * 1024 * 1024 /* 10 MB */;
+
+bool SystemNeedsMemoryReleased()
+{
+    if (g_low_memory_threshold <= 0) {
+        // Intentionally bypass other metrics when disabled entirely
+        return false;
+    }
+#ifdef WIN32
+    MEMORYSTATUSEX mem_status;
+    mem_status.dwLength = sizeof(mem_status);
+    if (GlobalMemoryStatusEx(&mem_status)) {
+        if (mem_status.dwMemoryLoad >= 99 ||
+            mem_status.ullAvailPhys < g_low_memory_threshold ||
+            mem_status.ullAvailVirtual < g_low_memory_threshold) {
+            LogPrintf("%s: YES: %s%% memory load; %s available physical memory; %s available virtual memory\n", __func__, int(mem_status.dwMemoryLoad), size_t(mem_status.ullAvailPhys), size_t(mem_status.ullAvailVirtual));
+            return true;
+        }
+    }
+#endif
+#ifdef HAVE_LINUX_SYSINFO
+    struct sysinfo sys_info;
+    if (!sysinfo(&sys_info)) {
+        // Explicitly 64-bit in case of 32-bit userspace on 64-bit kernel
+        const uint64_t free_ram = uint64_t(sys_info.freeram) * sys_info.mem_unit;
+        const uint64_t buffer_ram = uint64_t(sys_info.bufferram) * sys_info.mem_unit;
+        if (free_ram + buffer_ram < g_low_memory_threshold) {
+            LogPrintf("%s: YES: %s free RAM + %s buffer RAM\n", __func__, free_ram, buffer_ram);
+            return true;
+        }
+    }
+#endif
+    // NOTE: sysconf(_SC_AVPHYS_PAGES) doesn't account for caches on at least Linux, so not safe to use here
+    return false;
+}

src/common/system.cpp

@@ -23,6 +23,10 @@ std::string ShellEscape(const std::string& arg);
 void runCommand(const std::string& strCommand);
 #endif
 
+extern size_t g_low_memory_threshold;
+
+bool SystemNeedsMemoryReleased();
+
 /**
  * Return the number of cores available on the current system.
  * @note This does count virtual cores, such as those provided by HyperThreading.

src/common/system.h

@@ -492,6 +492,7 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-lowmem=<n>", strprintf("If system available memory falls below <n> MiB, flush caches (0 to disable, default: %s)", g_low_memory_threshold / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -647,7 +648,8 @@ void SetupServerArgs(ArgsManager& argsman)
                              "is of this size or less (default: %u)",
                              MAX_OP_RETURN_RELAY),
                    ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
-    argsman.AddArg("-mempoolfullrbf", strprintf("(DEPRECATED) Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", DEFAULT_MEMPOOL_FULL_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-mempoolfullrbf", strprintf("Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", (DEFAULT_MEMPOOL_RBF_POLICY == RBFPolicy::Always)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-mempoolreplacement", strprintf("Set to 0 to disable RBF entirely, \"fee,optin\" to honour RBF opt-out signal, or \"fee,-optin\" to always RBF aka full RBF (default: %s)", "fee,-optin"), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
     argsman.AddArg("-permitbaremultisig", strprintf("Relay transactions creating non-P2SH multisig outputs (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY,
                    OptionsCategory::NODE_RELAY);
     argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)",
@@ -1540,6 +1542,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     bool do_reindex{args.GetBoolArg("-reindex", false)};
     const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)};
 
+    if (gArgs.IsArgSet("-lowmem")) {
+        g_low_memory_threshold = gArgs.GetIntArg("-lowmem", 0 /* not used */) * 1024 * 1024;
+    }
+    if (g_low_memory_threshold > 0) {
+        LogPrintf("* Flushing caches if available system memory drops below %s MiB\n", g_low_memory_threshold / 1024 / 1024);
+    }
+
+    if (mempool_opts.rbf_policy == RBFPolicy::Always) {
+        nLocalServices = ServiceFlags(nLocalServices | NODE_REPLACE_BY_FEE);
+    }
+
     for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) {
         bilingual_str mempool_error;
         node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error);

src/init.cpp

@@ -20,6 +20,7 @@
 #include <stdint.h>
 #include <string>
 #include <tuple>
+#include <variant>
 #include <vector>
 
 class BanMan;
@@ -214,7 +215,7 @@ class Node
     virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0;
 
     //! Broadcast transaction.
-    virtual node::TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0;
+    virtual node::TransactionError broadcastTransaction(CTransactionRef tx, const std::variant<CAmount, CFeeRate>& max_tx_fee, std::string& err_string) = 0;
 
     //! Get wallet loader.
     virtual WalletLoader& walletLoader() = 0;

src/interfaces/node.h

@@ -226,7 +226,7 @@ struct NewMempoolTransactionInfo {
      * This boolean indicates whether the transaction was added
      * without enforcing mempool fee limits.
      */
-    const bool m_mempool_limit_bypassed;
+    const ignore_rejects_type m_ignore_rejects;
     /* This boolean indicates whether the transaction is part of a package. */
     const bool m_submitted_in_package;
     /*
@@ -239,11 +239,11 @@ struct NewMempoolTransactionInfo {
 
     explicit NewMempoolTransactionInfo(const CTransactionRef& tx, const CAmount& fee,
                                        const int64_t vsize, const unsigned int height,
-                                       const bool mempool_limit_bypassed, const bool submitted_in_package,
+                                       const ignore_rejects_type& ignore_rejects, const bool submitted_in_package,
                                        const bool chainstate_is_current,
                                        const bool has_no_mempool_parents)
         : info{tx, fee, vsize, height},
-          m_mempool_limit_bypassed{mempool_limit_bypassed},
+          m_ignore_rejects{ignore_rejects},
           m_submitted_in_package{submitted_in_package},
           m_chainstate_is_current{chainstate_is_current},
           m_has_no_mempool_parents{has_no_mempool_parents} {}

src/kernel/mempool_entry.h

@@ -15,14 +15,16 @@
 
 class ValidationSignals;
 
+enum class RBFPolicy { Never, OptIn, Always };
+
 /** Default for -maxmempool, maximum megabytes of mempool memory usage */
 static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300};
 /** Default for -maxmempool when blocksonly is set */
 static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5};
 /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */
 static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336};
-/** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */
-static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{true};
+/** Default for -mempoolreplacement; must update docs in init.cpp manually */
+static constexpr RBFPolicy DEFAULT_MEMPOOL_RBF_POLICY{RBFPolicy::Always};
 /** Whether to fall back to legacy V1 serialization when writing mempool.dat */
 static constexpr bool DEFAULT_PERSIST_V1_DAT{false};
 /** Default for -acceptnonstdtxn */
@@ -55,7 +57,7 @@ struct MemPoolOptions {
     std::optional<unsigned> max_datacarrier_bytes{DEFAULT_ACCEPT_DATACARRIER ? std::optional{MAX_OP_RETURN_RELAY} : std::nullopt};
     bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG};
     bool require_standard{true};
-    bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF};
+    RBFPolicy rbf_policy{DEFAULT_MEMPOOL_RBF_POLICY};
     bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT};
     MemPoolLimits limits{};
 

src/kernel/mempool_options.h

@@ -355,7 +355,7 @@ class NodeImpl : public Node
         if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin;
         return {};
     }
-    TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override
+    TransactionError broadcastTransaction(CTransactionRef tx, const std::variant<CAmount, CFeeRate>& max_tx_fee, std::string& err_string) override
     {
         return BroadcastTransaction(*m_context, std::move(tx), err_string, max_tx_fee, /*relay=*/ true, /*wait_callback=*/ false);
     }

src/node/interfaces.cpp

@@ -12,10 +12,12 @@
 #include <consensus/amount.h>
 #include <kernel/chainparams.h>
 #include <logging.h>
+#include <node/interface_ui.h>
 #include <policy/feerate.h>
 #include <policy/policy.h>
 #include <tinyformat.h>
 #include <util/moneystr.h>
+#include <util/string.h>
 #include <util/translation.h>
 
 #include <chrono>
@@ -92,9 +94,58 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP
         return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())};
     }
 
-    mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf);
-    if (!mempool_opts.full_rbf) {
-        LogInfo("Warning: mempoolfullrbf=0 set but deprecated and will be removed in a future release\n");
+    if (argsman.IsArgSet("-mempoolreplacement") || argsman.IsArgSet("-mempoolfullrbf")) {
+        // Generally, mempoolreplacement overrides mempoolfullrbf, but the latter is used to infer intent in some cases
+        std::optional<bool> optin_flag;
+        bool fee_flag{false};
+        if (argsman.GetBoolArg("-mempoolreplacement", false)) {
+            fee_flag = true;
+        } else {
+            for (auto& opt : util::SplitString(argsman.GetArg("-mempoolreplacement", ""), ",+")) {
+                if (opt == "optin") {
+                    optin_flag = true;
+                } else if (opt == "-optin") {
+                    optin_flag = false;
+                } else if (opt == "fee") {
+                    fee_flag = true;
+                }
+            }
+        }
+        if (optin_flag.value_or(false)) {
+            // "optin" is explicitly specified
+            mempool_opts.rbf_policy = RBFPolicy::OptIn;
+        } else if (argsman.GetBoolArg("-mempoolfullrbf", false)) {
+            const bool mempoolreplacement_false{argsman.IsArgSet("-mempoolreplacement") && !(fee_flag || optin_flag.has_value())};
+            if (mempoolreplacement_false) {
+                // This is a contradiction, but override rather than error
+                InitWarning(_("False mempoolreplacement option contradicts true mempoolfullrbf; disallowing all RBF"));
+                mempool_opts.rbf_policy = RBFPolicy::Never;
+            } else {
+                mempool_opts.rbf_policy = RBFPolicy::Always;
+            }
+        } else if (!optin_flag.value_or(true)) {
+            // "-optin" is explicitly specified
+            mempool_opts.rbf_policy = fee_flag ? RBFPolicy::Always : RBFPolicy::Never;
+        } else if (fee_flag) {
+            // Just "fee" by itself
+            if (!argsman.GetBoolArg("-mempoolfullrbf", true)) {
+                mempool_opts.rbf_policy = RBFPolicy::OptIn;
+            } else {
+                // Fallback to default, unless it's been changed to Never
+                if (mempool_opts.rbf_policy == RBFPolicy::Never) {
+                    mempool_opts.rbf_policy = RBFPolicy::Always;
+                }
+            }
+        } else if (!argsman.IsArgSet("-mempoolreplacement")) {
+            // mempoolfullrbf is always explicitly false here
+            // Fallback to default, as long as it isn't Always
+            if (mempool_opts.rbf_policy == RBFPolicy::Always) {
+                mempool_opts.rbf_policy = RBFPolicy::OptIn;
+            }
+        } else {
+            // mempoolreplacement is explicitly false here
+            mempool_opts.rbf_policy = RBFPolicy::Never;
+        }
     }
 
     mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat);