Merge branch 'truc_opts-28+knots' into unique_spk_mempool-28+knots

Author: luke-jr

configure.ac

@@ -927,6 +927,21 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]],
  [ AC_MSG_RESULT([no])]
 )
 
+AC_MSG_CHECKING(for compatible sysinfo call)
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/sysinfo.h>]],
+ [[
+  struct sysinfo info;
+  int rv = sysinfo(&info);
+  unsigned long test = info.freeram + info.bufferram + info.mem_unit;
+ ]])],
+ [
+  AC_MSG_RESULT(yes);
+  AC_DEFINE(HAVE_LINUX_SYSINFO, 1, [Define this symbol if you have a Linux-compatible sysinfo call])
+ ],[
+  AC_MSG_RESULT(no)
+ ]
+)
+
 dnl Check for posix_fallocate
 AC_MSG_CHECKING([for posix_fallocate])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[

src/bitcoin-cli.cpp

@@ -22,6 +22,10 @@
 #include <malloc.h>
 #endif
 
+#ifdef HAVE_LINUX_SYSINFO
+#include <sys/sysinfo.h>
+#endif
+
 #include <cstdlib>
 #include <locale>
 #include <stdexcept>
@@ -110,3 +114,39 @@ int64_t GetStartupTime()
 {
     return nStartupTime;
 }
+
+size_t g_low_memory_threshold = 10 * 1024 * 1024 /* 10 MB */;
+
+bool SystemNeedsMemoryReleased()
+{
+    if (g_low_memory_threshold <= 0) {
+        // Intentionally bypass other metrics when disabled entirely
+        return false;
+    }
+#ifdef WIN32
+    MEMORYSTATUSEX mem_status;
+    mem_status.dwLength = sizeof(mem_status);
+    if (GlobalMemoryStatusEx(&mem_status)) {
+        if (mem_status.dwMemoryLoad >= 99 ||
+            mem_status.ullAvailPhys < g_low_memory_threshold ||
+            mem_status.ullAvailVirtual < g_low_memory_threshold) {
+            LogPrintf("%s: YES: %s%% memory load; %s available physical memory; %s available virtual memory\n", __func__, int(mem_status.dwMemoryLoad), size_t(mem_status.ullAvailPhys), size_t(mem_status.ullAvailVirtual));
+            return true;
+        }
+    }
+#endif
+#ifdef HAVE_LINUX_SYSINFO
+    struct sysinfo sys_info;
+    if (!sysinfo(&sys_info)) {
+        // Explicitly 64-bit in case of 32-bit userspace on 64-bit kernel
+        const uint64_t free_ram = uint64_t(sys_info.freeram) * sys_info.mem_unit;
+        const uint64_t buffer_ram = uint64_t(sys_info.bufferram) * sys_info.mem_unit;
+        if (free_ram + buffer_ram < g_low_memory_threshold) {
+            LogPrintf("%s: YES: %s free RAM + %s buffer RAM\n", __func__, free_ram, buffer_ram);
+            return true;
+        }
+    }
+#endif
+    // NOTE: sysconf(_SC_AVPHYS_PAGES) doesn't account for caches on at least Linux, so not safe to use here
+    return false;
+}

src/common/system.cpp

@@ -23,6 +23,10 @@ std::string ShellEscape(const std::string& arg);
 void runCommand(const std::string& strCommand);
 #endif
 
+extern size_t g_low_memory_threshold;
+
+bool SystemNeedsMemoryReleased();
+
 /**
  * Return the number of cores available on the current system.
  * @note This does count virtual cores, such as those provided by HyperThreading.

src/common/system.h

@@ -492,6 +492,7 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-lowmem=<n>", strprintf("If system available memory falls below <n> MiB, flush caches (0 to disable, default: %s)", g_low_memory_threshold / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -647,7 +648,9 @@ void SetupServerArgs(ArgsManager& argsman)
                              "is of this size or less (default: %u)",
                              MAX_OP_RETURN_RELAY),
                    ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
-    argsman.AddArg("-mempoolfullrbf", strprintf("(DEPRECATED) Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", DEFAULT_MEMPOOL_FULL_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-mempoolfullrbf", strprintf("Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", (DEFAULT_MEMPOOL_RBF_POLICY == RBFPolicy::Always)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-mempoolreplacement", strprintf("Set to 0 to disable RBF entirely, \"fee,optin\" to honour RBF opt-out signal, or \"fee,-optin\" to always RBF aka full RBF (default: %s)", "fee,-optin"), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-mempooltruc", strprintf("Behaviour for transactions requesting TRUC limits: \"reject\" the transactions entirely, \"accept\" them just like any other, or \"enforce\" to impose their requested restrictions (default: %s)", "enforce"), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
     argsman.AddArg("-permitbaremultisig", strprintf("Relay transactions creating non-P2SH multisig outputs (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY,
                    OptionsCategory::NODE_RELAY);
     argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)",
@@ -1540,6 +1543,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     bool do_reindex{args.GetBoolArg("-reindex", false)};
     const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)};
 
+    if (gArgs.IsArgSet("-lowmem")) {
+        g_low_memory_threshold = gArgs.GetIntArg("-lowmem", 0 /* not used */) * 1024 * 1024;
+    }
+    if (g_low_memory_threshold > 0) {
+        LogPrintf("* Flushing caches if available system memory drops below %s MiB\n", g_low_memory_threshold / 1024 / 1024);
+    }
+
+    if (mempool_opts.rbf_policy == RBFPolicy::Always) {
+        nLocalServices = ServiceFlags(nLocalServices | NODE_REPLACE_BY_FEE);
+    }
+
     for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) {
         bilingual_str mempool_error;
         node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error);

src/init.cpp

@@ -15,14 +15,19 @@
 
 class ValidationSignals;
 
+enum class RBFPolicy { Never, OptIn, Always };
+enum class TRUCPolicy { Reject, Accept, Enforce };
+
 /** Default for -maxmempool, maximum megabytes of mempool memory usage */
 static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300};
 /** Default for -maxmempool when blocksonly is set */
 static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5};
 /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */
 static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336};
-/** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */
-static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{true};
+/** Default for -mempoolreplacement; must update docs in init.cpp manually */
+static constexpr RBFPolicy DEFAULT_MEMPOOL_RBF_POLICY{RBFPolicy::Always};
+/** Default for -mempooltruc; must update docs in init.cpp manually */
+static constexpr TRUCPolicy DEFAULT_MEMPOOL_TRUC_POLICY{TRUCPolicy::Enforce};
 /** Whether to fall back to legacy V1 serialization when writing mempool.dat */
 static constexpr bool DEFAULT_PERSIST_V1_DAT{false};
 /** Default for -acceptnonstdtxn */
@@ -55,7 +60,8 @@ struct MemPoolOptions {
     std::optional<unsigned> max_datacarrier_bytes{DEFAULT_ACCEPT_DATACARRIER ? std::optional{MAX_OP_RETURN_RELAY} : std::nullopt};
     bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG};
     bool require_standard{true};
-    bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF};
+    RBFPolicy rbf_policy{DEFAULT_MEMPOOL_RBF_POLICY};
+    TRUCPolicy truc_policy{DEFAULT_MEMPOOL_TRUC_POLICY};
     bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT};
     MemPoolLimits limits{};
 

src/kernel/mempool_options.h

@@ -12,10 +12,12 @@
 #include <consensus/amount.h>
 #include <kernel/chainparams.h>
 #include <logging.h>
+#include <node/interface_ui.h>
 #include <policy/feerate.h>
 #include <policy/policy.h>
 #include <tinyformat.h>
 #include <util/moneystr.h>
+#include <util/string.h>
 #include <util/translation.h>
 
 #include <chrono>
@@ -92,9 +94,86 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP
         return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())};
     }
 
-    mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf);
-    if (!mempool_opts.full_rbf) {
-        LogInfo("Warning: mempoolfullrbf=0 set but deprecated and will be removed in a future release\n");
+    if (argsman.IsArgSet("-mempoolreplacement") || argsman.IsArgSet("-mempoolfullrbf")) {
+        // Generally, mempoolreplacement overrides mempoolfullrbf, but the latter is used to infer intent in some cases
+        std::optional<bool> optin_flag;
+        bool fee_flag{false};
+        if (argsman.GetBoolArg("-mempoolreplacement", false)) {
+            fee_flag = true;
+        } else {
+            for (auto& opt : util::SplitString(argsman.GetArg("-mempoolreplacement", ""), ",+")) {
+                if (opt == "optin") {
+                    optin_flag = true;
+                } else if (opt == "-optin") {
+                    optin_flag = false;
+                } else if (opt == "fee") {
+                    fee_flag = true;
+                }
+            }
+        }
+        if (optin_flag.value_or(false)) {
+            // "optin" is explicitly specified
+            mempool_opts.rbf_policy = RBFPolicy::OptIn;
+        } else if (argsman.GetBoolArg("-mempoolfullrbf", false)) {
+            const bool mempoolreplacement_false{argsman.IsArgSet("-mempoolreplacement") && !(fee_flag || optin_flag.has_value())};
+            if (mempoolreplacement_false) {
+                // This is a contradiction, but override rather than error
+                InitWarning(_("False mempoolreplacement option contradicts true mempoolfullrbf; disallowing all RBF"));
+                mempool_opts.rbf_policy = RBFPolicy::Never;
+            } else {
+                mempool_opts.rbf_policy = RBFPolicy::Always;
+            }
+        } else if (!optin_flag.value_or(true)) {
+            // "-optin" is explicitly specified
+            mempool_opts.rbf_policy = fee_flag ? RBFPolicy::Always : RBFPolicy::Never;
+        } else if (fee_flag) {
+            // Just "fee" by itself
+            if (!argsman.GetBoolArg("-mempoolfullrbf", true)) {
+                mempool_opts.rbf_policy = RBFPolicy::OptIn;
+            } else {
+                // Fallback to default, unless it's been changed to Never
+                if (mempool_opts.rbf_policy == RBFPolicy::Never) {
+                    mempool_opts.rbf_policy = RBFPolicy::Always;
+                }
+            }
+        } else if (!argsman.IsArgSet("-mempoolreplacement")) {
+            // mempoolfullrbf is always explicitly false here
+            // Fallback to default, as long as it isn't Always
+            if (mempool_opts.rbf_policy == RBFPolicy::Always) {
+                mempool_opts.rbf_policy = RBFPolicy::OptIn;
+            }
+        } else {
+            // mempoolreplacement is explicitly false here
+            mempool_opts.rbf_policy = RBFPolicy::Never;
+        }
+    }
+
+    if (argsman.IsArgSet("-mempooltruc")) {
+        std::optional<bool> accept_flag, enforce_flag;
+        if (argsman.GetBoolArg("-mempooltruc", false)) {
+            enforce_flag = true;
+        }
+        for (auto& opt : util::SplitString(argsman.GetArg("-mempooltruc", ""), ",+")) {
+            if (opt == "optin" || opt == "enforce") {
+                enforce_flag = true;
+            } else if (opt == "-optin" || opt == "-enforce") {
+                enforce_flag = false;
+            } else if (opt == "accept") {
+                accept_flag = true;
+            } else if (opt == "reject" || opt == "0") {
+                accept_flag = false;
+            }
+        }
+
+        if (accept_flag && !*accept_flag) {  // reject
+            mempool_opts.truc_policy = TRUCPolicy::Reject;
+        } else if (enforce_flag && *enforce_flag) {  // enforce
+            mempool_opts.truc_policy = TRUCPolicy::Enforce;
+        } else if ((!accept_flag) && !enforce_flag) {
+            // nothing specified, leave at default
+        } else {  // accept or -enforce
+            mempool_opts.truc_policy = TRUCPolicy::Accept;
+        }
     }
 
     mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat);

src/node/mempool_args.cpp

@@ -99,6 +99,7 @@ static std::string serviceFlagToStr(size_t bit)
     case NODE_COMPACT_FILTERS: return "COMPACT_FILTERS";
     case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED";
     case NODE_P2P_V2:          return "P2P_V2";
+    case NODE_REPLACE_BY_FEE:  return "REPLACE_BY_FEE?";
     // Not using default, so we get warned when a case is missing
     }
 

src/protocol.cpp

@@ -336,6 +336,8 @@ enum ServiceFlags : uint64_t {
     // collisions and other cases where nodes may be advertising a service they
     // do not actually support. Other service bits should be allocated via the
     // BIP process.
+
+    NODE_REPLACE_BY_FEE = (1 << 26),
 };
 
 /**

src/protocol.h

@@ -33,6 +33,7 @@
 #include <validation.h>
 
 #include <any>
+#include <optional>
 #include <vector>
 
 #include <univalue.h>
@@ -650,8 +651,8 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s
 
     std::string param;
     const RESTResponseFormat rf = ParseDataFormat(param, str_uri_part);
-    if (param != "contents" && param != "info") {
-        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/mempool/<info|contents>.json");
+    if (param != "contents" && param != "info" && param != "info/with_fee_histogram") {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/mempool/<info|info/with_fee_histogram|contents>.json");
     }
 
     const CTxMemPool* mempool = GetMemPool(context, req);
@@ -685,8 +686,10 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s
                 return RESTERR(req, HTTP_BAD_REQUEST, "Verbose results cannot contain mempool sequence values. (hint: set \"verbose=false\")");
             }
             str_json = MempoolToJSON(*mempool, verbose, mempool_sequence).write() + "\n";
+        } else if (param == "info/with_fee_histogram") {
+            str_json = MempoolInfoToJSON(*mempool, MempoolInfoToJSON_const_histogram_floors).write() + "\n";
         } else {
-            str_json = MempoolInfoToJSON(*mempool).write() + "\n";
+            str_json = MempoolInfoToJSON(*mempool, std::nullopt).write() + "\n";
         }
 
         req->WriteHeader("Content-Type", "application/json");