Merge pull request dashpay#5842 from kwvg/bps_jan_03

backport: merge bitcoin#19289, bitcoin#19998, bitcoin#17775, bitcoin#19401, bitcoin#20167, bitcoin#20187, bitcoin#19961, bitcoin#21114, bitcoin#21170 (auxiliary backports: part 12)

Author: PastaPastaPasta

doc/tor.md

@@ -8,6 +8,16 @@ may not. In particular, the Tor Browser Bundle defaults to listening on port 915
 See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort)
 for how to properly configure Tor.
 
+## How to see information about your Tor configuration via Dash Core
+
+There are several ways to see your local onion address in Dash Core:
+- in the debug log (grep for "tor:" or "AddLocal")
+- in the output of RPC `getnetworkinfo` in the "localaddresses" section
+- in the output of the CLI `-netinfo` peer connections dashboard
+
+You may set the `-debug=tor` config logging option to have additional
+information in the debug log about your Tor configuration.
+
 
 ## 1. Run Dash Core behind a Tor proxy
 
@@ -160,14 +170,19 @@ The directory can be different of course, but virtual port numbers should be equ
 your dashd's P2P listen port (9999 by default), and target addresses and ports
 should be equal to binding address and port for inbound Tor connections (127.0.0.1:9996 by default).
 
-	-externalip=X   You can tell Dash Core about its publicly reachable address using
-	                this option, and this can be a .onion address. Given the above
-	                configuration, you can find your .onion address in
+	-externalip=X   You can tell Dash Core about its publicly reachable addresses using
+	                this option, and this can be an onion address. Given the above
+	                configuration, you can find your onion address in
 	                /var/lib/tor/dashcore-service/hostname. For connections
 	                coming from unroutable addresses (such as 127.0.0.1, where the
-	                Tor proxy typically runs), .onion addresses are given
+	                Tor proxy typically runs), onion addresses are given
 	                preference for your node to advertise itself with.
 
+	                You can set multiple local addresses with -externalip. The
+	                one that will be rumoured to a particular peer is the most
+	                compatible one and also using heuristics, e.g. the address
+	                with the most incoming connections, etc.
+
 	-listen         You'll need to enable listening for incoming connections, as this
 	                is off by default behind a proxy.
 
@@ -180,7 +195,7 @@ should be equal to binding address and port for inbound Tor connections (127.0.0
 
 In a typical situation, where you're only reachable via Tor, this should suffice:
 
-	./dashd -proxy=127.0.0.1:9050 -externalip=ssapp53tmftyjmjb.onion -listen
+	./dashd -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen
 
 (obviously, replace the .onion address with your own). It should be noted that you still
 listen on all devices and another node could establish a clearnet connection, when knowing
@@ -198,7 +213,7 @@ and open port 9999 on your firewall (or use port mapping, i.e., `-upnp` or `-nat
 If you only want to use Tor to reach .onion addresses, but not use it as a proxy
 for normal IPv4/IPv6 communication, use:
 
-	./dashd -onion=127.0.0.1:9050 -externalip=ssapp53tmftyjmjb.onion -discover
+	./dashd -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover
 
 
 ## 3.1. List of known Dash Core Tor relays

src/bench/rpc_blockchain.cpp

@@ -15,24 +15,49 @@
 
 #include <univalue.h>
 
-static void BlockToJsonVerbose(benchmark::Bench& bench) {
+namespace {
+
+struct TestBlockAndIndex {
     TestingSetup test_setup{};
+    CBlock block{};
+    uint256 blockHash{};
+    CBlockIndex blockindex{};
+
+    TestBlockAndIndex()
+    {
+        CDataStream stream(benchmark::data::block813851, SER_NETWORK, PROTOCOL_VERSION);
+        char a = '\0';
+        stream.write(&a, 1); // Prevent compaction
 
-    CDataStream stream(benchmark::data::block813851, SER_NETWORK, PROTOCOL_VERSION);
-    char a = '\0';
-    stream.write(&a, 1); // Prevent compaction
+        stream >> block;
 
-    CBlock block;
-    stream >> block;
+        blockHash = block.GetHash();
+        blockindex.phashBlock = &blockHash;
+        blockindex.nBits = 403014710;
+    }
+};
 
-    CBlockIndex blockindex;
-    const uint256 blockHash = block.GetHash();
-    blockindex.phashBlock = &blockHash;
-    blockindex.nBits = 403014710;
+} // namespace
 
+static void BlockToJsonVerbose(benchmark::Bench& bench)
+{
+    TestBlockAndIndex data;
     bench.run([&] {
-        (void)blockToJSON(block, &blockindex, &blockindex, *llmq::chainLocksHandler, *llmq::quorumInstantSendManager, /*verbose*/ true);
+        auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, *llmq::chainLocksHandler, *llmq::quorumInstantSendManager, /*verbose*/ true);
+        ankerl::nanobench::doNotOptimizeAway(univalue);
     });
 }
 
 BENCHMARK(BlockToJsonVerbose);
+
+static void BlockToJsonVerboseWrite(benchmark::Bench& bench)
+{
+    TestBlockAndIndex data;
+    auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, *llmq::chainLocksHandler, *llmq::quorumInstantSendManager, /*verbose*/ true);
+    bench.run([&] {
+        auto str = univalue.write();
+        ankerl::nanobench::doNotOptimizeAway(str);
+    });
+}
+
+BENCHMARK(BlockToJsonVerboseWrite);

src/core_io.h

@@ -24,7 +24,7 @@ struct CSpentIndexTxInfo;
 // core_read.cpp
 CScript ParseScript(const std::string& s);
 std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false);
-[[nodiscard]] bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx);
+[[nodiscard]] bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx);
 [[nodiscard]] bool DecodeHexBlk(CBlock&, const std::string& strHexBlk);
 bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header);
 /**

src/core_read.cpp

@@ -100,25 +100,31 @@ CScript ParseScript(const std::string& s)
     return result;
 }
 
-bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx)
+static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& tx_data)
 {
-    if (!IsHex(strHexTx))
-        return false;
-
-    std::vector<unsigned char> txData(ParseHex(strHexTx));
-    CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION);
+    CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION);
     try {
         ssData >> tx;
-        if (!ssData.empty())
+        if (!ssData.empty()) {
             return false;
-    }
-    catch (const std::exception&) {
+        }
+    } catch (const std::exception&) {
         return false;
     }
 
     return true;
 }
 
+bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx)
+{
+    if (!IsHex(hex_tx)) {
+        return false;
+    }
+
+    std::vector<unsigned char> txData(ParseHex(hex_tx));
+    return DecodeTx(tx, txData);
+}
+
 bool DecodeHexBlockHeader(CBlockHeader& header, const std::string& hex_header)
 {
     if (!IsHex(hex_header)) return false;

src/net.cpp

@@ -385,6 +385,11 @@ CNode* CConnman::FindNode(const CService& addr, bool fExcludeDisconnecting)
     return nullptr;
 }
 
+bool CConnman::AlreadyConnectedToAddress(const CAddress& addr)
+{
+    return FindNode(addr.ToStringIPPort());
+}
+
 bool CConnman::CheckIncomingNonce(uint64_t nonce)
 {
     LOCK(cs_vNodes);
@@ -2397,11 +2402,30 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect)
             if (nTries > 100)
                 break;
 
-            CAddrInfo addr = addrman.SelectTriedCollision();
+            CAddrInfo addr;
 
-            // SelectTriedCollision returns an invalid address if it is empty.
-            if (!fFeeler || !addr.IsValid()) {
-                addr = addrman.Select(fFeeler);
+            if (fFeeler) {
+                // First, try to get a tried table collision address. This returns
+                // an empty (invalid) address if there are no collisions to try.
+                addr = addrman.SelectTriedCollision();
+
+                if (!addr.IsValid()) {
+                    // No tried table collisions. Select a new table address
+                    // for our feeler.
+                    addr = addrman.Select(true);
+                } else if (AlreadyConnectedToAddress(addr)) {
+                    // If test-before-evict logic would have us connect to a
+                    // peer that we're already connected to, just mark that
+                    // address as Good(). We won't be able to initiate the
+                    // connection anyway, so this avoids inadvertently evicting
+                    // a currently-connected peer.
+                    addrman.Good(addr);
+                    // Select a new table address for our feeler instead.
+                    addr = addrman.Select(true);
+                }
+            } else {
+                // Not a feeler
+                addr = addrman.Select();
             }
 
             auto dmn = mnList.GetMNByService(addr);
@@ -2768,7 +2792,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai
 
     if (!pszDest) {
         // banned, discouraged or exact match?
-        if ((m_banman && (m_banman->IsDiscouraged(addrConnect) || m_banman->IsBanned(addrConnect))) || FindNode(addrConnect.ToStringIPPort()))
+        if ((m_banman && (m_banman->IsDiscouraged(addrConnect) || m_banman->IsBanned(addrConnect))) || AlreadyConnectedToAddress(addrConnect))
             return;
         // local and not a connection to itself?
         bool fAllowLocal = Params().AllowMultiplePorts() && addrConnect.GetPort() != GetListenPort();

src/net.h

@@ -625,6 +625,12 @@ friend class CNode;
     CNode* FindNode(const std::string& addrName, bool fExcludeDisconnecting = true);
     CNode* FindNode(const CService& addr, bool fExcludeDisconnecting = true);
 
+    /**
+     * Determine whether we're already connected to a given address, in order to
+     * avoid initiating duplicate connections.
+     */
+    bool AlreadyConnectedToAddress(const CAddress& addr);
+
     bool AttemptToEvictConnection();
     CNode* ConnectNode(CAddress addrConnect, const char *pszDest = nullptr, bool fCountFailure = false, ConnectionType conn_type = ConnectionType::OUTBOUND_FULL_RELAY);
     void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const;

src/net_processing.cpp

@@ -3017,14 +3017,8 @@ void PeerManagerImpl::ProcessMessage(
             // empty and no one will know who we are, so these mechanisms are
             // important to help us connect to the network.
             //
-            // We also update the addrman to record connection success for
-            // these peers (which include OUTBOUND_FULL_RELAY and FEELER
-            // connections) so that addrman will have an up-to-date notion of
-            // which peers are online and available.
-            //
-            // We skip these operations for BLOCK_RELAY peers to avoid
-            // potentially leaking information about our BLOCK_RELAY
-            // connections via the addrman or address relay.
+            // We skip this for BLOCK_RELAY peers to avoid potentially leaking
+            // information about our BLOCK_RELAY connections via address relay.
             if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload())
             {
                 CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices());
@@ -3043,11 +3037,24 @@ void PeerManagerImpl::ProcessMessage(
             // Get recent addresses
             m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make(NetMsgType::GETADDR));
             pfrom.fGetAddr = true;
+        }
 
-            // Moves address from New to Tried table in Addrman, resolves
-            // tried-table collisions, etc.
+        if (!pfrom.IsInboundConn()) {
+            // For non-inbound connections, we update the addrman to record
+            // connection success so that addrman will have an up-to-date
+            // notion of which peers are online and available.
+            //
+            // While we strive to not leak information about block-relay-only
+            // connections via the addrman, not moving an address to the tried
+            // table is also potentially detrimental because new-table entries
+            // are subject to eviction in the event of addrman collisions.  We
+            // mitigate the information-leak by never calling
+            // CAddrMan::Connected() on block-relay-only peers; see
+            // FinalizeNode().
+            //
+            // This moves an address from New to Tried table in Addrman,
+            // resolves tried-table collisions, etc.
             m_addrman.Good(pfrom.addr);
-
         }
 
         std::string remoteAddr;

src/rpc/blockchain.cpp

@@ -214,20 +214,9 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex
 
 UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, llmq::CChainLocksHandler& clhandler, llmq::CInstantSendManager& isman, bool txDetails)
 {
-    // Serialize passed information without accessing chain state of the active chain!
-    AssertLockNotHeld(cs_main); // For performance reasons
+    UniValue result = blockheaderToJSON(tip, blockindex, clhandler, isman);
 
-    UniValue result(UniValue::VOBJ);
-    result.pushKV("hash", blockindex->GetBlockHash().GetHex());
-    const CBlockIndex* pnext;
-    int confirmations = ComputeNextBlockAndDepth(tip, blockindex, pnext);
-    result.pushKV("confirmations", confirmations);
     result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION));
-    result.pushKV("height", blockindex->nHeight);
-    result.pushKV("version", block.nVersion);
-    result.pushKV("versionHex", strprintf("%08x", block.nVersion));
-    result.pushKV("merkleroot", block.hashMerkleRoot.GetHex());
-    bool chainLock = clhandler.HasChainLock(blockindex->nHeight, blockindex->GetBlockHash());
     UniValue txs(UniValue::VARR);
     for(const auto& tx : block.vtx)
     {
@@ -236,7 +225,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn
             UniValue objTx(UniValue::VOBJ);
             TxToUniv(*tx, uint256(), objTx, true);
             bool fLocked = isman.IsLocked(tx->GetHash());
-            objTx.pushKV("instantlock", fLocked || chainLock);
+            objTx.pushKV("instantlock", fLocked || result["chainlock"].get_bool());
             objTx.pushKV("instantlock_internal", fLocked);
             txs.push_back(objTx);
         }
@@ -249,20 +238,6 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn
             result.pushKV("cbTx", opt_cbTx->ToJson());
         }
     }
-    result.pushKV("time", block.GetBlockTime());
-    result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast());
-    result.pushKV("nonce", (uint64_t)block.nNonce);
-    result.pushKV("bits", strprintf("%08x", block.nBits));
-    result.pushKV("difficulty", GetDifficulty(blockindex));
-    result.pushKV("chainwork", blockindex->nChainWork.GetHex());
-    result.pushKV("nTx", (uint64_t)blockindex->nTx);
-
-    if (blockindex->pprev)
-        result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex());
-    if (pnext)
-        result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex());
-
-    result.pushKV("chainlock", chainLock);
 
     return result;
 }
@@ -1168,11 +1143,20 @@ static UniValue getblock(const JSONRPCRequest& request)
                             {
                                 {RPCResult::Type::STR_HEX, "hash", "the block hash (same as provided)"},
                                 {RPCResult::Type::NUM, "confirmations", "The number of confirmations, or -1 if the block is not on the main chain"},
-                                {RPCResult::Type::NUM, "size", "The block size"},
                                 {RPCResult::Type::NUM, "height", "The block height or index"},
                                 {RPCResult::Type::NUM, "version", "The block version"},
                                 {RPCResult::Type::STR_HEX, "versionHex", "The block version formatted in hexadecimal"},
                                 {RPCResult::Type::STR_HEX, "merkleroot", "The merkle root"},
+                                {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME},
+                                {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME},
+                                {RPCResult::Type::NUM, "nonce", "The nonce"},
+                                {RPCResult::Type::STR_HEX, "bits", "The bits"},
+                                {RPCResult::Type::NUM, "difficulty", "The difficulty"},
+                                {RPCResult::Type::STR_HEX, "chainwork", "Expected number of hashes required to produce the current chain"},
+                                {RPCResult::Type::NUM, "nTx", "The number of transactions in the block"},
+                                {RPCResult::Type::STR_HEX, "previousblockhash", /* optional */ true, "The hash of the previous block (if available)"},
+                                {RPCResult::Type::STR_HEX, "nextblockhash", /* optional */ true, "The hash of the next block (if available)"},
+                                {RPCResult::Type::NUM, "size", "The block size"},
                                 {RPCResult::Type::ARR, "tx", "The transaction ids",
                                     {{RPCResult::Type::STR_HEX, "", "The transaction id"}}},
                                 {RPCResult::Type::OBJ, "cbTx", "The coinbase special transaction",
@@ -1182,15 +1166,6 @@ static UniValue getblock(const JSONRPCRequest& request)
                                         {RPCResult::Type::STR_HEX, "merkleRootMNList", "The merkle root of the masternode list"},
                                         {RPCResult::Type::STR_HEX, "merkleRootQuorums", "The merkle root of the quorum list"},
                                     }},
-                                {RPCResult::Type::NUM_TIME, "time",       "The block time expressed in " + UNIX_EPOCH_TIME},
-                                {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME},
-                                {RPCResult::Type::NUM, "nonce", "The nonce"},
-                                {RPCResult::Type::STR_HEX, "bits", "The bits"},
-                                {RPCResult::Type::NUM, "difficulty", "The difficulty"},
-                                {RPCResult::Type::STR_HEX, "chainwork", "Expected number of hashes required to produce the chain up to this block (in hex)"},
-                                {RPCResult::Type::NUM, "nTx", "The number of transactions in the block"},
-                                {RPCResult::Type::STR_HEX, "previousblockhash", "The hash of the previous block"},
-                                {RPCResult::Type::STR_HEX, "nextblockhash", "The hash of the next block"},
                             }},
                     RPCResult{"for verbosity = 2",
                         RPCResult::Type::OBJ, "", "",