Merge bitcoin/bitcoin#28523: rpc: add hidden getrawaddrman RPC to list addrman table entries

352d5eb test: getrawaddrman RPC (0xb10c)
da384a2 rpc: getrawaddrman for addrman entries (0xb10c)

Pull request description:

  Inspired by `getaddrmaninfo` (#27511), this adds a hidden/test-only `getrawaddrman` RPC. The RPC returns information on all addresses in the address manager new and tried tables. Addrman table contents can be used in tests and during development.

  The RPC result encodes the `bucket` and `position`, the internal location of addresses in the tables, in the address object's string key. This allows users to choose to consume or to ignore the location information. If the internals of the address manager implementation change, the location encoding might change too.

  ```
  getrawaddrman

  EXPERIMENTAL warning: this call may be changed in future releases.

  Returns information on all address manager entries for the new and tried tables.

  Result:
  {                                  (json object)
    "table" : {                      (json object) buckets with addresses in the address manager table ( new, tried )
      "bucket/position" : {          (json object) the location in the address manager table (<bucket>/<position>)
        "address" : "str",           (string) The address of the node
        "port" : n,                  (numeric) The port number of the node
        "network" : "str",           (string) The network (ipv4, ipv6, onion, i2p, cjdns) of the address
        "services" : n,              (numeric) The services offered by the node
        "time" : xxx,                (numeric) The UNIX epoch time when the node was last seen
        "source" : "str",            (string) The address that relayed the address to us
        "source_network" : "str"     (string) The network (ipv4, ipv6, onion, i2p, cjdns) of the source address
      },
      ...
    },
    ...
  }

  Examples:
  > bitcoin-cli getrawaddrman
  > curl --user myusername --data-binary '{"jsonrpc": "1.0", "id": "curltest", "method": "getrawaddrman", "params": []}' -H 'content-type: text/plain;' http://127.0.0.1:8332/
  ```

ACKs for top commit:
  willcl-ark:
    reACK 352d5eb
  amitiuttarwar:
    reACK 352d5eb
  stratospher:
    reACK 352d5eb.
  achow101:
    ACK 352d5eb

Tree-SHA512: cc462666b5c709617c66b0e3e9a17c4c81e9e295f91bdd9572492d1cb6466fc9b6d48ee805ebe82f9f16010798370effe5c8f4db15065b8c7c0d8637675d615e

Author: achow101

src/addrman.cpp

@@ -838,6 +838,30 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct
     return addresses;
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrManImpl::GetEntries_(bool from_tried) const
+{
+    AssertLockHeld(cs);
+
+    const int bucket_count = from_tried ? ADDRMAN_TRIED_BUCKET_COUNT : ADDRMAN_NEW_BUCKET_COUNT;
+    std::vector<std::pair<AddrInfo, AddressPosition>> infos;
+    for (int bucket = 0; bucket < bucket_count; ++bucket) {
+        for (int position = 0; position < ADDRMAN_BUCKET_SIZE; ++position) {
+            int id = GetEntry(from_tried, bucket, position);
+            if (id >= 0) {
+                AddrInfo info = mapInfo.at(id);
+                AddressPosition location = AddressPosition(
+                    from_tried,
+                    /*multiplicity_in=*/from_tried ? 1 : info.nRefCount,
+                    bucket,
+                    position);
+                infos.push_back(std::make_pair(info, location));
+            }
+        }
+    }
+
+    return infos;
+}
+
 void AddrManImpl::Connected_(const CService& addr, NodeSeconds time)
 {
     AssertLockHeld(cs);
@@ -1199,6 +1223,15 @@ std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct,
     return addresses;
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrManImpl::GetEntries(bool from_tried) const
+{
+    LOCK(cs);
+    Check();
+    auto addrInfos = GetEntries_(from_tried);
+    Check();
+    return addrInfos;
+}
+
 void AddrManImpl::Connected(const CService& addr, NodeSeconds time)
 {
     LOCK(cs);
@@ -1289,6 +1322,11 @@ std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std
     return m_impl->GetAddr(max_addresses, max_pct, network);
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrMan::GetEntries(bool use_tried) const
+{
+    return m_impl->GetEntries(use_tried);
+}
+
 void AddrMan::Connected(const CService& addr, NodeSeconds time)
 {
     m_impl->Connected(addr, time);

src/addrman.h

@@ -25,11 +25,12 @@ class InvalidAddrManVersionError : public std::ios_base::failure
 };
 
 class AddrManImpl;
+class AddrInfo;
 
 /** Default for -checkaddrman */
 static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0};
 
-/** Test-only struct, capturing info about an address in AddrMan */
+/** Location information for an address in AddrMan */
 struct AddressPosition {
     // Whether the address is in the new or tried table
     const bool tried;
@@ -168,6 +169,17 @@ class AddrMan
      */
     std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const;
 
+    /**
+     * Returns an information-location pair for all addresses in the selected addrman table.
+     * If an address appears multiple times in the new table, an information-location pair
+     * is returned for each occurence. Addresses only ever appear once in the tried table.
+     *
+     * @param[in] from_tried     Selects which table to return entries from.
+     *
+     * @return                   A vector consisting of pairs of AddrInfo and AddressPosition.
+     */
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries(bool from_tried) const;
+
     /** We have successfully connected to this peer. Calling this function
      *  updates the CAddress's nTime, which is used in our IsTerrible()
      *  decisions and gossiped to peers. Callers should be careful that updating

src/addrman_impl.h

@@ -132,6 +132,9 @@ class AddrManImpl
     std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries(bool from_tried) const
+        EXCLUSIVE_LOCKS_REQUIRED(!cs);
+
     void Connected(const CService& addr, NodeSeconds time)
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
@@ -260,6 +263,8 @@ class AddrManImpl
 
     std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs);
 
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries_(bool from_tried) const EXCLUSIVE_LOCKS_REQUIRED(cs);
+
     void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs);
 
     void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs);

src/rpc/net.cpp

@@ -5,6 +5,7 @@
 #include <rpc/server.h>
 
 #include <addrman.h>
+#include <addrman_impl.h>
 #include <banman.h>
 #include <chainparams.h>
 #include <clientversion.h>
@@ -1079,6 +1080,74 @@ static RPCHelpMan getaddrmaninfo()
     };
 }
 
+UniValue AddrmanEntryToJSON(const AddrInfo& info)
+{
+    UniValue ret(UniValue::VOBJ);
+    ret.pushKV("address", info.ToStringAddr());
+    ret.pushKV("port", info.GetPort());
+    ret.pushKV("services", (uint64_t)info.nServices);
+    ret.pushKV("time", int64_t{TicksSinceEpoch<std::chrono::seconds>(info.nTime)});
+    ret.pushKV("network", GetNetworkName(info.GetNetClass()));
+    ret.pushKV("source", info.source.ToStringAddr());
+    ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass()));
+    return ret;
+}
+
+UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos)
+{
+    UniValue table(UniValue::VOBJ);
+    for (const auto& e : tableInfos) {
+        AddrInfo info = e.first;
+        AddressPosition location = e.second;
+        std::ostringstream key;
+        key << location.bucket << "/" << location.position;
+        // Address manager tables have unique entries so there is no advantage
+        // in using UniValue::pushKV, which checks if the key already exists
+        // in O(N). UniValue::pushKVEnd is used instead which currently is O(1).
+        table.pushKVEnd(key.str(), AddrmanEntryToJSON(info));
+    }
+    return table;
+}
+
+static RPCHelpMan getrawaddrman()
+{
+    return RPCHelpMan{"getrawaddrman",
+        "EXPERIMENTAL warning: this call may be changed in future releases.\n"
+        "\nReturns information on all address manager entries for the new and tried tables.\n",
+        {},
+        RPCResult{
+            RPCResult::Type::OBJ_DYN, "", "", {
+                {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", {
+                    {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (<bucket>/<position>)", {
+                        {RPCResult::Type::STR, "address", "The address of the node"},
+                        {RPCResult::Type::NUM, "port", "The port number of the node"},
+                        {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"},
+                        {RPCResult::Type::NUM, "services", "The services offered by the node"},
+                        {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"},
+                        {RPCResult::Type::STR, "source", "The address that relayed the address to us"},
+                        {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"},
+                    }}
+                }}
+            }
+        },
+        RPCExamples{
+            HelpExampleCli("getrawaddrman", "")
+            + HelpExampleRpc("getrawaddrman", "")
+        },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue {
+            NodeContext& node = EnsureAnyNodeContext(request.context);
+            if (!node.addrman) {
+                throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Address manager functionality missing or disabled");
+            }
+
+            UniValue ret(UniValue::VOBJ);
+            ret.pushKV("new", AddrmanTableToJSON(node.addrman->GetEntries(false)));
+            ret.pushKV("tried", AddrmanTableToJSON(node.addrman->GetEntries(true)));
+            return ret;
+        },
+    };
+}
+
 void RegisterNetRPCCommands(CRPCTable& t)
 {
     static const CRPCCommand commands[]{
@@ -1099,6 +1168,7 @@ void RegisterNetRPCCommands(CRPCTable& t)
         {"hidden", &addpeeraddress},
         {"hidden", &sendmsgtopeer},
         {"hidden", &getaddrmaninfo},
+        {"hidden", &getrawaddrman},
     };
     for (const auto& c : commands) {
         t.appendCommand(c.name, &c);

src/test/fuzz/rpc.cpp

@@ -142,6 +142,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "getnodeaddresses",
     "getpeerinfo",
     "getprioritisedtransactions",
+    "getrawaddrman",
     "getrawmempool",
     "getrawtransaction",
     "getrpcinfo",

test/functional/rpc_net.py

@@ -12,6 +12,7 @@
 import time
 
 import test_framework.messages
+from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE
 from test_framework.p2p import (
     P2PInterface,
     P2P_SERVICES,
@@ -67,6 +68,7 @@ def run_test(self):
         self.test_addpeeraddress()
         self.test_sendmsgtopeer()
         self.test_getaddrmaninfo()
+        self.test_getrawaddrman()
 
     def test_connection_count(self):
         self.log.info("Test getconnectioncount")
@@ -388,5 +390,115 @@ def test_getaddrmaninfo(self):
             assert_equal(res[net]["tried"], 0)
             assert_equal(res[net]["total"], 0)
 
+    def test_getrawaddrman(self):
+        self.log.info("Test getrawaddrman")
+        node = self.nodes[1]
+
+        self.log.debug("Test that getrawaddrman is a hidden RPC")
+        # It is hidden from general help, but its detailed help may be called directly.
+        assert "getrawaddrman" not in node.help()
+        assert "getrawaddrman" in node.help("getrawaddrman")
+
+        def check_addr_information(result, expected):
+            """Utility to compare a getrawaddrman result entry with an expected entry"""
+            assert_equal(result["address"], expected["address"])
+            assert_equal(result["port"], expected["port"])
+            assert_equal(result["services"], expected["services"])
+            assert_equal(result["network"], expected["network"])
+            assert_equal(result["source"], expected["source"])
+            assert_equal(result["source_network"], expected["source_network"])
+            # To avoid failing on slow test runners, use a 10s vspan here.
+            assert_approx(result["time"], time.time(), vspan=10)
+
+        def check_getrawaddrman_entries(expected):
+            """Utility to compare a getrawaddrman result with expected addrman contents"""
+            getrawaddrman = node.getrawaddrman()
+            getaddrmaninfo = node.getaddrmaninfo()
+            for (table_name, table_info) in expected.items():
+                assert_equal(len(getrawaddrman[table_name]), len(table_info["entries"]))
+                assert_equal(len(getrawaddrman[table_name]), getaddrmaninfo["all_networks"][table_name])
+
+                for bucket_position in getrawaddrman[table_name].keys():
+                    bucket = int(bucket_position.split("/")[0])
+                    position = int(bucket_position.split("/")[1])
+
+                    # bucket and position only be sanity checked here as the
+                    # test-addrman isn't deterministic
+                    assert 0 <= int(bucket) < table_info["bucket_count"]
+                    assert 0 <= int(position) < ADDRMAN_BUCKET_SIZE
+
+                    entry = getrawaddrman[table_name][bucket_position]
+                    expected_entry = list(filter(lambda e: e["address"] == entry["address"], table_info["entries"]))[0]
+                    check_addr_information(entry, expected_entry)
+
+        # we expect one addrman new and tried table entry, which were added in a previous test
+        expected = {
+            "new": {
+                "bucket_count": ADDRMAN_NEW_BUCKET_COUNT,
+                "entries": [
+                    {
+                        "address": "2.0.0.0",
+                        "port": 8333,
+                        "services": 9,
+                        "network": "ipv4",
+                        "source": "2.0.0.0",
+                        "source_network": "ipv4",
+                    }
+                ]
+            },
+            "tried": {
+                "bucket_count": ADDRMAN_TRIED_BUCKET_COUNT,
+                "entries": [
+                    {
+                        "address": "1.2.3.4",
+                        "port": 8333,
+                        "services": 9,
+                        "network": "ipv4",
+                        "source": "1.2.3.4",
+                        "source_network": "ipv4",
+                    }
+                ]
+            }
+        }
+
+        self.log.debug("Test that the getrawaddrman contains information about the addresses added in a previous test")
+        check_getrawaddrman_entries(expected)
+
+        self.log.debug("Add one new address to each addrman table")
+        expected["new"]["entries"].append({
+            "address": "2803:0:1234:abcd::1",
+            "services": 9,
+            "network": "ipv6",
+            "source": "2803:0:1234:abcd::1",
+            "source_network": "ipv6",
+            "port": -1,  # set once addpeeraddress is successful
+        })
+        expected["tried"]["entries"].append({
+            "address": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion",
+            "services": 9,
+            "network": "onion",
+            "source": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion",
+            "source_network": "onion",
+            "port": -1,  # set once addpeeraddress is successful
+        })
+
+        port = 0
+        for (table_name, table_info) in expected.items():
+            # There's a slight chance that the to-be-added address collides with an already
+            # present table entry. To avoid this, we increment the port until an address has been
+            # added. Incrementing the port changes the position in the new table bucket (bucket
+            # stays the same) and changes both the bucket and the position in the tried table.
+            while True:
+                if node.addpeeraddress(address=table_info["entries"][1]["address"], port=port, tried=table_name == "tried")["success"]:
+                    table_info["entries"][1]["port"] = port
+                    self.log.debug(f"Added {table_info['entries'][1]['address']} to {table_name} table")
+                    break
+                else:
+                    port += 1
+
+        self.log.debug("Test that the newly added addresses appear in getrawaddrman")
+        check_getrawaddrman_entries(expected)
+
+
 if __name__ == '__main__':
     NetTest().main()