Merge #8173: Use SipHash for node eviction (cont'd)

eebc232 test: Add more test vectors for siphash (Wladimir J. van der Laan)
8884830 Use C++11 thread-safe static initializers (Pieter Wuille)
c31b24f Use 64-bit SipHash of netgroups in eviction (Pieter Wuille)
9bf156b Support SipHash with arbitrary byte writes (Pieter Wuille)
053930f Avoid recalculating vchKeyedNetGroup in eviction logic. (Patrick Strateman)

Author: sipa

src/coins.cpp

@@ -56,11 +56,7 @@ void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; }
 bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); }
 CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
 
-SaltedTxidHasher::SaltedTxidHasher()
-{
-    GetRandBytes((unsigned char*)&k0, sizeof(k0));
-    GetRandBytes((unsigned char*)&k1, sizeof(k1));
-}
+SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {}
 
 CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { }
 

src/coins.h

@@ -269,7 +269,7 @@ class SaltedTxidHasher
 {
 private:
     /** Salt */
-    uint64_t k0, k1;
+    const uint64_t k0, k1;
 
 public:
     SaltedTxidHasher();

src/hash.cpp

@@ -100,12 +100,15 @@ CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
     v[2] = 0x6c7967656e657261ULL ^ k0;
     v[3] = 0x7465646279746573ULL ^ k1;
     count = 0;
+    tmp = 0;
 }
 
 CSipHasher& CSipHasher::Write(uint64_t data)
 {
     uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
 
+    assert(count % 8 == 0);
+
     v3 ^= data;
     SIPROUND;
     SIPROUND;
@@ -116,18 +119,48 @@ CSipHasher& CSipHasher::Write(uint64_t data)
     v[2] = v2;
     v[3] = v3;
 
-    count++;
+    count += 8;
+    return *this;
+}
+
+CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+    uint64_t t = tmp;
+    int c = count;
+
+    while (size--) {
+        t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
+        c++;
+        if ((c & 7) == 0) {
+            v3 ^= t;
+            SIPROUND;
+            SIPROUND;
+            v0 ^= t;
+            t = 0;
+        }
+    }
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+    count = c;
+    tmp = t;
+
     return *this;
 }
 
 uint64_t CSipHasher::Finalize() const
 {
     uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
 
-    v3 ^= ((uint64_t)count) << 59;
+    uint64_t t = tmp | (((uint64_t)count) << 56);
+
+    v3 ^= t;
     SIPROUND;
     SIPROUND;
-    v0 ^= ((uint64_t)count) << 59;
+    v0 ^= t;
     v2 ^= 0xFF;
     SIPROUND;
     SIPROUND;

src/hash.h

@@ -171,19 +171,38 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
 
 void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]);
 
-/** SipHash-2-4, using a uint64_t-based (rather than byte-based) interface */
+/** SipHash-2-4 */
 class CSipHasher
 {
 private:
     uint64_t v[4];
+    uint64_t tmp;
     int count;
 
 public:
+    /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
     CSipHasher(uint64_t k0, uint64_t k1);
+    /** Hash a 64-bit integer worth of data
+     *  It is treated as if this was the little-endian interpretation of 8 bytes.
+     *  This function can only be used when a multiple of 8 bytes have been written so far.
+     */
     CSipHasher& Write(uint64_t data);
+    /** Hash arbitrary bytes. */
+    CSipHasher& Write(const unsigned char* data, size_t size);
+    /** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
     uint64_t Finalize() const;
 };
 
+/** Optimized SipHash-2-4 implementation for uint256.
+ *
+ *  It is identical to:
+ *    SipHasher(k0, k1)
+ *      .Write(val.GetUint64(0))
+ *      .Write(val.GetUint64(1))
+ *      .Write(val.GetUint64(2))
+ *      .Write(val.GetUint64(3))
+ *      .Finalize()
+ */
 uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
 
 #endif // BITCOIN_HASH_H

src/main.cpp

@@ -4783,11 +4783,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                     LOCK(cs_vNodes);
                     // Use deterministic randomness to send to the same nodes for 24 hours
                     // at a time so the addrKnowns of the chosen nodes prevent repeats
-                    static uint64_t salt0 = 0, salt1 = 0;
-                    while (salt0 == 0 && salt1 == 0) {
-                        GetRandBytes((unsigned char*)&salt0, sizeof(salt0));
-                        GetRandBytes((unsigned char*)&salt1, sizeof(salt1));
-                    }
+                    static const uint64_t salt0 = GetRand(std::numeric_limits<uint64_t>::max());
+                    static const uint64_t salt1 = GetRand(std::numeric_limits<uint64_t>::max());
                     uint64_t hashAddr = addr.GetHash();
                     multimap<uint64_t, CNode*> mapMix;
                     const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60));

src/net.cpp

@@ -14,6 +14,7 @@
 #include "clientversion.h"
 #include "consensus/consensus.h"
 #include "crypto/common.h"
+#include "crypto/sha256.h"
 #include "hash.h"
 #include "primitives/transaction.h"
 #include "scheduler.h"
@@ -838,6 +839,7 @@ struct NodeEvictionCandidate
     int64_t nTimeConnected;
     int64_t nMinPingUsecTime;
     CAddress addr;
+    uint64_t nKeyedNetGroup;
 };
 
 static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b)
@@ -850,36 +852,8 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons
     return a.nTimeConnected > b.nTimeConnected;
 }
 
-class CompareNetGroupKeyed
-{
-    std::vector<unsigned char> vchSecretKey;
-public:
-    CompareNetGroupKeyed()
-    {
-        vchSecretKey.resize(32, 0);
-        GetRandBytes(vchSecretKey.data(), vchSecretKey.size());
-    }
-
-    bool operator()(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b)
-    {
-        std::vector<unsigned char> vchGroupA, vchGroupB;
-        CSHA256 hashA, hashB;
-        std::vector<unsigned char> vchA(32), vchB(32);
-
-        vchGroupA = a.addr.GetGroup();
-        vchGroupB = b.addr.GetGroup();
-
-        hashA.Write(begin_ptr(vchGroupA), vchGroupA.size());
-        hashB.Write(begin_ptr(vchGroupB), vchGroupB.size());
-
-        hashA.Write(begin_ptr(vchSecretKey), vchSecretKey.size());
-        hashB.Write(begin_ptr(vchSecretKey), vchSecretKey.size());
-
-        hashA.Finalize(begin_ptr(vchA));
-        hashB.Finalize(begin_ptr(vchB));
-
-        return vchA < vchB;
-    }
+static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) {
+    return a.nKeyedNetGroup < b.nKeyedNetGroup;
 };
 
 /** Try to find a connection to evict when the node is full.
@@ -902,7 +876,7 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
                 continue;
             if (node->fDisconnect)
                 continue;
-            NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr};
+            NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->nKeyedNetGroup};
             vEvictionCandidates.push_back(candidate);
         }
     }
@@ -912,9 +886,8 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
     // Protect connections with certain characteristics
 
     // Deterministically select 4 peers to protect by netgroup.
-    // An attacker cannot predict which netgroups will be protected.
-    static CompareNetGroupKeyed comparerNetGroupKeyed;
-    std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), comparerNetGroupKeyed);
+    // An attacker cannot predict which netgroups will be protected
+    std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed);
     vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end());
 
     if (vEvictionCandidates.empty()) return false;
@@ -935,24 +908,24 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
 
     // Identify the network group with the most connections and youngest member.
     // (vEvictionCandidates is already sorted by reverse connect time)
-    std::vector<unsigned char> naMostConnections;
+    uint64_t naMostConnections;
     unsigned int nMostConnections = 0;
     int64_t nMostConnectionsTime = 0;
-    std::map<std::vector<unsigned char>, std::vector<NodeEvictionCandidate> > mapAddrCounts;
+    std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapAddrCounts;
     BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) {
-        mapAddrCounts[node.addr.GetGroup()].push_back(node);
-        int64_t grouptime = mapAddrCounts[node.addr.GetGroup()][0].nTimeConnected;
-        size_t groupsize = mapAddrCounts[node.addr.GetGroup()].size();
+        mapAddrCounts[node.nKeyedNetGroup].push_back(node);
+        int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected;
+        size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size();
 
         if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) {
             nMostConnections = groupsize;
             nMostConnectionsTime = grouptime;
-            naMostConnections = node.addr.GetGroup();
+            naMostConnections = node.nKeyedNetGroup;
         }
     }
 
     // Reduce to the network group with the most connections
-    vEvictionCandidates = mapAddrCounts[naMostConnections];
+    vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]);
 
     // Do not disconnect peers if there is only one unprotected connection from their network group.
     // This step excessively favors netgroup diversity, and should be removed once more protective criteria are established.
@@ -2346,6 +2319,8 @@ unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", DEFAULT_MAX
 
 CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn) :
     ssSend(SER_NETWORK, INIT_PROTO_VERSION),
+    addr(addrIn),
+    nKeyedNetGroup(CalculateKeyedNetGroup(addrIn)),
     addrKnown(5000, 0.001),
     filterInventoryKnown(50000, 0.000001)
 {
@@ -2358,7 +2333,6 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa
     nRecvBytes = 0;
     nTimeConnected = GetTime();
     nTimeOffset = 0;
-    addr = addrIn;
     addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn;
     nVersion = 0;
     strSubVer = "";
@@ -2625,3 +2599,13 @@ bool CBanDB::Read(banmap_t& banSet)
 int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) {
     return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5);
 }
+
+/* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad)
+{
+    static const uint64_t k0 = GetRand(std::numeric_limits<uint64_t>::max());
+    static const uint64_t k1 = GetRand(std::numeric_limits<uint64_t>::max());
+
+    std::vector<unsigned char> vchNetGroup(ad.GetGroup());
+
+    return CSipHasher(k0, k1).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize();
+}

src/net.h

@@ -335,7 +335,7 @@ class CNode
     int64_t nLastRecv;
     int64_t nTimeConnected;
     int64_t nTimeOffset;
-    CAddress addr;
+    const CAddress addr;
     std::string addrName;
     CService addrLocal;
     int nVersion;
@@ -362,6 +362,8 @@ class CNode
     CBloomFilter* pfilter;
     int nRefCount;
     NodeId id;
+
+    const uint64_t nKeyedNetGroup;
 protected:
 
     // Denial-of-service detection/prevention
@@ -450,6 +452,8 @@ class CNode
     CNode(const CNode&);
     void operator=(const CNode&);
 
+    static uint64_t CalculateKeyedNetGroup(const CAddress& ad);
+
 public:
 
     NodeId GetId() const {

src/test/hash_tests.cpp

@@ -47,24 +47,81 @@ BOOST_AUTO_TEST_CASE(murmurhash3)
 #undef T
 }
 
+/*
+   SipHash-2-4 output with
+   k = 00 01 02 ...
+   and
+   in = (empty string)
+   in = 00 (1 byte)
+   in = 00 01 (2 bytes)
+   in = 00 01 02 (3 bytes)
+   ...
+   in = 00 01 02 ... 3e (63 bytes)
+
+   from: https://131002.net/siphash/siphash24.c
+*/
+uint64_t siphash_4_2_testvec[] = {
+    0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d,
+    0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137,
+    0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7,
+    0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5,
+    0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd,
+    0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8,
+    0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad,
+    0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342,
+    0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae,
+    0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c,
+    0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95,
+    0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb,
+    0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a,
+    0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499,
+    0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93,
+    0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572
+};
+
 BOOST_AUTO_TEST_CASE(siphash)
 {
     CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0x726fdb47dd0e0e31ull);
-    hasher.Write(0x0706050403020100ULL);
+    static const unsigned char t0[1] = {0};
+    hasher.Write(t0, 1);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x74f839c593dc67fdull);
+    static const unsigned char t1[7] = {1,2,3,4,5,6,7};
+    hasher.Write(t1, 7);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0x93f5f5799a932462ull);
     hasher.Write(0x0F0E0D0C0B0A0908ULL);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0x3f2acc7f57c29bdbull);
-    hasher.Write(0x1716151413121110ULL);
-    BOOST_CHECK_EQUAL(hasher.Finalize(),  0xb8ad50c6f649af94ull);
-    hasher.Write(0x1F1E1D1C1B1A1918ULL);
+    static const unsigned char t2[2] = {16,17};
+    hasher.Write(t2, 2);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x4bc1b3f0968dd39cull);
+    static const unsigned char t3[9] = {18,19,20,21,22,23,24,25,26};
+    hasher.Write(t3, 9);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x2f2e6163076bcfadull);
+    static const unsigned char t4[5] = {27,28,29,30,31};
+    hasher.Write(t4, 5);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0x7127512f72f27cceull);
     hasher.Write(0x2726252423222120ULL);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0x0e3ea96b5304a7d0ull);
     hasher.Write(0x2F2E2D2C2B2A2928ULL);
     BOOST_CHECK_EQUAL(hasher.Finalize(),  0xe612a3cb9ecba951ull);
 
     BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull);
+
+    // Check test vectors from spec, one byte at a time
+    CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
+    for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); ++x)
+    {
+        BOOST_CHECK_EQUAL(hasher2.Finalize(), siphash_4_2_testvec[x]);
+        hasher2.Write(&x, 1);
+    }
+    // Check test vectors from spec, eight bytes at a time
+    CSipHasher hasher3(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
+    for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); x+=8)
+    {
+        BOOST_CHECK_EQUAL(hasher3.Finalize(), siphash_4_2_testvec[x]);
+        hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)|
+                     (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56));
+    }
 }
 
 BOOST_AUTO_TEST_SUITE_END()