Merge #9792: FastRandomContext improvements and switch to ChaCha20

4fd2d2f Add a FastRandomContext::randrange and use it (Pieter Wuille)
1632922 Switch FastRandomContext to ChaCha20 (Pieter Wuille)
e04326f Add ChaCha20 (Pieter Wuille)
663fbae FastRandom benchmark (Pieter Wuille)
c21cbe6 Introduce FastRandomContext::randbool() (Pieter Wuille)

Tree-SHA512: 7fff61e3f6d6dc6ac846ca643d877b377db609646dd401a0e8f50b052c6b9bcd2f5fc34de6bbf28f04afd1724f6279ee163ead5f37d724fb782a00239f35db1d

Author: laanwj

configure.ac

@@ -549,6 +549,8 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
                  #include <byteswap.h>
                  #endif])
 
+AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll])
+
 dnl Check for MSG_NOSIGNAL
 AC_MSG_CHECKING(for MSG_NOSIGNAL)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]],

src/Makefile.am

@@ -246,6 +246,8 @@ crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 crypto_libbitcoin_crypto_a_SOURCES = \
   crypto/aes.cpp \
   crypto/aes.h \
+  crypto/chacha20.h \
+  crypto/chacha20.cpp \
   crypto/common.h \
   crypto/hmac_sha256.cpp \
   crypto/hmac_sha256.h \

src/Makefile.test.include

@@ -57,8 +57,8 @@ BITCOIN_TESTS =\
   test/policyestimator_tests.cpp \
   test/pow_tests.cpp \
   test/prevector_tests.cpp \
-  test/random_tests.cpp \
   test/raii_event_tests.cpp \
+  test/random_tests.cpp \
   test/reverselock_tests.cpp \
   test/rpc_tests.cpp \
   test/sanity_tests.cpp \

src/addrman.cpp

@@ -351,8 +351,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly)
             int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT);
             int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE);
             while (vvTried[nKBucket][nKBucketPos] == -1) {
-                nKBucket = (nKBucket + insecure_rand.rand32()) % ADDRMAN_TRIED_BUCKET_COUNT;
-                nKBucketPos = (nKBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE;
+                nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT;
+                nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE;
             }
             int nId = vvTried[nKBucket][nKBucketPos];
             assert(mapInfo.count(nId) == 1);
@@ -368,8 +368,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly)
             int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT);
             int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE);
             while (vvNew[nUBucket][nUBucketPos] == -1) {
-                nUBucket = (nUBucket + insecure_rand.rand32()) % ADDRMAN_NEW_BUCKET_COUNT;
-                nUBucketPos = (nUBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE;
+                nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT;
+                nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE;
             }
             int nId = vvNew[nUBucket][nUBucketPos];
             assert(mapInfo.count(nId) == 1);

src/addrman.h

@@ -136,13 +136,13 @@ class CAddrInfo : public CAddress
  */
 
 //! total number of buckets for tried addresses
-#define ADDRMAN_TRIED_BUCKET_COUNT 256
+#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8
 
 //! total number of buckets for new addresses
-#define ADDRMAN_NEW_BUCKET_COUNT 1024
+#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10
 
 //! maximum allowed number of entries in buckets for new and tried addresses
-#define ADDRMAN_BUCKET_SIZE 64
+#define ADDRMAN_BUCKET_SIZE_LOG2 6
 
 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread
 #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8
@@ -171,6 +171,11 @@ class CAddrInfo : public CAddress
 //! the maximum number of nodes to return in a getaddr call
 #define ADDRMAN_GETADDR_MAX 2500
 
+//! Convenience
+#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2)
+#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2)
+#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2)
+
 /** 
  * Stochastical (IP) address manager 
  */

src/bench/checkqueue.cpp

@@ -68,7 +68,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state)
         PrevectorJob(){
         }
         PrevectorJob(FastRandomContext& insecure_rand){
-            p.resize(insecure_rand.rand32() % (PREVECTOR_SIZE*2));
+            p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2));
         }
         bool operator()()
         {

src/bench/crypto_hash.cpp

@@ -7,6 +7,7 @@
 #include "bench.h"
 #include "bloom.h"
 #include "hash.h"
+#include "random.h"
 #include "uint256.h"
 #include "utiltime.h"
 #include "crypto/ripemd160.h"
@@ -69,10 +70,34 @@ static void SipHash_32b(benchmark::State& state)
     }
 }
 
+static void FastRandom_32bit(benchmark::State& state)
+{
+    FastRandomContext rng(true);
+    uint32_t x;
+    while (state.KeepRunning()) {
+        for (int i = 0; i < 1000000; i++) {
+            x += rng.rand32();
+        }
+    }
+}
+
+static void FastRandom_1bit(benchmark::State& state)
+{
+    FastRandomContext rng(true);
+    uint32_t x;
+    while (state.KeepRunning()) {
+        for (int i = 0; i < 1000000; i++) {
+            x += rng.randbool();
+        }
+    }
+}
+
 BENCHMARK(RIPEMD160);
 BENCHMARK(SHA1);
 BENCHMARK(SHA256);
 BENCHMARK(SHA512);
 
 BENCHMARK(SHA256_32b);
 BENCHMARK(SipHash_32b);
+BENCHMARK(FastRandom_32bit);
+BENCHMARK(FastRandom_1bit);

src/crypto/chacha20.cpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+// Based on the public domain implementation 'merged' by D. J. Bernstein
+// See https://cr.yp.to/chacha.html.
+
+#include "crypto/common.h"
+#include "crypto/chacha20.h"
+
+#include <string.h>
+
+constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); }
+
+#define QUARTERROUND(a,b,c,d) \
+  a += b; d = rotl32(d ^ a, 16); \
+  c += d; b = rotl32(b ^ c, 12); \
+  a += b; d = rotl32(d ^ a, 8); \
+  c += d; b = rotl32(b ^ c, 7);
+
+static const unsigned char sigma[] = "expand 32-byte k";
+static const unsigned char tau[] = "expand 16-byte k";
+
+void ChaCha20::SetKey(const unsigned char* k, size_t keylen)
+{
+    const unsigned char *constants;
+
+    input[4] = ReadLE32(k + 0);
+    input[5] = ReadLE32(k + 4);
+    input[6] = ReadLE32(k + 8);
+    input[7] = ReadLE32(k + 12);
+    if (keylen == 32) { /* recommended */
+        k += 16;
+        constants = sigma;
+    } else { /* keylen == 16 */
+        constants = tau;
+    }
+    input[8] = ReadLE32(k + 0);
+    input[9] = ReadLE32(k + 4);
+    input[10] = ReadLE32(k + 8);
+    input[11] = ReadLE32(k + 12);
+    input[0] = ReadLE32(constants + 0);
+    input[1] = ReadLE32(constants + 4);
+    input[2] = ReadLE32(constants + 8);
+    input[3] = ReadLE32(constants + 12);
+    input[12] = 0;
+    input[13] = 0;
+    input[14] = 0;
+    input[15] = 0;
+}
+
+ChaCha20::ChaCha20()
+{
+    memset(input, 0, sizeof(input));
+}
+
+ChaCha20::ChaCha20(const unsigned char* k, size_t keylen)
+{
+    SetKey(k, keylen);
+}
+
+void ChaCha20::SetIV(uint64_t iv)
+{
+    input[14] = iv;
+    input[15] = iv >> 32;
+}
+
+void ChaCha20::Seek(uint64_t pos)
+{
+    input[12] = pos;
+    input[13] = pos >> 32;
+}
+
+void ChaCha20::Output(unsigned char* c, size_t bytes)
+{
+    uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+    uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+    unsigned char *ctarget = NULL;
+    unsigned char tmp[64];
+    unsigned int i;
+
+    if (!bytes) return;
+
+    j0 = input[0];
+    j1 = input[1];
+    j2 = input[2];
+    j3 = input[3];
+    j4 = input[4];
+    j5 = input[5];
+    j6 = input[6];
+    j7 = input[7];
+    j8 = input[8];
+    j9 = input[9];
+    j10 = input[10];
+    j11 = input[11];
+    j12 = input[12];
+    j13 = input[13];
+    j14 = input[14];
+    j15 = input[15];
+
+    for (;;) {
+        if (bytes < 64) {
+            ctarget = c;
+            c = tmp;
+        }
+        x0 = j0;
+        x1 = j1;
+        x2 = j2;
+        x3 = j3;
+        x4 = j4;
+        x5 = j5;
+        x6 = j6;
+        x7 = j7;
+        x8 = j8;
+        x9 = j9;
+        x10 = j10;
+        x11 = j11;
+        x12 = j12;
+        x13 = j13;
+        x14 = j14;
+        x15 = j15;
+        for (i = 20;i > 0;i -= 2) {
+            QUARTERROUND( x0, x4, x8,x12)
+            QUARTERROUND( x1, x5, x9,x13)
+            QUARTERROUND( x2, x6,x10,x14)
+            QUARTERROUND( x3, x7,x11,x15)
+            QUARTERROUND( x0, x5,x10,x15)
+            QUARTERROUND( x1, x6,x11,x12)
+            QUARTERROUND( x2, x7, x8,x13)
+            QUARTERROUND( x3, x4, x9,x14)
+        }
+        x0 += j0;
+        x1 += j1;
+        x2 += j2;
+        x3 += j3;
+        x4 += j4;
+        x5 += j5;
+        x6 += j6;
+        x7 += j7;
+        x8 += j8;
+        x9 += j9;
+        x10 += j10;
+        x11 += j11;
+        x12 += j12;
+        x13 += j13;
+        x14 += j14;
+        x15 += j15;
+
+        ++j12;
+        if (!j12) ++j13;
+
+        WriteLE32(c + 0, x0);
+        WriteLE32(c + 4, x1);
+        WriteLE32(c + 8, x2);
+        WriteLE32(c + 12, x3);
+        WriteLE32(c + 16, x4);
+        WriteLE32(c + 20, x5);
+        WriteLE32(c + 24, x6);
+        WriteLE32(c + 28, x7);
+        WriteLE32(c + 32, x8);
+        WriteLE32(c + 36, x9);
+        WriteLE32(c + 40, x10);
+        WriteLE32(c + 44, x11);
+        WriteLE32(c + 48, x12);
+        WriteLE32(c + 52, x13);
+        WriteLE32(c + 56, x14);
+        WriteLE32(c + 60, x15);
+
+        if (bytes <= 64) {
+            if (bytes < 64) {
+                for (i = 0;i < bytes;++i) ctarget[i] = c[i];
+            }
+            input[12] = j12;
+            input[13] = j13;
+            return;
+        }
+        bytes -= 64;
+        c += 64;
+    }
+}

src/crypto/chacha20.h

@@ -0,0 +1,26 @@
+// Copyright (c) 2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_CRYPTO_CHACHA20_H
+#define BITCOIN_CRYPTO_CHACHA20_H
+
+#include <stdint.h>
+#include <stdlib.h>
+
+/** A PRNG class for ChaCha20. */
+class ChaCha20
+{
+private:
+    uint32_t input[16];
+
+public:
+    ChaCha20();
+    ChaCha20(const unsigned char* key, size_t keylen);
+    void SetKey(const unsigned char* key, size_t keylen);
+    void SetIV(uint64_t iv);
+    void Seek(uint64_t pos);
+    void Output(unsigned char* output, size_t bytes);
+};
+
+#endif // BITCOIN_CRYPTO_CHACHA20_H

src/crypto/common.h

@@ -79,4 +79,25 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x)
     memcpy(ptr, (char*)&v, 8);
 }
 
+/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */
+uint64_t static inline CountBits(uint64_t x)
+{
+#ifdef HAVE_DECL___BUILTIN_CLZL
+    if (sizeof(unsigned long) >= sizeof(uint64_t)) {
+        return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0;
+    }
+#endif
+#ifdef HAVE_DECL___BUILTIN_CLZLL
+    if (sizeof(unsigned long long) >= sizeof(uint64_t)) {
+        return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0;
+    }
+#endif
+    int ret = 0;
+    while (x) {
+        x >>= 1;
+        ++ret;
+    }
+    return ret;
+}
+
 #endif // BITCOIN_CRYPTO_COMMON_H