ENH: Add direct access to system crypto RNG

Add direct access to the system crypto generator
Add docstrings for xorshift and pcg RNGs

Author: Sheppard, Kevin

README.md

@@ -39,7 +39,6 @@ It is essentially complete.  There are a few rough edges that need to be smoothe
   * Document core RNG classes
   * Pickling support
   * Verify entropy based initialization is missing for some RNGs
-  * Add direct access to the system cryptographic random number generator
   * Multiple stream support for MLFG and MRG32K3A
   * Creation of additional streams from a RandomState where supported (i.e. 
   a `next_stream()` method)

randomstate/interface.pyx

@@ -1,26 +1,31 @@
 #!python
 #cython: wraparound=False, nonecheck=False, boundscheck=False, cdivision=True
 import sys
-import numpy as np
-cimport numpy as np
-cimport cython
 import operator
-from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t
-from cpython cimport Py_INCREF
 try:
     from threading import Lock
 except:
     from dummy_threading import Lock
+
+import numpy as np
+cimport numpy as np
+cimport cython
+from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t
+from cpython cimport Py_INCREF
+
+from binomial cimport binomial_t
+from cython_overrides cimport PyFloat_AsDouble, PyInt_AsLong, PyErr_Occurred, PyErr_Clear
+
 np.import_array()
 
-cdef extern from "Python.h":
-    double PyFloat_AsDouble(object ob)
-    long PyInt_AsLong(object ob)
-    int PyErr_Occurred()
-    void PyErr_Clear()
+#cdef extern from "Python.h":
+#    double PyFloat_AsDouble(object ob)
+#    long PyInt_AsLong(object ob)
+#    int PyErr_Occurred()
+#    void PyErr_Clear()
 
 include "config.pxi"
-include "src/common/binomial.pxi"
+#include "src/common/binomial.pxi"
 
 IF RNG_NAME == 'pcg32':
     include "shims/pcg-32/pcg-32.pxi"
@@ -108,9 +113,55 @@ cdef extern from "distributions.h":
     cdef void random_bounded_uint8_fill(aug_state *state, uint8_t off, uint8_t rng, int cnt, uint8_t *out) nogil
     cdef void random_bool_fill(aug_state *state, int8_t off, int8_t rng, int cnt, int8_t *out) nogil
 
+    cdef void entropy_fill(void *dest, size_t size)
+    cdef bint entropy_getbytes(void* dest, size_t size)
+
 include "array_utilities.pxi"
 include "bounded_integers.pxi"
 
+def read_entropy(size=None):
+    """
+    Read entropy from the system cryptographic provider
+
+    Parameters
+    ----------
+    size : int or tuple of ints, optional
+        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+        ``m * n * k`` samples are drawn.  Default is None, in which case a
+        single value is returned.
+
+    Returns
+    -------
+    entropy : scalar or ndarray
+        Entropy bits in 32-bit unsigned integers
+
+    Notes
+    -----
+    On Unix-like machines, reads from /dev/urandom. On Windows machines reads
+    from the RSA Full cryptographic service provider.
+
+    Raises RuntimeError if the command fails.
+    """
+    cdef bint success
+    cdef size_t n = 0
+    cdef uint32_t random = 0
+    cdef uint32_t [:] randoms
+
+    if size is None:
+        success = entropy_getbytes(<void *>&random, 4)
+    else:
+        n = compute_numel(size)
+        randoms = np.zeros(n, dtype=np.uint32)
+        print(n)
+        success = entropy_getbytes(<void *>(&randoms[0]), 4 * n)
+    if not success:
+        raise RuntimeError('Unable to read from system cryptographic provider')
+
+    if n == 0:
+        return random
+    return np.asarray(randoms).reshape(size)
+
+
 cdef double kahan_sum(double *darr, np.npy_intp n):
     cdef double c, y, t, sum
     cdef np.npy_intp i

randomstate/shims/pcg-32/pcg-32.pxi

@@ -39,5 +39,55 @@ cdef object _set_state(aug_state state, object state_info):
     state.rng.inc = state_info[1]
 
 DEF CLASS_DOCSTRING = """
-This is the pcg32 docstring.
-"""
+RandomState(seed=None)
+
+Container for the PCG-32 pseudo random number generator.
+
+PCG-32 is a 64-bit implementation of O'Neill's permutation congruential 
+generator ([1]_, [2]_). PCG-32 has a period of 2**64 and supports advancing 
+an arbitrary number of steps as well as 2**63 streams.
+
+`pcg32.RandomState` exposes a number of methods for generating random
+numbers drawn from a variety of probability distributions. In addition to the
+distribution-specific arguments, each method takes a keyword argument
+`size` that defaults to ``None``. If `size` is ``None``, then a single
+value is generated and returned. If `size` is an integer, then a 1-D
+array filled with generated values is returned. If `size` is a tuple,
+then an array with that shape is filled and returned.
+
+*No Compatibility Guarantee*
+'pcg32.RandomState' does not make a guarantee that a fixed seed and a
+fixed series of calls to 'pcg32.RandomState' methods using the same
+parameters will always produce the same results. This is different from
+'numpy.random.RandomState' guarantee. This is done to simplify improving
+random number generators.  To ensure identical results, you must use the
+same release version.
+
+Parameters
+----------
+seed : {None, long}, optional
+    Random seed initializing the pseudo-random number generator.
+    Can be an integer in [0, 2**64] or ``None`` (the default).
+    If `seed` is ``None``, then `xorshift1024.RandomState` will try to read data
+    from ``/dev/urandom`` (or the Windows analogue) if available or seed from
+    the clock otherwise.  
+inc : {None, int}, optional
+    Stream to return.  
+    Can be an integer in [0, 2**64] or ``None`` (the default).  If `inc` is 
+    ``None``, then 1 is used.  Can be used with the same seed to 
+    produce multiple streams using other values of inc.
+    
+Notes
+-----
+Supports the method advance to advance the PRNG an arbitrary number of steps. 
+The state of the PCG-32 PRNG is represented by 2 64-bit integers.
+
+See pcg64 for a similar implementation with a larger period.
+
+References
+----------
+.. [1] "PCG, A Family of Better Random Number Generators",
+       http://www.pcg-random.org/
+.. [2] O'Neill, Melissa E. "PCG: A Family of Simple Fast Space-Efficient 
+       Statistically Good Algorithms for Random Number Generation"
+"""

randomstate/shims/pcg-64/pcg-64-docstring.pxi

@@ -0,0 +1,53 @@
+DEF CLASS_DOCSTRING = """
+RandomState(seed=None)
+
+Container for the PCG-64 pseudo random number generator.
+
+PCG-64 is a 128-bit implementation of O'Neill's permutation congruential
+generator ([1]_, [2]_). PCG-64 has a period of 2**128 and supports advancing
+an arbitrary number of steps as well as 2**127 streams.
+
+`pcg64.RandomState` exposes a number of methods for generating random
+numbers drawn from a variety of probability distributions. In addition to the
+distribution-specific arguments, each method takes a keyword argument
+`size` that defaults to ``None``. If `size` is ``None``, then a single
+value is generated and returned. If `size` is an integer, then a 1-D
+array filled with generated values is returned. If `size` is a tuple,
+then an array with that shape is filled and returned.
+
+*No Compatibility Guarantee*
+'pcg64.RandomState' does not make a guarantee that a fixed seed and a
+fixed series of calls to 'pcg64.RandomState' methods using the same
+parameters will always produce the same results. This is different from
+'numpy.random.RandomState' guarantee. This is done to simplify improving
+random number generators.  To ensure identical results, you must use the
+same release version.
+
+Parameters
+----------
+seed : {None, long}, optional
+    Random seed initializing the pseudo-random number generator.
+    Can be an integer in [0, 2**128] or ``None`` (the default).
+    If `seed` is ``None``, then `xorshift1024.RandomState` will try to read data
+    from ``/dev/urandom`` (or the Windows analogue) if available or seed from
+    the clock otherwise.
+inc : {None, int}, optional
+    Stream to return.
+    Can be an integer in [0, 2**128] or ``None`` (the default).  If `inc` is
+    ``None``, then 1 is used.  Can be used with the same seed to
+    produce multiple streams using other values of inc.
+
+Notes
+-----
+Supports the method advance to advance the PRNG an arbitrary number of steps.
+The state of the PCG-64 PRNG is represented by 2 128-bit integers.
+
+See pcg32 for a similar implementation with a smaller period.
+
+References
+----------
+.. [1] "PCG, A Family of Better Random Number Generators",
+       http://www.pcg-random.org/
+.. [2] O'Neill, Melissa E. "PCG: A Family of Simple Fast Space-Efficient
+       Statistically Good Algorithms for Random Number Generation"
+"""

randomstate/shims/pcg-64/pcg-64-emulated.pxi

@@ -55,6 +55,4 @@ cdef object _set_state(aug_state state, object state_info):
     state.rng.state = pcg128_from_pylong(state_info[0])
     state.rng.inc = pcg128_from_pylong(state_info[1])
 
-DEF CLASS_DOCSTRING = """
-This is the pcg64 docstring.
-"""
+include "pcg-64-docstring.pxi"

randomstate/shims/pcg-64/pcg-64.pxi

@@ -43,6 +43,4 @@ cdef object _set_state(aug_state state, object state_info):
     state.rng.state = state_info[0]
     state.rng.inc = state_info[1]
 
-DEF CLASS_DOCSTRING = """
-This is the pcg64 docstring.
-"""
+include "pcg-64-docstring.pxi"

randomstate/shims/xorshift1024/xorshift1024.pxi

@@ -47,6 +47,48 @@ cdef object _set_state(aug_state state, object state_info):
         state.rng.s[i] = key[i]
     state.rng.p = state_info[1]
 
+
 DEF CLASS_DOCSTRING = """
-This is the xorshift1024 docstring.
+RandomState(seed=None)
+
+Container for the xorshift1024* pseudo random number generator.
+
+xorshift1024* is a 64-bit implementation of Saito and Matsumoto's XSadd
+generator [1]_. xorshift1024* has a period of 2**1024 - 1 and supports jumping
+the sequence in increments of 2**512, which allow multiple non-overlapping
+sequences to be generated.
+
+`xorshift1024.RandomState` exposes a number of methods for generating random
+numbers drawn from a variety of probability distributions. In addition to the
+distribution-specific arguments, each method takes a keyword argument
+`size` that defaults to ``None``. If `size` is ``None``, then a single
+value is generated and returned. If `size` is an integer, then a 1-D
+array filled with generated values is returned. If `size` is a tuple,
+then an array with that shape is filled and returned.
+
+*No Compatibility Guarantee*
+'xorshift1024.RandomState' does not make a guarantee that a fixed seed and a
+fixed series of calls to 'xorshift1024.RandomState' methods using the same
+parameters will always produce the same results. This is different from
+'numpy.random.RandomState' guarantee. This is done to simplify improving
+random number generators.  To ensure identical results, you must use the
+same release version.
+
+Parameters
+----------
+seed : {None, int}, optional
+    Random seed initializing the pseudo-random number generator.
+    Can be an integer or ``None`` (the default).
+    If `seed` is ``None``, then `xorshift1024.RandomState` will try to read data
+    from ``/dev/urandom`` (or the Windows analogue) if available or seed from
+    the clock otherwise.
+
+Notes
+-----
+See xorshift128 for a faster implementation that has a smaller period.
+
+References
+----------
+.. [1] "xorshift*/xorshift+ generators and the PRNG shootout",
+       http://xorshift.di.unimi.it/
 """

randomstate/shims/xorshift128/xorshift128-defs.pxi

@@ -49,8 +49,8 @@ RandomState(seed=None)
 Container for the xorshift128+ pseudo random number generator.
 
 xorshift128+ is a 64-bit implementation of Saito and Matsumoto's XSadd
-generator. xorshift128+ has a period of 2**128 _ 1 and supports jumping the
-sequence in increments of 2**64, which allow multiple non-overlapping
+generator [1]_. xorshift128+ has a period of 2**128 - 1 and supports jumping
+the sequence in increments of 2**64, which allow multiple non-overlapping
 sequences to be generated.
 
 `xorshift128.RandomState` exposes a number of methods for generating random
@@ -80,5 +80,11 @@ seed : {None, int}, optional
 
 Notes
 -----
-See xorshift1024 for an implementation with a larger period and jump size.
+See xorshift1024 for an implementation with a larger period
+(2**1024 - 1) and jump size.
+
+References
+----------
+.. [1] "xorshift*/xorshift+ generators and the PRNG shootout",
+       http://xorshift.di.unimi.it/
 """