API: restructure and document numpy.random C-API (#14604)

* API: restructure and document numpy.random C-API

* DOC: fix bad reference

* API: ship, document, and start to test numpy.random C-API examples

* API, DOC, TST: fix tests, refactor documentation to include snippets

* BUILD: move public headers to numpy/core/include/numpy/random

* TST: ignore DeprecationWarnings in setuptools and numba

* DOC: document the C-API as used from Cython

19 files changed, 130 insertions, 78 deletions

diff --git a/numpy/random/include/bitgen.h b/numpy/core/include/numpy/random/bitgen.h
index 83c2858dd..83c2858dd 100644
--- a/numpy/random/include/bitgen.h
+++ b/numpy/core/include/numpy/random/bitgen.h
diff --git a/numpy/random/include/distributions.h b/numpy/core/include/numpy/random/distributions.h
index c02ea605e..e489e69b8 100644
--- a/numpy/random/include/distributions.h
+++ b/numpy/core/include/numpy/random/distributions.h
@@ -8,7 +8,7 @@
 #include <stdint.h>
 
 #include "numpy/npy_math.h"
-#include "include/bitgen.h"
+#include "numpy/random/bitgen.h"
 
 /*
  * RAND_INT_TYPE is used to share integer generators with RandomState which
diff --git a/numpy/random/_bit_generator.pxd b/numpy/random/_bit_generator.pxd
index 30fa4a27d..bd5e47a20 100644
--- a/numpy/random/_bit_generator.pxd
+++ b/numpy/random/_bit_generator.pxd
@@ -1,7 +1,7 @@
 cimport numpy as np
 from libc.stdint cimport uint32_t, uint64_t
 
-cdef extern from "include/bitgen.h":
+cdef extern from "numpy/random/bitgen.h":
     struct bitgen:
         void *state
         uint64_t (*next_uint64)(void *st) nogil
diff --git a/numpy/random/_bounded_integers.pyx.in b/numpy/random/_bounded_integers.pyx.in
index c0068dab2..7e19471e4 100644
--- a/numpy/random/_bounded_integers.pyx.in
+++ b/numpy/random/_bounded_integers.pyx.in
@@ -8,7 +8,7 @@ __all__ = []
 
 np.import_array()
 
-cdef extern from "include/distributions.h":
+cdef extern from "numpy/random/distributions.h":
     # Generate random numbers in closed interval [off, off + rng].
     uint64_t random_bounded_uint64(bitgen_t *bitgen_state,
                                    uint64_t off, uint64_t rng,
diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx
index b6c222cc0..39d05b32b 100644
--- a/numpy/random/_generator.pyx
+++ b/numpy/random/_generator.pyx
@@ -28,7 +28,7 @@ from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
         )
 
 
-cdef extern from "include/distributions.h":
+cdef extern from "numpy/random/distributions.h":
 
     struct s_binomial_t:
         int has_binomial
diff --git a/numpy/random/examples/__init__.py b/numpy/random/examples/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/numpy/random/examples/__init__.py
diff --git a/numpy/random/examples/cython/extending.pyx b/numpy/random/examples/cython/extending.pyx
index a6a4ba4bf..2a866648d 100644
--- a/numpy/random/examples/cython/extending.pyx
+++ b/numpy/random/examples/cython/extending.pyx
@@ -8,7 +8,7 @@ import numpy as np
 cimport numpy as np
 cimport cython
 
-from numpy.random.common cimport bitgen_t
+from numpy.random._bit_generator cimport bitgen_t
 from numpy.random import PCG64
 
 np.import_array()
diff --git a/numpy/random/examples/cython/extending_distributions.pyx b/numpy/random/examples/cython/extending_distributions.pyx
index 3cefec97e..d17da45c1 100644
--- a/numpy/random/examples/cython/extending_distributions.pyx
+++ b/numpy/random/examples/cython/extending_distributions.pyx
@@ -1,21 +1,23 @@
 #!/usr/bin/env python
 #cython: language_level=3
 """
-This file shows how the distributions that are accessed through
-distributions.pxd can be used Cython code.
+This file shows how the to use a BitGenerator to create a distribution.
 """
 import numpy as np
 cimport numpy as np
 cimport cython
 from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer
-from numpy.random.common cimport *
-from numpy.random.distributions cimport random_gauss_zig
+from numpy.random._bit_generator cimport bitgen_t
 from numpy.random import PCG64
 
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def normals_zig(Py_ssize_t n):
+def uniforms(Py_ssize_t n):
+    """ Create an array of `n` uniformly distributed doubles.
+    A 'real' distribution would want to process the values into
+    some non-uniform distribution
+    """
     cdef Py_ssize_t i
     cdef bitgen_t *rng
     cdef const char *capsule_name = "BitGenerator"
@@ -23,21 +25,25 @@ def normals_zig(Py_ssize_t n):
 
     x = PCG64()
     capsule = x.capsule
+    # Optional check that the capsule if from a BitGenerator
     if not PyCapsule_IsValid(capsule, capsule_name):
         raise ValueError("Invalid pointer to anon_func_state")
+    # Cast the pointer
     rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
-    random_values = np.empty(n)
-    # Best practice is to release GIL and acquire the lock
+    random_values = np.empty(n, dtype='float64')
     with x.lock, nogil:
         for i in range(n):
-            random_values[i] = random_gauss_zig(rng)
+            # Call the function
+            random_values[i] = rng.next_double(rng.state)
     randoms = np.asarray(random_values)
-    return randoms
 
+    return randoms
+ 
+# cython example 2
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def uniforms(Py_ssize_t n):
+def uint16_uniforms(Py_ssize_t n):
     cdef Py_ssize_t i
     cdef bitgen_t *rng
     cdef const char *capsule_name = "BitGenerator"
@@ -45,15 +51,13 @@ def uniforms(Py_ssize_t n):
 
     x = PCG64()
     capsule = x.capsule
-    # Optional check that the capsule if from a BitGenerator
     if not PyCapsule_IsValid(capsule, capsule_name):
         raise ValueError("Invalid pointer to anon_func_state")
-    # Cast the pointer
     rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
-    random_values = np.empty(n)
+    random_values = np.empty(n, dtype='uint32')
+    # Best practice is to release GIL and acquire the lock
     with x.lock, nogil:
         for i in range(n):
-            # Call the function
-            random_values[i] = rng.next_double(rng.state)
+            random_values[i] = rng.next_uint32(rng.state)
     randoms = np.asarray(random_values)
     return randoms
diff --git a/numpy/random/examples/cython/setup.py b/numpy/random/examples/cython/setup.py
index 69f057ed5..315527a2d 100644
--- a/numpy/random/examples/cython/setup.py
+++ b/numpy/random/examples/cython/setup.py
@@ -9,15 +9,20 @@ import numpy as np
 from distutils.core import setup
 from Cython.Build import cythonize
 from setuptools.extension import Extension
-from os.path import join
+from os.path import join, abspath, dirname
+
+curpath = abspath(dirname(__file__))
 
 extending = Extension("extending",
-                      sources=['extending.pyx'],
-                      include_dirs=[np.get_include()])
+                      sources=[join(curpath, 'extending.pyx')],
+                      include_dirs=[
+                            np.get_include(),
+                            join(curpath, '..', '..')
+                        ],
+                      )
 distributions = Extension("extending_distributions",
-                          sources=['extending_distributions.pyx',
-                                   join('..', '..', 'src',
-                                        'distributions', 'distributions.c')],
+                          sources=[join(curpath, 'extending_distributions.pyx'),
+                                  ],
                           include_dirs=[np.get_include()])
 
 extensions = [extending, distributions]
diff --git a/numpy/random/examples/numba/extending.py b/numpy/random/examples/numba/extending.py
index d41c2d76f..0d240596b 100644
--- a/numpy/random/examples/numba/extending.py
+++ b/numpy/random/examples/numba/extending.py
@@ -1,14 +1,57 @@
-import datetime as dt
-
 import numpy as np
 import numba as nb
 
 from numpy.random import PCG64
+from timeit import timeit
+
+bit_gen = PCG64()
+next_d = bit_gen.cffi.next_double
+state_addr = bit_gen.cffi.state_address
+
+def normals(n, state):
+    out = np.empty(n)
+    for i in range((n + 1) // 2):
+        x1 = 2.0 * next_d(state) - 1.0
+        x2 = 2.0 * next_d(state) - 1.0
+        r2 = x1 * x1 + x2 * x2
+        while r2 >= 1.0 or r2 == 0.0:
+            x1 = 2.0 * next_d(state) - 1.0
+            x2 = 2.0 * next_d(state) - 1.0
+            r2 = x1 * x1 + x2 * x2
+        f = np.sqrt(-2.0 * np.log(r2) / r2)
+        out[2 * i] = f * x1
+        if 2 * i + 1 < n:
+            out[2 * i + 1] = f * x2
+    return out
+
+# Compile using Numba
+normalsj = nb.jit(normals, nopython=True)
+# Must use state address not state with numba
+n = 10000
+
+def numbacall():
+    return normalsj(n, state_addr)
+
+rg = np.random.Generator(PCG64())
 
-x = PCG64()
-f = x.ctypes.next_uint32
-s = x.ctypes.state
+def numpycall():
+    return rg.normal(size=n)
 
+# Check that the functions work
+r1 = numbacall()
+r2 = numpycall()
+assert r1.shape == (n,)
+assert r1.shape == r2.shape
+
+t1 = timeit(numbacall, number=1000)
+print('{:.2f} secs for {} PCG64 (Numba/PCG64) gaussian randoms'.format(t1, n))
+t2 = timeit(numpycall, number=1000)
+print('{:.2f} secs for {} PCG64 (NumPy/PCG64) gaussian randoms'.format(t2, n))
+
+# example 2
+
+next_u32 = bit_gen.ctypes.next_uint32
+ctypes_state = bit_gen.ctypes.state
 
 @nb.jit(nopython=True)
 def bounded_uint(lb, ub, state):
@@ -19,14 +62,14 @@ def bounded_uint(lb, ub, state):
     mask |= mask >> 8
     mask |= mask >> 16
 
-    val = f(state) & mask
+    val = next_u32(state) & mask
     while val > delta:
-        val = f(state) & mask
+        val = next_u32(state) & mask
 
     return lb + val
 
 
-print(bounded_uint(323, 2394691, s.value))
+print(bounded_uint(323, 2394691, ctypes_state.value))
 
 
 @nb.jit(nopython=True)
@@ -36,42 +79,6 @@ def bounded_uints(lb, ub, n, state):
         out[i] = bounded_uint(lb, ub, state)
 
 
-bounded_uints(323, 2394691, 10000000, s.value)
-
-g = x.cffi.next_double
-cffi_state = x.cffi.state
-state_addr = x.cffi.state_address
-
-
-def normals(n, state):
-    out = np.empty(n)
-    for i in range((n + 1) // 2):
-        x1 = 2.0 * g(state) - 1.0
-        x2 = 2.0 * g(state) - 1.0
-        r2 = x1 * x1 + x2 * x2
-        while r2 >= 1.0 or r2 == 0.0:
-            x1 = 2.0 * g(state) - 1.0
-            x2 = 2.0 * g(state) - 1.0
-            r2 = x1 * x1 + x2 * x2
-        f = np.sqrt(-2.0 * np.log(r2) / r2)
-        out[2 * i] = f * x1
-        if 2 * i + 1 < n:
-            out[2 * i + 1] = f * x2
-    return out
+bounded_uints(323, 2394691, 10000000, ctypes_state.value)
 
 
-print(normals(10, cffi_state).var())
-# Warm up
-normalsj = nb.jit(normals, nopython=True)
-normalsj(1, state_addr)
-
-start = dt.datetime.now()
-normalsj(1000000, state_addr)
-ms = 1000 * (dt.datetime.now() - start).total_seconds()
-print('1,000,000 Polar-transform (numba/PCG64) randoms in '
-      '{ms:0.1f}ms'.format(ms=ms))
-
-start = dt.datetime.now()
-np.random.standard_normal(1000000)
-ms = 1000 * (dt.datetime.now() - start).total_seconds()
-print('1,000,000 Polar-transform (NumPy) randoms in {ms:0.1f}ms'.format(ms=ms))
diff --git a/numpy/random/include/legacy-distributions.h b/numpy/random/include/legacy-distributions.h
index 6a0fc7dc4..b8ba0841c 100644
--- a/numpy/random/include/legacy-distributions.h
+++ b/numpy/random/include/legacy-distributions.h
@@ -2,7 +2,7 @@
 #define _RANDOMDGEN__DISTRIBUTIONS_LEGACY_H_
 
 
-#include "distributions.h"
+#include "numpy/random/distributions.h"
 
 typedef struct aug_bitgen {
   bitgen_t *bit_generator;
diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx
index b21607282..691a6e6e7 100644
--- a/numpy/random/mtrand.pyx
+++ b/numpy/random/mtrand.pyx
@@ -25,7 +25,7 @@ from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
             check_array_constraint, check_constraint, disc, discrete_broadcast_iii,
         )
 
-cdef extern from "include/distributions.h":
+cdef extern from "numpy/random/distributions.h":
     struct s_binomial_t:
         int has_binomial
         double psave
diff --git a/numpy/random/setup.py b/numpy/random/setup.py
index f9059d7d7..93d5144ea 100644
--- a/numpy/random/setup.py
+++ b/numpy/random/setup.py
@@ -34,6 +34,7 @@ def configuration(parent_package='', top_path=None):
 
     defs.append(('NPY_NO_DEPRECATED_API', 0))
     config.add_data_dir('tests')
+    config.add_data_dir('examples')
 
     EXTRA_LINK_ARGS = []
     # Math lib
@@ -92,6 +93,7 @@ def configuration(parent_package='', top_path=None):
                              depends=['%s.pyx' % gen, '%s.pxd' % gen,],
                              define_macros=defs,
                              )
+        config.add_data_files('{0}.pxd'.format(gen))
     other_srcs = [
         'src/distributions/logfactorial.c',
         'src/distributions/distributions.c',
@@ -110,6 +112,7 @@ def configuration(parent_package='', top_path=None):
                              depends=['%s.pyx' % gen],
                              define_macros=defs,
                              )
+    config.add_data_files('_bounded_inteters.pxd')
     config.add_extension('mtrand',
                          sources=['mtrand.c',
                                   'src/legacy/legacy-distributions.c',
diff --git a/numpy/random/src/distributions/distributions.c b/numpy/random/src/distributions/distributions.c
index ab8de8bcb..df3323408 100644
--- a/numpy/random/src/distributions/distributions.c
+++ b/numpy/random/src/distributions/distributions.c
@@ -1,4 +1,4 @@
-#include "include/distributions.h"
+#include "numpy/random/distributions.h"
 #include "ziggurat_constants.h"
 #include "logfactorial.h"
 
diff --git a/numpy/random/src/distributions/random_hypergeometric.c b/numpy/random/src/distributions/random_hypergeometric.c
index da5ea9c68..0da49bd62 100644
--- a/numpy/random/src/distributions/random_hypergeometric.c
+++ b/numpy/random/src/distributions/random_hypergeometric.c
@@ -1,4 +1,4 @@
-#include "include/distributions.h"
+#include "numpy/random/distributions.h"
 #include "logfactorial.h"
 #include <stdint.h>
 
diff --git a/numpy/random/src/distributions/random_mvhg_count.c b/numpy/random/src/distributions/random_mvhg_count.c
index 9c0cc045d..0c46ea417 100644
--- a/numpy/random/src/distributions/random_mvhg_count.c
+++ b/numpy/random/src/distributions/random_mvhg_count.c
@@ -2,7 +2,7 @@
 #include <stdlib.h>
 #include <stdbool.h>
 
-#include "include/distributions.h"
+#include "numpy/random/distributions.h"
 
 /*
  *  random_mvhg_count
diff --git a/numpy/random/src/distributions/random_mvhg_marginals.c b/numpy/random/src/distributions/random_mvhg_marginals.c
index 301a4acad..7e4c24988 100644
--- a/numpy/random/src/distributions/random_mvhg_marginals.c
+++ b/numpy/random/src/distributions/random_mvhg_marginals.c
@@ -3,7 +3,7 @@
 #include <stdbool.h>
 #include <math.h>
 
-#include "include/distributions.h"
+#include "numpy/random/distributions.h"
 #include "logfactorial.h"
 
 
diff --git a/numpy/random/tests/test_extending.py b/numpy/random/tests/test_extending.py
new file mode 100644
index 000000000..01e43b9e8
--- /dev/null
+++ b/numpy/random/tests/test_extending.py
@@ -0,0 +1,32 @@
+import os, sys
+import pytest
+import warnings
+
+try:
+    with warnings.catch_warnings(record=True) as w:
+        # numba issue gh-4733
+        warnings.filterwarnings('always', '', DeprecationWarning)
+        import numba
+    import cffi
+except ImportError:
+    numba = None
+
+def test_cython():
+    curdir = os.getcwd()
+    argv = sys.argv
+    examples = (os.path.dirname(__file__), '..', 'examples')
+    try:
+        os.chdir(os.path.join(*examples))
+        sys.argv = argv[:1] + ['build']
+        with warnings.catch_warnings(record=True) as w:
+            # setuptools issue gh-1885
+            warnings.filterwarnings('always', '', DeprecationWarning)
+            from numpy.random.examples.cython import setup
+    finally:
+        sys.argv = argv
+        os.chdir(curdir)
+
+@pytest.mark.skipif(numba is None, reason="requires numba")
+def test_numba():
+        from numpy.random.examples.numba import extending
+
diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py
index c71d03432..ba3961bef 100644
--- a/numpy/tests/test_public_api.py
+++ b/numpy/tests/test_public_api.py
@@ -334,6 +334,7 @@ SKIP_LIST = [
     "numpy.core.cversions",
     "numpy.core.generate_numpy_api",
     "numpy.distutils.msvc9compiler",
+    'numpy.random.examples',
 ]
 
 
@@ -386,7 +387,7 @@ SKIP_LIST_2 = [
     'numpy.matlib.fft',
     'numpy.matlib.random',
     'numpy.matlib.ctypeslib',
-    'numpy.matlib.ma'
+    'numpy.matlib.ma',
 ]