Merge branch 'master' into npy-2.1

6 files changed, 168 insertions, 54 deletions

diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py
index 2cf799723..062c21725 100644
--- a/numpy/lib/tests/test_format.py
+++ b/numpy/lib/tests/test_format.py
@@ -412,6 +412,7 @@ record_arrays = [
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')),
     np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')),
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')),
+    np.zeros(1, dtype=[('c', ('<f8', (5,)), (2,))])
 ]
 
 
@@ -629,6 +630,61 @@ def test_pickle_disallow():
     assert_raises(ValueError, np.save, path, np.array([None], dtype=object),
                   allow_pickle=False)
 
+@pytest.mark.parametrize('dt', [
+    np.dtype(np.dtype([('a', np.int8),
+                       ('b', np.int16),
+                       ('c', np.int32),
+                      ], align=True),
+             (3,)),
+    np.dtype([('x', np.dtype({'names':['a','b'],
+                              'formats':['i1','i1'],
+                              'offsets':[0,4],
+                              'itemsize':8,
+                             },
+                    (3,)),
+               (4,),
+             )]),
+    np.dtype([('x',
+                   ('<f8', (5,)),
+                   (2,),
+               )]),
+    np.dtype([('x', np.dtype((
+        np.dtype((
+            np.dtype({'names':['a','b'],
+                      'formats':['i1','i1'],
+                      'offsets':[0,4],
+                      'itemsize':8}),
+            (3,)
+            )),
+        (4,)
+        )))
+        ]),
+    np.dtype([
+        ('a', np.dtype((
+            np.dtype((
+                np.dtype((
+                    np.dtype([
+                        ('a', int),
+                        ('b', np.dtype({'names':['a','b'],
+                                        'formats':['i1','i1'],
+                                        'offsets':[0,4],
+                                        'itemsize':8})),
+                    ]),
+                    (3,),
+                )),
+                (4,),
+            )),
+            (5,),
+        )))
+        ]),
+    ])
+
+def test_descr_to_dtype(dt):
+    dt1 = format.descr_to_dtype(dt.descr)
+    assert_equal_(dt1, dt)
+    arr1 = np.zeros(3, dt)
+    arr2 = roundtrip(arr1)
+    assert_array_equal(arr1, arr2)
 
 def test_version_2_0():
     f = BytesIO()
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index 6d32c365a..a3d4c6efb 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -696,6 +696,9 @@ class TestDiff(object):
         assert_raises(np.AxisError, diff, x, axis=3)
         assert_raises(np.AxisError, diff, x, axis=-4)
 
+        x = np.array(1.11111111111, np.float64)
+        assert_raises(ValueError, diff, x)
+
     def test_nd(self):
         x = 20 * rand(10, 20, 30)
         out1 = x[:, :, 1:] - x[:, :, :-1]
@@ -945,7 +948,7 @@ class TestGradient(object):
         assert_equal(type(out), type(x))
         # And make sure that the output and input don't have aliased mask
         # arrays
-        assert_(x.mask is not out.mask)
+        assert_(x._mask is not out._mask)
         # Also check that edge_order=2 doesn't alter the original mask
         x2 = np.ma.arange(5)
         x2[2] = np.ma.masked
diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py
index c96b01d42..afaa526af 100644
--- a/numpy/lib/tests/test_histograms.py
+++ b/numpy/lib/tests/test_histograms.py
@@ -554,15 +554,11 @@ class TestHistogramOptimBinNums(object):
             return a / (a + b)
 
         ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)]
-              for seed in range(256)]
+              for seed in range(10)]
 
         # the average difference between the two methods decreases as the dataset size increases.
-        assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5),
-                            [0.1065248,
-                             0.0968844,
-                             0.0331818,
-                             0.0178057],
-                            decimal=3)
+        avg = abs(np.mean(ll, axis=0) - 0.5)
+        assert_almost_equal(avg, [0.15, 0.09, 0.08, 0.03], decimal=2)
 
     def test_simple_range(self):
         """
diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py
index e687e2f54..2f7e97831 100644
--- a/numpy/lib/tests/test_index_tricks.py
+++ b/numpy/lib/tests/test_index_tricks.py
@@ -106,6 +106,9 @@ class TestRavelUnravelIndex(object):
                 np.ravel_multi_index(arr, (41, 7, 120, 36, 2706, 8, 6)),
                 [5627771580, 117259570957])
 
+        # test unravel_index for big indices (issue #9538)
+        assert_raises(ValueError, np.unravel_index, 1, (2**32-1, 2**31+1))
+
         # test overflow checking for too big array (issue #7546)
         dummy_arr = ([0],[0])
         half_max = np.iinfo(np.intp).max // 2
diff --git a/numpy/lib/tests/test_packbits.py b/numpy/lib/tests/test_packbits.py
index 00d5ca827..95a465c36 100644
--- a/numpy/lib/tests/test_packbits.py
+++ b/numpy/lib/tests/test_packbits.py
@@ -2,7 +2,8 @@ from __future__ import division, absolute_import, print_function
 
 import numpy as np
 from numpy.testing import assert_array_equal, assert_equal, assert_raises
-
+import pytest
+from itertools import chain
 
 def test_packbits():
     # Copied from the docstring.
@@ -50,8 +51,8 @@ def test_packbits_empty_with_axis():
                 assert_equal(b.dtype, np.uint8)
                 assert_equal(b.shape, out_shape)
 
-
-def test_packbits_large():
+@pytest.mark.parametrize('bitorder', ('little', 'big'))
+def test_packbits_large(bitorder):
     # test data large enough for 16 byte vectorization
     a = np.array([1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0,
                   0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1,
@@ -71,7 +72,7 @@ def test_packbits_large():
     a = a.repeat(3)
     for dtype in '?bBhHiIlLqQ':
         arr = np.array(a, dtype=dtype)
-        b = np.packbits(arr, axis=None)
+        b = np.packbits(arr, axis=None, bitorder=bitorder)
         assert_equal(b.dtype, np.uint8)
         r = [252, 127, 192, 3, 254, 7, 252, 0, 7, 31, 240, 0, 28, 1, 255, 252,
              113, 248, 3, 255, 192, 28, 15, 192, 28, 126, 0, 224, 127, 255,
@@ -81,9 +82,10 @@ def test_packbits_large():
              255, 224, 1, 255, 252, 126, 63, 0, 1, 192, 252, 14, 63, 0, 15,
              199, 252, 113, 255, 3, 128, 56, 252, 14, 7, 0, 113, 255, 255, 142, 56, 227,
              129, 248, 227, 129, 199, 31, 128]
-        assert_array_equal(b, r)
+        if bitorder == 'big':
+            assert_array_equal(b, r)
         # equal for size being multiple of 8
-        assert_array_equal(np.unpackbits(b)[:-4], a)
+        assert_array_equal(np.unpackbits(b, bitorder=bitorder)[:-4], a)
 
         # check last byte of different remainders (16 byte vectorization)
         b = [np.packbits(arr[:-i], axis=None)[-1] for i in range(1, 16)]
@@ -229,6 +231,20 @@ def test_unpackbits():
                                     [0, 0, 0, 0, 0, 1, 1, 1],
                                     [0, 0, 0, 1, 0, 1, 1, 1]]))
 
+def test_pack_unpack_order():
+    a = np.array([[2], [7], [23]], dtype=np.uint8)
+    b = np.unpackbits(a, axis=1)
+    assert_equal(b.dtype, np.uint8)
+    b_little = np.unpackbits(a, axis=1, bitorder='little')
+    b_big = np.unpackbits(a, axis=1, bitorder='big')
+    assert_array_equal(b, b_big)
+    assert_array_equal(a, np.packbits(b_little, axis=1, bitorder='little'))
+    assert_array_equal(b[:,::-1], b_little)
+    assert_array_equal(a, np.packbits(b_big, axis=1, bitorder='big'))
+    assert_raises(ValueError, np.unpackbits, a, bitorder='r')
+    assert_raises(TypeError, np.unpackbits, a, bitorder=10)
+
+
 
 def test_unpackbits_empty():
     a = np.empty((0,), dtype=np.uint8)
@@ -268,8 +284,7 @@ def test_unpackbits_large():
     assert_array_equal(np.packbits(np.unpackbits(d, axis=0), axis=0), d)
 
 
-def test_unpackbits_count():
-    # test complete invertibility of packbits and unpackbits with count
+class TestCount():
     x = np.array([
         [1, 0, 1, 0, 0, 1, 0],
         [0, 1, 1, 1, 0, 0, 0],
@@ -279,53 +294,85 @@ def test_unpackbits_count():
         [0, 0, 1, 1, 1, 0, 0],
         [0, 1, 0, 1, 0, 1, 0],
     ], dtype=np.uint8)
-
     padded1 = np.zeros(57, dtype=np.uint8)
     padded1[:49] = x.ravel()
+    padded1b = np.zeros(57, dtype=np.uint8)
+    padded1b[:49] = x[::-1].copy().ravel()
+    padded2 = np.zeros((9, 9), dtype=np.uint8)
+    padded2[:7, :7] = x
 
-    packed = np.packbits(x)
-    for count in range(58):
-        unpacked = np.unpackbits(packed, count=count)
+    @pytest.mark.parametrize('bitorder', ('little', 'big'))
+    @pytest.mark.parametrize('count', chain(range(58), range(-1, -57, -1)))
+    def test_roundtrip(self, bitorder, count):
+        if count < 0:
+            # one extra zero of padding
+            cutoff = count - 1
+        else:
+            cutoff = count
+        # test complete invertibility of packbits and unpackbits with count
+        packed = np.packbits(self.x, bitorder=bitorder)
+        unpacked = np.unpackbits(packed, count=count, bitorder=bitorder)
         assert_equal(unpacked.dtype, np.uint8)
-        assert_array_equal(unpacked, padded1[:count])
-    for count in range(-1, -57, -1):
-        unpacked = np.unpackbits(packed, count=count)
-        assert_equal(unpacked.dtype, np.uint8)
-        # count -1 because padded1 has 57 instead of 56 elements
-        assert_array_equal(unpacked, padded1[:count-1])
-    for kwargs in [{}, {'count': None}]:
+        assert_array_equal(unpacked, self.padded1[:cutoff])
+
+    @pytest.mark.parametrize('kwargs', [
+                    {}, {'count': None},
+                    ])
+    def test_count(self, kwargs):
+        packed = np.packbits(self.x)
         unpacked = np.unpackbits(packed, **kwargs)
         assert_equal(unpacked.dtype, np.uint8)
-        assert_array_equal(unpacked, padded1[:-1])
-    assert_raises(ValueError, np.unpackbits, packed, count=-57)
-
-    padded2 = np.zeros((9, 9), dtype=np.uint8)
-    padded2[:7, :7] = x
-
-    packed0 = np.packbits(x, axis=0)
-    packed1 = np.packbits(x, axis=1)
-    for count in range(10):
-        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
+        assert_array_equal(unpacked, self.padded1[:-1])
+
+    @pytest.mark.parametrize('bitorder', ('little', 'big'))
+    # delta==-1 when count<0 because one extra zero of padding
+    @pytest.mark.parametrize('count', chain(range(8), range(-1, -9, -1)))
+    def test_roundtrip_axis(self, bitorder, count):
+        if count < 0:
+            # one extra zero of padding
+            cutoff = count - 1
+        else:
+            cutoff = count
+        packed0 = np.packbits(self.x, axis=0, bitorder=bitorder)
+        unpacked0 = np.unpackbits(packed0, axis=0, count=count,
+                                  bitorder=bitorder)
         assert_equal(unpacked0.dtype, np.uint8)
-        assert_array_equal(unpacked0, padded2[:count, :x.shape[1]])
-        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
-        assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[1], :count])
-    for count in range(-1, -9, -1):
-        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
-        assert_equal(unpacked0.dtype, np.uint8)
-        # count -1 because one extra zero of padding
-        assert_array_equal(unpacked0, padded2[:count-1, :x.shape[1]])
-        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
+        assert_array_equal(unpacked0, self.padded2[:cutoff, :self.x.shape[1]])
+
+        packed1 = np.packbits(self.x, axis=1, bitorder=bitorder)
+        unpacked1 = np.unpackbits(packed1, axis=1, count=count,
+                                  bitorder=bitorder)
         assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[0], :count-1])
-    for kwargs in [{}, {'count': None}]:
+        assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :cutoff])
+
+    @pytest.mark.parametrize('kwargs', [
+                    {}, {'count': None},
+                    {'bitorder' : 'little'},
+                    {'bitorder': 'little', 'count': None},
+                    {'bitorder' : 'big'},
+                    {'bitorder': 'big', 'count': None},
+                    ])
+    def test_axis_count(self, kwargs):
+        packed0 = np.packbits(self.x, axis=0)
         unpacked0 = np.unpackbits(packed0, axis=0, **kwargs)
         assert_equal(unpacked0.dtype, np.uint8)
-        assert_array_equal(unpacked0, padded2[:-1, :x.shape[1]])
+        if kwargs.get('bitorder', 'big') == 'big':
+            assert_array_equal(unpacked0, self.padded2[:-1, :self.x.shape[1]])
+        else:
+            assert_array_equal(unpacked0[::-1, :], self.padded2[:-1, :self.x.shape[1]])
+
+        packed1 = np.packbits(self.x, axis=1)
         unpacked1 = np.unpackbits(packed1, axis=1, **kwargs)
         assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[0], :-1])
-    assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
-    assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)
-
+        if kwargs.get('bitorder', 'big') == 'big':
+            assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :-1])
+        else:
+            assert_array_equal(unpacked1[:, ::-1], self.padded2[:self.x.shape[0], :-1])
+
+    def test_bad_count(self):
+        packed0 = np.packbits(self.x, axis=0)
+        assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
+        packed1 = np.packbits(self.x, axis=1)
+        assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)
+        packed = np.packbits(self.x)
+        assert_raises(ValueError, np.unpackbits, packed, count=-57)
diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py
index 112678294..f713fb64d 100644
--- a/numpy/lib/tests/test_recfunctions.py
+++ b/numpy/lib/tests/test_recfunctions.py
@@ -243,6 +243,15 @@ class TestRecFunctions(object):
         assert_(dd.base is d)
         assert_(ddd.base is d)
 
+        # including uniform fields with subarrays unpacked
+        d = np.array([(1, [2,  3], [[ 4,  5], [ 6,  7]]),
+                      (8, [9, 10], [[11, 12], [13, 14]])],
+                     dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))])
+        dd = structured_to_unstructured(d)
+        ddd = unstructured_to_structured(dd, d.dtype)
+        assert_(dd.base is d)
+        assert_(ddd.base is d)
+
         # test that nested fields with identical names don't break anything
         point = np.dtype([('x', int), ('y', int)])
         triangle = np.dtype([('a', point), ('b', point), ('c', point)])