diff options
Diffstat (limited to 'numpy')
31 files changed, 345 insertions, 186 deletions
diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index b65920fde..9ebd12cbd 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -1454,11 +1454,10 @@ add_newdoc('numpy.core.multiarray', 'arange', Values are generated within the half-open interval ``[start, stop)`` (in other words, the interval including `start` but excluding `stop`). For integer arguments the function is equivalent to the Python built-in - `range <https://docs.python.org/library/functions.html#func-range>`_ function, - but returns an ndarray rather than a list. + `range` function, but returns an ndarray rather than a list. When using a non-integer step, such as 0.1, the results will often not - be consistent. It is better to use ``linspace`` for these cases. + be consistent. It is better to use `numpy.linspace` for these cases. Parameters ---------- @@ -2843,40 +2842,19 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', ----- Below are the public attributes of this object which were documented in "Guide to NumPy" (we have omitted undocumented public attributes, - as well as documented private attributes): - - * data: A pointer to the memory area of the array as a Python integer. - This memory area may contain data that is not aligned, or not in correct - byte-order. The memory area may not even be writeable. The array - flags and data-type of this array should be respected when passing this - attribute to arbitrary C-code to avoid trouble that can include Python - crashing. User Beware! The value of this attribute is exactly the same - as self._array_interface_['data'][0]. - - * shape (c_intp*self.ndim): A ctypes array of length self.ndim where - the basetype is the C-integer corresponding to dtype('p') on this - platform. This base-type could be c_int, c_long, or c_longlong - depending on the platform. The c_intp type is defined accordingly in - numpy.ctypeslib. The ctypes array contains the shape of the underlying - array. - - * strides (c_intp*self.ndim): A ctypes array of length self.ndim where - the basetype is the same as for the shape attribute. This ctypes array - contains the strides information from the underlying array. This strides - information is important for showing how many bytes must be jumped to - get to the next element in the array. - - * data_as(obj): Return the data pointer cast to a particular c-types object. - For example, calling self._as_parameter_ is equivalent to - self.data_as(ctypes.c_void_p). Perhaps you want to use the data as a - pointer to a ctypes array of floating-point data: - self.data_as(ctypes.POINTER(ctypes.c_double)). - - * shape_as(obj): Return the shape tuple as an array of some other c-types - type. For example: self.shape_as(ctypes.c_short). - - * strides_as(obj): Return the strides tuple as an array of some other - c-types type. For example: self.strides_as(ctypes.c_longlong). + as well as documented private attributes): + + .. autoattribute:: numpy.core._internal._ctypes.data + + .. autoattribute:: numpy.core._internal._ctypes.shape + + .. autoattribute:: numpy.core._internal._ctypes.strides + + .. automethod:: numpy.core._internal._ctypes.data_as + + .. automethod:: numpy.core._internal._ctypes.shape_as + + .. automethod:: numpy.core._internal._ctypes.strides_as Be careful using the ctypes attribute - especially on temporary arrays or arrays constructed on the fly. For example, calling @@ -7158,10 +7136,10 @@ add_newdoc('numpy.core.multiarray', 'datetime_data', array(250, dtype='timedelta64[s]') The result can be used to construct a datetime that uses the same units - as a timedelta:: + as a timedelta >>> np.datetime64('2010', np.datetime_data(dt_25s)) - numpy.datetime64('2010-01-01T00:00:00','25s') + numpy.datetime64('2010-01-01T00:00:00', '25s') """) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 1cf89aab0..48ede14d0 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -9,7 +9,7 @@ from __future__ import division, absolute_import, print_function import re import sys -from numpy.compat import basestring +from numpy.compat import basestring, unicode from .multiarray import dtype, array, ndarray try: import ctypes @@ -257,33 +257,72 @@ class _ctypes(object): self._zerod = False def data_as(self, obj): + """ + Return the data pointer cast to a particular c-types object. + For example, calling ``self._as_parameter_`` is equivalent to + ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a + pointer to a ctypes array of floating-point data: + ``self.data_as(ctypes.POINTER(ctypes.c_double))``. + """ return self._ctypes.cast(self._data, obj) def shape_as(self, obj): + """ + Return the shape tuple as an array of some other c-types + type. For example: ``self.shape_as(ctypes.c_short)``. + """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.shape) def strides_as(self, obj): + """ + Return the strides tuple as an array of some other + c-types type. For example: ``self.strides_as(ctypes.c_longlong)``. + """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.strides) def get_data(self): + """ + A pointer to the memory area of the array as a Python integer. + This memory area may contain data that is not aligned, or not in correct + byte-order. The memory area may not even be writeable. The array + flags and data-type of this array should be respected when passing this + attribute to arbitrary C-code to avoid trouble that can include Python + crashing. User Beware! The value of this attribute is exactly the same + as ``self._array_interface_['data'][0]``. + """ return self._data def get_shape(self): + """ + (c_intp*self.ndim): A ctypes array of length self.ndim where + the basetype is the C-integer corresponding to ``dtype('p')`` on this + platform. This base-type could be `ctypes.c_int`, `ctypes.c_long`, or + `ctypes.c_longlong` depending on the platform. + The c_intp type is defined accordingly in `numpy.ctypeslib`. + The ctypes array contains the shape of the underlying array. + """ return self.shape_as(_getintp_ctype()) def get_strides(self): + """ + (c_intp*self.ndim): A ctypes array of length self.ndim where + the basetype is the same as for the shape attribute. This ctypes array + contains the strides information from the underlying array. This strides + information is important for showing how many bytes must be jumped to + get to the next element in the array. + """ return self.strides_as(_getintp_ctype()) def get_as_parameter(self): return self._ctypes.c_void_p(self._data) - data = property(get_data, None, doc="c-types data") - shape = property(get_shape, None, doc="c-types shape") - strides = property(get_strides, None, doc="c-types strides") + data = property(get_data) + shape = property(get_shape) + strides = property(get_strides) _as_parameter_ = property(get_as_parameter, None, doc="_as parameter_") @@ -294,7 +333,7 @@ def _newnames(datatype, order): """ oldnames = datatype.names nameslist = list(oldnames) - if isinstance(order, str): + if isinstance(order, (str, unicode)): order = [order] seen = set() if isinstance(order, (list, tuple)): diff --git a/numpy/core/records.py b/numpy/core/records.py index 612d39322..a483871ba 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -42,7 +42,7 @@ import warnings from . import numeric as sb from . import numerictypes as nt -from numpy.compat import isfileobj, bytes, long +from numpy.compat import isfileobj, bytes, long, unicode from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype @@ -174,7 +174,7 @@ class format_parser(object): if (names): if (type(names) in [list, tuple]): pass - elif isinstance(names, str): + elif isinstance(names, (str, unicode)): names = names.split(',') else: raise NameError("illegal input names %s" % repr(names)) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 0325f3c6a..c8e3da8bc 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -175,6 +175,14 @@ _is_natively_aligned_at(PyArray_Descr *descr, return 1; } +/* + * Fill in str with an appropriate PEP 3118 format string, based on + * descr. For structured dtypes, calls itself recursively. Each call extends + * str at offset then updates offset, and uses descr->byteorder, (and + * possibly the byte order in obj) to determine the byte-order char. + * + * Returns 0 for success, -1 for failure + */ static int _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, PyObject* obj, Py_ssize_t *offset, @@ -195,8 +203,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, PyObject *item, *subarray_tuple; Py_ssize_t total_count = 1; Py_ssize_t dim_size; + Py_ssize_t old_offset; char buf[128]; - int old_offset; int ret; if (PyTuple_Check(descr->subarray->shape)) { @@ -230,15 +238,15 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, return ret; } else if (PyDataType_HASFIELDS(descr)) { - int base_offset = *offset; + Py_ssize_t base_offset = *offset; _append_str(str, "T{"); for (k = 0; k < PyTuple_GET_SIZE(descr->names); ++k) { PyObject *name, *item, *offset_obj, *tmp; PyArray_Descr *child; char *p; - Py_ssize_t len; - int new_offset; + Py_ssize_t len, new_offset; + int ret; name = PyTuple_GET_ITEM(descr->names, k); item = PyDict_GetItem(descr->fields, name); @@ -266,8 +274,11 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, } /* Insert child item */ - _buffer_format_string(child, str, obj, offset, + ret = _buffer_format_string(child, str, obj, offset, active_byteorder); + if (ret < 0) { + return -1; + } /* Insert field name */ #if defined(NPY_PY3K) @@ -393,8 +404,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, case NPY_CFLOAT: if (_append_str(str, "Zf")) return -1; break; case NPY_CDOUBLE: if (_append_str(str, "Zd")) return -1; break; case NPY_CLONGDOUBLE: if (_append_str(str, "Zg")) return -1; break; - /* XXX: datetime */ - /* XXX: timedelta */ + /* XXX NPY_DATETIME */ + /* XXX NPY_TIMEDELTA */ case NPY_OBJECT: if (_append_char(str, 'O')) return -1; break; case NPY_STRING: { char buf[128]; @@ -472,7 +483,29 @@ _buffer_info_new(PyObject *obj) goto fail; } - if (PyArray_IsScalar(obj, Generic)) { + if (PyArray_IsScalar(obj, Datetime) || PyArray_IsScalar(obj, Timedelta)) { + /* + * Special case datetime64 scalars to remain backward compatible. + * This will change in a future version. + * Note arrays of datetime64 and strutured arrays with datetime64 + * fields will not hit this code path and are currently unsupported + * in _buffer_format_string. + */ + _append_char(&fmt, 'B'); + _append_char(&fmt, '\0'); + info->ndim = 1; + info->shape = malloc(sizeof(Py_ssize_t) * 2); + if (info->shape == NULL) { + PyErr_NoMemory(); + goto fail; + } + info->strides = info->shape + info->ndim; + info->shape[0] = 8; + info->strides[0] = 1; + info->format = fmt.s; + return info; + } + else if (PyArray_IsScalar(obj, Generic)) { descr = PyArray_DescrFromScalar(obj); if (descr == NULL) { goto fail; @@ -798,8 +831,6 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) /* Fill in information */ info = _buffer_get_info(self); if (info == NULL) { - PyErr_SetString(PyExc_BufferError, - "could not get scalar buffer information"); goto fail; } @@ -822,6 +853,9 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) } #endif view->len = elsize; + if (PyArray_IsScalar(self, Datetime) || PyArray_IsScalar(self, Timedelta)) { + elsize = 1; /* descr->elsize,char is 8,'M', but we return 1,'B' */ + } view->itemsize = elsize; Py_DECREF(descr); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a2cf17f4e..f1b8a0209 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1391,10 +1391,12 @@ _array_from_buffer_3118(PyObject *memoryview) if (!is_ctypes) { /* This object has no excuse for a broken PEP3118 buffer */ - PyErr_SetString( + PyErr_Format( PyExc_RuntimeError, - "Item size computed from the PEP 3118 buffer format " - "string does not match the actual item size."); + "Item size %zd for PEP 3118 buffer format " + "string %s does not match the dtype %c item size %d.", + view->itemsize, view->format, descr->type, + descr->elsize); Py_DECREF(descr); return NULL; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 942554cae..8e058d5fb 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -620,6 +620,10 @@ class TestDateTime(object): assert_equal(pickle.loads(pickle.dumps(dt)), dt) dt = np.dtype('M8[W]') assert_equal(pickle.loads(pickle.dumps(dt)), dt) + scalar = np.datetime64('2016-01-01T00:00:00.000000000') + assert_equal(pickle.loads(pickle.dumps(scalar)), scalar) + delta = scalar - np.datetime64('2015-01-01T00:00:00.000000000') + assert_equal(pickle.loads(pickle.dumps(delta)), delta) # Check that loading pickles from 1.6 works pkl = b"cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \ @@ -1698,7 +1702,6 @@ class TestDateTime(object): assert_equal(np.busday_offset(np.datetime64('NaT'), 1, roll='preceding'), np.datetime64('NaT')) - def test_datetime_busdaycalendar(self): # Check that it removes NaT, duplicates, and weekends # and sorts the result. diff --git a/numpy/core/tests/test_einsum.py b/numpy/core/tests/test_einsum.py index 8ce374a75..6b5b9c06e 100644 --- a/numpy/core/tests/test_einsum.py +++ b/numpy/core/tests/test_einsum.py @@ -965,7 +965,6 @@ class TestEinsumPath(object): path, path_str = np.einsum_path(*edge_test4, optimize='optimal') self.assert_path_equal(path, ['einsum_path', (0, 1), (0, 1, 2, 3, 4, 5)]) - def test_path_type_input(self): # Test explicit path handeling path_test = self.build_operands('dcc,fce,ea,dbf->ab') diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py index 276cd9f93..1934d542a 100644 --- a/numpy/core/tests/test_indexing.py +++ b/numpy/core/tests/test_indexing.py @@ -194,7 +194,6 @@ class TestIndexing(object): assert_raises(IndexError, arr.__getitem__, (slice(None), index)) - def test_boolean_indexing_onedim(self): # Indexing a 2-dimensional array with # boolean array of length one diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 1511f5b6b..1c59abaa7 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -108,7 +108,6 @@ class TestFlags(object): assert_equal(self.a.flags['X'], False) assert_equal(self.a.flags['WRITEBACKIFCOPY'], False) - def test_string_align(self): a = np.zeros(4, dtype=np.dtype('|S4')) assert_(a.flags.aligned) @@ -2729,7 +2728,6 @@ class TestMethods(object): # Order of axis argument doesn't matter: assert_equal(b.diagonal(0, 2, 1), [[0, 3], [4, 7]]) - def test_diagonal_view_notwriteable(self): # this test is only for 1.9, the diagonal view will be # writeable in 1.10. @@ -4386,7 +4384,6 @@ class TestIO(object): d.tofile(f) assert_equal(os.path.getsize(self.filename), d.nbytes * 2) - def test_io_open_buffered_fromfile(self): # gh-6632 self.x.tofile(self.filename) @@ -4748,55 +4745,72 @@ class TestRecord(object): # Error raised when multiple fields have the same name assert_raises(ValueError, test_assign) - if sys.version_info[0] >= 3: - def test_bytes_fields(self): - # Bytes are not allowed in field names and not recognized in titles - # on Py3 - assert_raises(TypeError, np.dtype, [(b'a', int)]) - assert_raises(TypeError, np.dtype, [(('b', b'a'), int)]) - - dt = np.dtype([((b'a', 'b'), int)]) - assert_raises(TypeError, dt.__getitem__, b'a') - - x = np.array([(1,), (2,), (3,)], dtype=dt) - assert_raises(IndexError, x.__getitem__, b'a') - - y = x[0] - assert_raises(IndexError, y.__getitem__, b'a') - - def test_multiple_field_name_unicode(self): - def test_assign_unicode(): - dt = np.dtype([("\u20B9", "f8"), - ("B", "f8"), - ("\u20B9", "f8")]) - - # Error raised when multiple fields have the same name(unicode included) - assert_raises(ValueError, test_assign_unicode) - - else: - def test_unicode_field_titles(self): - # Unicode field titles are added to field dict on Py2 - title = u'b' - dt = np.dtype([((title, 'a'), int)]) - dt[title] - dt['a'] - x = np.array([(1,), (2,), (3,)], dtype=dt) - x[title] - x['a'] - y = x[0] - y[title] - y['a'] - - def test_unicode_field_names(self): - # Unicode field names are converted to ascii on Python 2: - encodable_name = u'b' - assert_equal(np.dtype([(encodable_name, int)]).names[0], b'b') - assert_equal(np.dtype([(('a', encodable_name), int)]).names[0], b'b') - - # But raises UnicodeEncodeError if it can't be encoded: - nonencodable_name = u'\uc3bc' - assert_raises(UnicodeEncodeError, np.dtype, [(nonencodable_name, int)]) - assert_raises(UnicodeEncodeError, np.dtype, [(('a', nonencodable_name), int)]) + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") + def test_bytes_fields(self): + # Bytes are not allowed in field names and not recognized in titles + # on Py3 + assert_raises(TypeError, np.dtype, [(b'a', int)]) + assert_raises(TypeError, np.dtype, [(('b', b'a'), int)]) + + dt = np.dtype([((b'a', 'b'), int)]) + assert_raises(TypeError, dt.__getitem__, b'a') + + x = np.array([(1,), (2,), (3,)], dtype=dt) + assert_raises(IndexError, x.__getitem__, b'a') + + y = x[0] + assert_raises(IndexError, y.__getitem__, b'a') + + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") + def test_multiple_field_name_unicode(self): + def test_assign_unicode(): + dt = np.dtype([("\u20B9", "f8"), + ("B", "f8"), + ("\u20B9", "f8")]) + + # Error raised when multiple fields have the same name(unicode included) + assert_raises(ValueError, test_assign_unicode) + + @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") + def test_unicode_field_titles(self): + # Unicode field titles are added to field dict on Py2 + title = u'b' + dt = np.dtype([((title, 'a'), int)]) + dt[title] + dt['a'] + x = np.array([(1,), (2,), (3,)], dtype=dt) + x[title] + x['a'] + y = x[0] + y[title] + y['a'] + + @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") + def test_unicode_field_names(self): + # Unicode field names are converted to ascii on Python 2: + encodable_name = u'b' + assert_equal(np.dtype([(encodable_name, int)]).names[0], b'b') + assert_equal(np.dtype([(('a', encodable_name), int)]).names[0], b'b') + + # But raises UnicodeEncodeError if it can't be encoded: + nonencodable_name = u'\uc3bc' + assert_raises(UnicodeEncodeError, np.dtype, [(nonencodable_name, int)]) + assert_raises(UnicodeEncodeError, np.dtype, [(('a', nonencodable_name), int)]) + + def test_fromarrays_unicode(self): + # A single name string provided to fromarrays() is allowed to be unicode + # on both Python 2 and 3: + x = np.core.records.fromarrays([[0], [1]], names=u'a,b', formats=u'i4,i4') + assert_equal(x['a'][0], 0) + assert_equal(x['b'][0], 1) + + def test_unicode_order(self): + # Test that we can sort with order as a unicode field name in both Python 2 and + # 3: + name = u'b' + x = np.array([1, 3, 2], dtype=[(name, int)]) + x.sort(order=name) + assert_equal(x[u'b'], np.array([1, 2, 3])) def test_field_names(self): # Test unicode and 8-bit / byte strings can be used @@ -4909,7 +4923,6 @@ class TestRecord(object): assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'), [FutureWarning]) - def test_record_hash(self): a = np.array([(1, 2), (1, 2)], dtype='i1,i2') a.flags.writeable = False @@ -6470,6 +6483,14 @@ class TestNewBufferProtocol(object): # Issue #4015. self._check_roundtrip(0) + def test_invalid_buffer_format(self): + # datetime64 cannot be used fully in a buffer yet + # Should be fixed in the next Numpy major release + dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')]) + a = np.empty(3, dt) + assert_raises((ValueError, BufferError), memoryview, a) + assert_raises((ValueError, BufferError), memoryview, np.array((3), 'M8[D]')) + def test_export_simple_1d(self): x = np.array([1, 2, 3, 4, 5], dtype='i') y = memoryview(x) diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 13bc6b34a..5e8165bc5 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2358,7 +2358,6 @@ class TestIterNested(object): j.close() assert_equal(a, [[1, 2, 3], [4, 5, 6]]) - def test_dtype_buffered(self): # Test nested iteration with buffering to change dtype diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index a1a92ef32..e7181736f 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -1275,7 +1275,6 @@ class TestArrayComparisons(object): assert_equal(a == None, [False, False, False]) assert_equal(a != None, [True, True, True]) - def test_array_equiv(self): res = np.array_equiv(np.array([1, 2]), np.array([1, 2])) assert_(res) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 36478ddb7..8244dfe20 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1828,7 +1828,6 @@ class TestRegression(object): assert_equal(oct(a), oct(0)) assert_equal(hex(a), hex(0)) - def test_object_array_self_copy(self): # An object array being copied into itself DECREF'ed before INCREF'ing # causing segmentation faults (gh-3787) diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py index 6d57a5014..cb6c521e1 100644 --- a/numpy/core/tests/test_scalarbuffer.py +++ b/numpy/core/tests/test_scalarbuffer.py @@ -5,7 +5,7 @@ import sys import numpy as np import pytest -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_, assert_equal, assert_raises # PEP3118 format strings for native (standard alignment and byteorder) types scalars_and_codes = [ @@ -77,3 +77,28 @@ class TestScalarPEP3118(object): mv_a = memoryview(a) assert_equal(mv_x.itemsize, mv_a.itemsize) assert_equal(mv_x.format, mv_a.format) + + def test_datetime_memoryview(self): + # gh-11656 + # Values verified with v1.13.3, shape is not () as in test_scalar_dim + def as_dict(m): + return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize, + ndim=m.ndim, format=m.format) + + dt1 = np.datetime64('2016-01-01') + dt2 = np.datetime64('2017-01-01') + expected = {'strides': (1,), 'itemsize': 1, 'ndim': 1, + 'shape': (8,), 'format': 'B'} + v = memoryview(dt1) + res = as_dict(v) + assert_equal(res, expected) + + v = memoryview(dt2 - dt1) + res = as_dict(v) + assert_equal(res, expected) + + dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')]) + a = np.empty(1, dt) + # Fails to create a PEP 3118 valid buffer + assert_raises((ValueError, BufferError), memoryview, a[0]) + diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 85c9a4929..d4bdb3d4e 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1173,7 +1173,6 @@ class TestBitwiseUFuncs(object): assert_(np.bitwise_xor(zeros, zeros).dtype == dt, msg) assert_(np.bitwise_and(zeros, zeros).dtype == dt, msg) - def test_identity(self): assert_(np.bitwise_or.identity == 0, 'bitwise_or') assert_(np.bitwise_xor.identity == 0, 'bitwise_xor') diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 5880ea154..d84455a8f 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -312,12 +312,12 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): If True, the input arrays are both assumed to be unique, which can speed up the calculation. Default is False. return_indices : bool - If True, the indices which correspond to the intersection of the - two arrays are returned. The first instance of a value is used - if there are multiple. Default is False. - - .. versionadded:: 1.15.0 - + If True, the indices which correspond to the intersection of the two + arrays are returned. The first instance of a value is used if there are + multiple. Default is False. + + .. versionadded:: 1.15.0 + Returns ------- intersect1d : ndarray @@ -326,7 +326,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): The indices of the first occurrences of the common values in `ar1`. Only provided if `return_indices` is True. comm2 : ndarray - The indices of the first occurrences of the common values in `ar2`. + The indices of the first occurrences of the common values in `ar2`. Only provided if `return_indices` is True. @@ -345,7 +345,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): >>> from functools import reduce >>> reduce(np.intersect1d, ([1, 3, 4, 3], [3, 1, 2, 1], [6, 3, 4, 2])) array([3]) - + To return the indices of the values common to the input arrays along with the intersected values: >>> x = np.array([1, 1, 2, 3, 4]) @@ -355,8 +355,11 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): (array([0, 2, 4]), array([1, 0, 2])) >>> xy, x[x_ind], y[y_ind] (array([1, 2, 4]), array([1, 2, 4]), array([1, 2, 4])) - + """ + ar1 = np.asanyarray(ar1) + ar2 = np.asanyarray(ar2) + if not assume_unique: if return_indices: ar1, ind1 = unique(ar1, return_index=True) @@ -367,7 +370,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): else: ar1 = ar1.ravel() ar2 = ar2.ravel() - + aux = np.concatenate((ar1, ar2)) if return_indices: aux_sort_indices = np.argsort(aux, kind='mergesort') @@ -389,6 +392,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): else: return int1d + def setxor1d(ar1, ar2, assume_unique=False): """ Find the set exclusive-or of two arrays. diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 422b356f7..f03f30fb0 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -260,6 +260,32 @@ def _get_outer_edges(a, range): return first_edge, last_edge +def _unsigned_subtract(a, b): + """ + Subtract two values where a >= b, and produce an unsigned result + + This is needed when finding the difference between the upper and lower + bound of an int16 histogram + """ + # coerce to a single type + signed_to_unsigned = { + np.byte: np.ubyte, + np.short: np.ushort, + np.intc: np.uintc, + np.int_: np.uint, + np.longlong: np.ulonglong + } + dt = np.result_type(a, b) + try: + dt = signed_to_unsigned[dt.type] + except KeyError: + return np.subtract(a, b, dtype=dt) + else: + # we know the inputs are integers, and we are deliberately casting + # signed to unsigned + return np.subtract(a, b, casting='unsafe', dtype=dt) + + def _get_bin_edges(a, bins, range, weights): """ Computes the bins used internally by `histogram`. @@ -311,7 +337,7 @@ def _get_bin_edges(a, bins, range, weights): # Do not call selectors on empty arrays width = _hist_bin_selectors[bin_name](a) if width: - n_equal_bins = int(np.ceil((last_edge - first_edge) / width)) + n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width)) else: # Width can be zero for some estimators, e.g. FD when # the IQR of the data is zero. @@ -703,7 +729,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, n = np.zeros(n_equal_bins, ntype) # Pre-compute histogram scaling factor - norm = n_equal_bins / (last_edge - first_edge) + norm = n_equal_bins / _unsigned_subtract(last_edge, first_edge) # We iterate over blocks here for two reasons: the first is that for # large arrays, it is actually faster (for example for a 10^8 array it @@ -731,7 +757,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, # Compute the bin indices, and for values that lie exactly on # last_edge we need to subtract one - f_indices = (tmp_a - first_edge) * norm + f_indices = _unsigned_subtract(tmp_a, first_edge) * norm indices = f_indices.astype(np.intp) indices[indices == n_equal_bins] -= 1 diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index abd2da1a2..8d6b0f139 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -1178,13 +1178,15 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, This optional parameter specifies the interpolation method to use when the desired quantile lies between two data points ``i < j``: - * linear: ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * lower: ``i``. - * higher: ``j``. - * nearest: ``i`` or ``j``, whichever is nearest. - * midpoint: ``(i + j) / 2``. + + * linear: ``i + (j - i) * fraction``, where ``fraction`` + is the fractional part of the index surrounded by ``i`` + and ``j``. + * lower: ``i``. + * higher: ``j``. + * nearest: ``i`` or ``j``, whichever is nearest. + * midpoint: ``(i + j) / 2``. + keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index bc5993802..ca13738c1 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -242,7 +242,7 @@ def broadcast_arrays(*args, **kwargs): subok = kwargs.pop('subok', False) if kwargs: raise TypeError('broadcast_arrays() got an unexpected keyword ' - 'argument {!r}'.format(kwargs.keys()[0])) + 'argument {!r}'.format(list(kwargs.keys())[0])) args = [np.array(_m, copy=False, subok=subok) for _m in args] shape = _broadcast_shape(*args) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 70fff3bb0..85788941c 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -33,14 +33,14 @@ def urlopen_stub(url, data=None): old_urlopen = None -def setup(): +def setup_module(): global old_urlopen old_urlopen = urllib_request.urlopen urllib_request.urlopen = urlopen_stub -def teardown(): +def teardown_module(): urllib_request.urlopen = old_urlopen # A valid website for more robust testing diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index dace5ade8..c76afb8e5 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -30,19 +30,30 @@ class TestSetOps(object): ed = np.array([1, 2, 5]) c = intersect1d(a, b) assert_array_equal(c, ed) - assert_array_equal([], intersect1d([], [])) - + + def test_intersect1d_array_like(self): + # See gh-11772 + class Test(object): + def __array__(self): + return np.arange(3) + + a = Test() + res = intersect1d(a, a) + assert_array_equal(res, a) + res = intersect1d([1, 2, 3], [1, 2, 3]) + assert_array_equal(res, [1, 2, 3]) + def test_intersect1d_indices(self): # unique inputs - a = np.array([1, 2, 3, 4]) + a = np.array([1, 2, 3, 4]) b = np.array([2, 1, 4, 6]) c, i1, i2 = intersect1d(a, b, assume_unique=True, return_indices=True) ee = np.array([1, 2, 4]) assert_array_equal(c, ee) assert_array_equal(a[i1], ee) assert_array_equal(b[i2], ee) - + # non-unique inputs a = np.array([1, 2, 2, 3, 4, 3, 2]) b = np.array([1, 8, 4, 2, 2, 3, 2, 3]) @@ -51,7 +62,7 @@ class TestSetOps(object): assert_array_equal(c, ef) assert_array_equal(a[i1], ef) assert_array_equal(b[i2], ef) - + # non1d, unique inputs a = np.array([[2, 4, 5, 6], [7, 8, 1, 15]]) b = np.array([[3, 2, 7, 6], [10, 12, 8, 9]]) @@ -61,7 +72,7 @@ class TestSetOps(object): ea = np.array([2, 6, 7, 8]) assert_array_equal(ea, a[ui1]) assert_array_equal(ea, b[ui2]) - + # non1d, not assumed to be uniqueinputs a = np.array([[2, 4, 5, 6, 6], [4, 7, 8, 7, 2]]) b = np.array([[3, 2, 7, 7], [10, 12, 8, 7]]) @@ -71,7 +82,7 @@ class TestSetOps(object): ea = np.array([2, 7, 8]) assert_array_equal(ea, a[ui1]) assert_array_equal(ea, b[ui2]) - + def test_setxor1d(self): a = np.array([5, 7, 1, 2]) b = np.array([2, 4, 3, 1, 5]) diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index f136b5c81..561f5f938 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -310,6 +310,20 @@ class TestHistogram(object): assert_equal(d_edge.dtype, dates.dtype) assert_equal(t_edge.dtype, td) + def do_signed_overflow_bounds(self, dtype): + exponent = 8 * np.dtype(dtype).itemsize - 1 + arr = np.array([-2**exponent + 4, 2**exponent - 4], dtype=dtype) + hist, e = histogram(arr, bins=2) + assert_equal(e, [-2**exponent + 4, 0, 2**exponent - 4]) + assert_equal(hist, [1, 1]) + + def test_signed_overflow_bounds(self): + self.do_signed_overflow_bounds(np.byte) + self.do_signed_overflow_bounds(np.short) + self.do_signed_overflow_bounds(np.intc) + self.do_signed_overflow_bounds(np.int_) + self.do_signed_overflow_bounds(np.longlong) + def do_precision_lower_bound(self, float_small, float_large): eps = np.finfo(float_large).eps diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 315251daa..7e9c026e4 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -113,7 +113,6 @@ class TestRavelUnravelIndex(object): assert_(x.flags.writeable) assert_(y.flags.writeable) - def test_0d(self): # gh-580 x = np.unravel_index(0, ()) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index f58c9e33d..1f3664d92 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -348,7 +348,6 @@ class TestSaveTxt(object): assert_raises(ValueError, np.savetxt, c, np.array(1)) assert_raises(ValueError, np.savetxt, c, np.array([[[1], [2]]])) - def test_record(self): a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) c = BytesIO() @@ -2025,7 +2024,6 @@ M 33 21.99 assert_equal(test['f0'], 0) assert_equal(test['f1'], "testNonethe" + utf8.decode("UTF-8")) - def test_utf8_file_nodtype_unicode(self): # bytes encoding with non-latin1 -> unicode upcast utf8 = u'\u03d6' diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index 3c2ca8b87..b2bd7da3e 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -3,7 +3,8 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.core._rational_tests import rational from numpy.testing import ( - assert_equal, assert_array_equal, assert_raises, assert_ + assert_equal, assert_array_equal, assert_raises, assert_, + assert_raises_regex ) from numpy.lib.stride_tricks import ( as_strided, broadcast_arrays, _broadcast_shape, broadcast_to @@ -57,6 +58,17 @@ def test_same(): assert_array_equal(x, bx) assert_array_equal(y, by) +def test_broadcast_kwargs(): + # ensure that a TypeError is appropriately raised when + # np.broadcast_arrays() is called with any keyword + # argument other than 'subok' + x = np.arange(10) + y = np.arange(10) + + with assert_raises_regex(TypeError, + r'broadcast_arrays\(\) got an unexpected keyword*'): + broadcast_arrays(x, y, dtype='float64') + def test_one_off(): x = np.array([[1, 2, 3]]) diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index ce4a37b09..98a77d8f5 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -959,7 +959,6 @@ class TestMatrixPower(object): #FIXME the 'e' dtype might work in future dtnoinv = [object, np.dtype('e'), np.dtype('g'), np.dtype('G')] - def test_large_power(self, dt): power = matrix_power rshft = self.rshft_1.astype(dt) @@ -1022,7 +1021,6 @@ class TestMatrixPower(object): assert_raises(TypeError, matrix_power, mat, 1.5) assert_raises(TypeError, matrix_power, mat, [1]) - def test_exceptions_non_square(self, dt): assert_raises(LinAlgError, matrix_power, np.array([1], dt), 1) assert_raises(LinAlgError, matrix_power, np.array([[1], [2]], dt), 1) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 129809b5d..a08a0d956 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -514,8 +514,6 @@ class TestMaskedArray(object): fill_value=999999)''') ) - - def test_str_repr_legacy(self): oldopts = np.get_printoptions() np.set_printoptions(legacy='1.13') @@ -788,7 +786,6 @@ class TestMaskedArray(object): control = "(0, [[--, 0.0, --], [0.0, 0.0, --]], 0.0)" assert_equal(str(t_2d0), control) - def test_flatten_structured_array(self): # Test flatten_structured_array on arrays # On ndarray @@ -3174,18 +3171,13 @@ class TestMaskedArrayMethods(object): assert_equal(test.mask, mask_first.mask) # Test sort on dtype with subarray (gh-8069) + # Just check that the sort does not error, structured array subarrays + # are treated as byte strings and that leads to differing behavior + # depending on endianess and `endwith`. dt = np.dtype([('v', int, 2)]) a = a.view(dt) - mask_last = mask_last.view(dt) - mask_first = mask_first.view(dt) - test = sort(a) - assert_equal(test, mask_last) - assert_equal(test.mask, mask_last.mask) - test = sort(a, endwith=False) - assert_equal(test, mask_first) - assert_equal(test.mask, mask_first.mask) def test_argsort(self): # Test argsort diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 2e0885024..8328c69c0 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -1454,7 +1454,6 @@ class TestBroadcast(object): assert_raises(ValueError, zipf, np.nan) assert_raises(ValueError, zipf, [0, 0, np.nan]) - def test_geometric(self): p = [0.5] bad_p_one = [-1] diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index c2cf58377..1728d9d1f 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -338,12 +338,14 @@ class NoseTester(object): Identifies the tests to run. This can be a string to pass to the nosetests executable with the '-A' option, or one of several special values. Special values are: + * 'fast' - the default - which corresponds to the ``nosetests -A`` option of 'not slow'. * 'full' - fast (as above) and slow tests as in the 'no -A' option to nosetests - this is the same as ''. * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. + * attribute_identifier - string passed directly to nosetests as '-A'. + verbose : int, optional Verbosity value for test outputs, in the range 1-10. Default is 1. extra_argv : list, optional @@ -352,16 +354,14 @@ class NoseTester(object): If True, run doctests in module. Default is False. coverage : bool, optional If True, report coverage of NumPy code. Default is False. - (This requires the `coverage module: - <http://nedbatchelder.com/code/modules/coverage.html>`_). + (This requires the + `coverage module <https://nedbatchelder.com/code/modules/coveragehtml>`_). raise_warnings : None, str or sequence of warnings, optional This specifies which warnings to configure as 'raise' instead - of being shown once during the test execution. Valid strings are: - - - "develop" : equals ``(Warning,)`` - - "release" : equals ``()``, don't raise on any warnings. + of being shown once during the test execution. Valid strings are: - The default is to use the class initialization value. + * "develop" : equals ``(Warning,)`` + * "release" : equals ``()``, do not raise on any warnings. timer : bool or int, optional Timing of individual tests with ``nose-timer`` (which needs to be installed). If True, time tests and report on all of them. @@ -489,12 +489,14 @@ class NoseTester(object): Identifies the benchmarks to run. This can be a string to pass to the nosetests executable with the '-A' option, or one of several special values. Special values are: + * 'fast' - the default - which corresponds to the ``nosetests -A`` option of 'not slow'. * 'full' - fast (as above) and slow benchmarks as in the 'no -A' option to nosetests - this is the same as ''. * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. + * attribute_identifier - string passed directly to nosetests as '-A'. + verbose : int, optional Verbosity value for benchmark outputs, in the range 1-10. Default is 1. extra_argv : list, optional diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 0e2f8ba91..a3832fcde 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -687,6 +687,8 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, isnan, inf, bool_ + from numpy.core.fromnumeric import all as npall + x = array(x, copy=False, subok=True) y = array(y, copy=False, subok=True) @@ -697,14 +699,21 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, return x.dtype.char in "Mm" def func_assert_same_pos(x, y, func=isnan, hasval='nan'): - """Handling nan/inf: combine results of running func on x and y, - checking that they are True at the same locations.""" - # Both the != True comparison here and the cast to bool_ at - # the end are done to deal with `masked`, which cannot be - # compared usefully, and for which .all() yields masked. + """Handling nan/inf. + + Combine results of running func on x and y, checking that they are True + at the same locations. + + """ + # Both the != True comparison here and the cast to bool_ at the end are + # done to deal with `masked`, which cannot be compared usefully, and + # for which np.all yields masked. The use of the function np.all is + # for back compatibility with ndarray subclasses that changed the + # return values of the all method. We are not committed to supporting + # such subclasses, but some used to work. x_id = func(x) y_id = func(y) - if (x_id == y_id).all() != True: + if npall(x_id == y_id) != True: msg = build_err_msg([x, y], err_msg + '\nx and y %s location mismatch:' % (hasval), verbose=verbose, header=header, diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index ea684140d..d00820b80 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -53,7 +53,6 @@ class TestNoseDecorators(object): assert_(f_istest.__test__) assert_(not f_isnottest.__test__) - def test_skip_functions_hardcoded(self): @dec.skipif(True) def f1(x): diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index 84d310992..2c60e2867 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -1391,7 +1391,6 @@ class TestAssertNoGcCycles(object): assert_no_gc_cycles(no_cycle) - def test_asserts(self): def make_cycle(): a = [] @@ -1406,7 +1405,6 @@ class TestAssertNoGcCycles(object): with assert_raises(AssertionError): assert_no_gc_cycles(make_cycle) - def test_fails(self): """ Test that in cases where the garbage cannot be collected, we raise an |