diff options
Diffstat (limited to 'numpy')
40 files changed, 482 insertions, 377 deletions
diff --git a/numpy/conftest.py b/numpy/conftest.py index 7834dd39d..18d5d1ce9 100644 --- a/numpy/conftest.py +++ b/numpy/conftest.py @@ -17,6 +17,8 @@ def pytest_configure(config): config.addinivalue_line("markers", "valgrind_error: Tests that are known to error under valgrind.") config.addinivalue_line("markers", + "leaks_references: Tests that are known to leak references.") + config.addinivalue_line("markers", "slow: Tests that are very slow.") diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 84339ef23..bb0c77471 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -10,6 +10,8 @@ NOTE: Many of the methods of ndarray have corresponding functions. """ from __future__ import division, absolute_import, print_function +import sys + from numpy.core import numerictypes as _numerictypes from numpy.core import dtype from numpy.core.function_base import add_newdoc @@ -92,7 +94,7 @@ add_newdoc('numpy.core', 'flatiter', ('coords', >>> fl = x.flat >>> fl.coords (0, 0) - >>> fl.next() + >>> next(fl) 0 >>> fl.coords (0, 1) @@ -111,7 +113,7 @@ add_newdoc('numpy.core', 'flatiter', ('index', >>> fl = x.flat >>> fl.index 0 - >>> fl.next() + >>> next(fl) 0 >>> fl.index 1 @@ -664,7 +666,7 @@ add_newdoc('numpy.core', 'broadcast', ('iters', >>> y = np.array([[4], [5], [6]]) >>> b = np.broadcast(x, y) >>> row, col = b.iters - >>> row.next(), col.next() + >>> next(row), next(col) (1, 4) """)) @@ -1471,57 +1473,58 @@ add_newdoc('numpy.core.multiarray', 'promote_types', """) -add_newdoc('numpy.core.multiarray', 'newbuffer', - """ - newbuffer(size) +if sys.version_info.major < 3: + add_newdoc('numpy.core.multiarray', 'newbuffer', + """ + newbuffer(size) - Return a new uninitialized buffer object. + Return a new uninitialized buffer object. - Parameters - ---------- - size : int - Size in bytes of returned buffer object. + Parameters + ---------- + size : int + Size in bytes of returned buffer object. - Returns - ------- - newbuffer : buffer object - Returned, uninitialized buffer object of `size` bytes. + Returns + ------- + newbuffer : buffer object + Returned, uninitialized buffer object of `size` bytes. - """) + """) -add_newdoc('numpy.core.multiarray', 'getbuffer', - """ - getbuffer(obj [,offset[, size]]) + add_newdoc('numpy.core.multiarray', 'getbuffer', + """ + getbuffer(obj [,offset[, size]]) - Create a buffer object from the given object referencing a slice of - length size starting at offset. + Create a buffer object from the given object referencing a slice of + length size starting at offset. - Default is the entire buffer. A read-write buffer is attempted followed - by a read-only buffer. + Default is the entire buffer. A read-write buffer is attempted followed + by a read-only buffer. - Parameters - ---------- - obj : object + Parameters + ---------- + obj : object - offset : int, optional + offset : int, optional - size : int, optional + size : int, optional - Returns - ------- - buffer_obj : buffer + Returns + ------- + buffer_obj : buffer - Examples - -------- - >>> buf = np.getbuffer(np.ones(5), 1, 3) - >>> len(buf) - 3 - >>> buf[0] - '\\x00' - >>> buf - <read-write buffer for 0x8af1e70, size 3, offset 1 at 0x8ba4ec0> + Examples + -------- + >>> buf = np.getbuffer(np.ones(5), 1, 3) + >>> len(buf) + 3 + >>> buf[0] + '\\x00' + >>> buf + <read-write buffer for 0x8af1e70, size 3, offset 1 at 0x8ba4ec0> - """) + """) add_newdoc('numpy.core.multiarray', 'c_einsum', """ @@ -1987,13 +1990,6 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('__array_struct__', """Array protocol: C-struct side.""")) -add_newdoc('numpy.core.multiarray', 'ndarray', ('_as_parameter_', - """Allow the array to be interpreted as a ctypes object by returning the - data-memory location as an integer - - """)) - - add_newdoc('numpy.core.multiarray', 'ndarray', ('base', """ Base object if memory is from some other object. @@ -3255,87 +3251,6 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('min', """)) -add_newdoc('numpy.core.multiarray', 'shares_memory', - """ - shares_memory(a, b, max_work=None) - - Determine if two arrays share memory - - Parameters - ---------- - a, b : ndarray - Input arrays - max_work : int, optional - Effort to spend on solving the overlap problem (maximum number - of candidate solutions to consider). The following special - values are recognized: - - max_work=MAY_SHARE_EXACT (default) - The problem is solved exactly. In this case, the function returns - True only if there is an element shared between the arrays. - max_work=MAY_SHARE_BOUNDS - Only the memory bounds of a and b are checked. - - Raises - ------ - numpy.TooHardError - Exceeded max_work. - - Returns - ------- - out : bool - - See Also - -------- - may_share_memory - - Examples - -------- - >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) - False - - """) - - -add_newdoc('numpy.core.multiarray', 'may_share_memory', - """ - may_share_memory(a, b, max_work=None) - - Determine if two arrays might share memory - - A return of True does not necessarily mean that the two arrays - share any element. It just means that they *might*. - - Only the memory bounds of a and b are checked by default. - - Parameters - ---------- - a, b : ndarray - Input arrays - max_work : int, optional - Effort to spend on solving the overlap problem. See - `shares_memory` for details. Default for ``may_share_memory`` - is to do a bounds check. - - Returns - ------- - out : bool - - See Also - -------- - shares_memory - - Examples - -------- - >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) - False - >>> x = np.zeros([3, 4]) - >>> np.may_share_memory(x[:,0], x[:,1]) - True - - """) - - add_newdoc('numpy.core.multiarray', 'ndarray', ('newbyteorder', """ arr.newbyteorder(new_order='S') @@ -3437,81 +3352,6 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('put', """)) -add_newdoc('numpy.core.multiarray', 'copyto', - """ - copyto(dst, src, casting='same_kind', where=True) - - Copies values from one array to another, broadcasting as necessary. - - Raises a TypeError if the `casting` rule is violated, and if - `where` is provided, it selects which elements to copy. - - .. versionadded:: 1.7.0 - - Parameters - ---------- - dst : ndarray - The array into which values are copied. - src : array_like - The array from which values are copied. - casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional - Controls what kind of data casting may occur when copying. - - * 'no' means the data types should not be cast at all. - * 'equiv' means only byte-order changes are allowed. - * 'safe' means only casts which can preserve values are allowed. - * 'same_kind' means only safe casts or casts within a kind, - like float64 to float32, are allowed. - * 'unsafe' means any data conversions may be done. - where : array_like of bool, optional - A boolean array which is broadcasted to match the dimensions - of `dst`, and selects elements to copy from `src` to `dst` - wherever it contains the value True. - - """) - -add_newdoc('numpy.core.multiarray', 'putmask', - """ - putmask(a, mask, values) - - Changes elements of an array based on conditional and input values. - - Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``. - - If `values` is not the same size as `a` and `mask` then it will repeat. - This gives behavior different from ``a[mask] = values``. - - Parameters - ---------- - a : array_like - Target array. - mask : array_like - Boolean mask array. It has to be the same shape as `a`. - values : array_like - Values to put into `a` where `mask` is True. If `values` is smaller - than `a` it will be repeated. - - See Also - -------- - place, put, take, copyto - - Examples - -------- - >>> x = np.arange(6).reshape(2, 3) - >>> np.putmask(x, x>2, x**2) - >>> x - array([[ 0, 1, 2], - [ 9, 16, 25]]) - - If `values` is smaller than `a` it is repeated: - - >>> x = np.arange(5) - >>> np.putmask(x, x>1, [-33, -44]) - >>> x - array([ 0, 1, -33, -44, -33]) - - """) - add_newdoc('numpy.core.multiarray', 'ndarray', ('ravel', """ @@ -6997,3 +6837,4 @@ for float_name in ('half', 'single', 'double', 'longdouble'): >>> np.{ftype}(-.25).as_integer_ratio() (-1, 4) """.format(ftype=float_name))) + diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 739ae7711..108364824 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -114,6 +114,7 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, threshold : int, optional Total number of array elements which trigger summarization rather than full repr (default 1000). + To always use the full repr without summarization, pass `sys.maxsize`. edgeitems : int, optional Number of array items in summary at beginning and end of each dimension (default 3). diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 08f17aae4..3389e7d66 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -908,14 +908,18 @@ def sort(a, axis=-1, kind=None, order=None): .. versionadded:: 1.12.0 - quicksort has been changed to an introsort which will switch - heapsort when it does not make enough progress. This makes its - worst case O(n*log(n)). - - 'stable' automatically choses the best stable sorting algorithm - for the data type being sorted. It, along with 'mergesort' is - currently mapped to timsort or radix sort depending on the - data type. API forward compatibility currently limits the + quicksort has been changed to `introsort <https://en.wikipedia.org/wiki/Introsort>`_. + When sorting does not make enough progress it switches to + `heapsort <https://en.wikipedia.org/wiki/Heapsort>`_. + This implementation makes quicksort O(n*log(n)) in the worst case. + + 'stable' automatically chooses the best stable sorting algorithm + for the data type being sorted. + It, along with 'mergesort' is currently mapped to + `timsort <https://en.wikipedia.org/wiki/Timsort>`_ + or `radix sort <https://en.wikipedia.org/wiki/Radix_sort>`_ + depending on the data type. + API forward compatibility currently limits the ability to select the implementation and it is hardwired for the different data types. @@ -924,7 +928,7 @@ def sort(a, axis=-1, kind=None, order=None): Timsort is added for better performance on already or nearly sorted data. On random data timsort is almost identical to mergesort. It is now used for stable sort while quicksort is still the - default sort if none is chosen. For details of timsort, refer to + default sort if none is chosen. For timsort details, refer to `CPython listsort.txt <https://github.com/python/cpython/blob/3.7/Objects/listsort.txt>`_. 'mergesort' and 'stable' are mapped to radix sort for integer data types. Radix sort is an O(n) sort instead of O(n log n). diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 296213823..0dd6ee420 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -431,35 +431,62 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): #always succeed +def _add_docstring(obj, doc): + try: + add_docstring(obj, doc) + except Exception: + pass + + def add_newdoc(place, obj, doc): """ - Adds documentation to obj which is in module place. + Add documentation to an existing object, typically one defined in C + + The purpose is to allow easier editing of the docstrings without requiring + a re-compile. This exists primarily for internal use within numpy itself. - If doc is a string add it to obj as a docstring + Parameters + ---------- + place : str + The absolute name of the module to import from + obj : str + The name of the object to add documentation to, typically a class or + function name + doc : {str, Tuple[str, str], List[Tuple[str, str]]} + If a string, the documentation to apply to `obj` - If doc is a tuple, then the first element is interpreted as - an attribute of obj and the second as the docstring - (method, docstring) + If a tuple, then the first element is interpreted as an attribute of + `obj` and the second as the docstring to apply - ``(method, docstring)`` - If doc is a list, then each element of the list should be a - sequence of length two --> [(method1, docstring1), - (method2, docstring2), ...] + If a list, then each element of the list should be a tuple of length + two - ``[(method1, docstring1), (method2, docstring2), ...]`` - This routine never raises an error. + Notes + ----- + This routine never raises an error if the docstring can't be written, but + will raise an error if the object being documented does not exist. This routine cannot modify read-only docstrings, as appear in new-style classes or built-in functions. Because this routine never raises an error the caller must check manually that the docstrings were changed. + + Since this function grabs the ``char *`` from a c-level str object and puts + it into the ``tp_doc`` slot of the type of `obj`, it violates a number of + C-API best-practices, by: + + - modifying a `PyTypeObject` after calling `PyType_Ready` + - calling `Py_INCREF` on the str and losing the reference, so the str + will never be released + + If possible it should be avoided. """ - try: - new = getattr(__import__(place, globals(), {}, [obj]), obj) - if isinstance(doc, str): - add_docstring(new, doc.strip()) - elif isinstance(doc, tuple): - add_docstring(getattr(new, doc[0]), doc[1].strip()) - elif isinstance(doc, list): - for val in doc: - add_docstring(getattr(new, val[0]), val[1].strip()) - except Exception: - pass + new = getattr(__import__(place, globals(), {}, [obj]), obj) + if isinstance(doc, str): + _add_docstring(new, doc.strip()) + elif isinstance(doc, tuple): + attr, docstring = doc + _add_docstring(getattr(new, attr), docstring.strip()) + elif isinstance(doc, list): + for attr, docstring in doc: + _add_docstring(getattr(new, attr), docstring.strip()) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 4f2c5b78e..c0fcc10ff 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -7,6 +7,7 @@ by importing from the extension module. """ import functools +import sys import warnings import sys @@ -16,7 +17,7 @@ import numpy as np from numpy.core._multiarray_umath import * from numpy.core._multiarray_umath import ( _fastCopyAndTranspose, _flagdict, _insert, _reconstruct, _vec_string, - _ARRAY_API, _monotonicity + _ARRAY_API, _monotonicity, _get_ndarray_c_version ) __all__ = [ @@ -31,15 +32,17 @@ __all__ = [ 'count_nonzero', 'c_einsum', 'datetime_as_string', 'datetime_data', 'digitize', 'dot', 'dragon4_positional', 'dragon4_scientific', 'dtype', 'empty', 'empty_like', 'error', 'flagsobj', 'flatiter', 'format_longfloat', - 'frombuffer', 'fromfile', 'fromiter', 'fromstring', 'getbuffer', 'inner', + 'frombuffer', 'fromfile', 'fromiter', 'fromstring', 'inner', 'int_asbuffer', 'interp', 'interp_complex', 'is_busday', 'lexsort', 'matmul', 'may_share_memory', 'min_scalar_type', 'ndarray', 'nditer', - 'nested_iters', 'newbuffer', 'normalize_axis_index', 'packbits', + 'nested_iters', 'normalize_axis_index', 'packbits', 'promote_types', 'putmask', 'ravel_multi_index', 'result_type', 'scalar', 'set_datetimeparse_function', 'set_legacy_print_mode', 'set_numeric_ops', 'set_string_function', 'set_typeDict', 'shares_memory', 'test_interrupt', 'tracemalloc_domain', 'typeinfo', 'unpackbits', 'unravel_index', 'vdot', 'where', 'zeros'] +if sys.version_info.major < 3: + __all__ += ['newbuffer', 'getbuffer'] # For backward compatibility, make sure pickle imports these functions from here _reconstruct.__module__ = 'numpy.core.multiarray' diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 9061c0518..1365e87bb 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -928,6 +928,7 @@ test_as_c_array(PyObject *NPY_UNUSED(self), PyObject *args) num_dims = PyArray_NDIM(array_obj); descr = PyArray_DESCR(array_obj); + Py_INCREF(descr); /* PyArray_AsCArray steals a reference to this */ switch (num_dims) { case 1: @@ -970,6 +971,7 @@ test_as_c_array(PyObject *NPY_UNUSED(self), PyObject *args) PyArray_Free((PyObject *) array_obj, (void *) array3); break; default: + Py_DECREF(descr); PyErr_SetString(PyExc_ValueError, "array.ndim not in [1, 3]"); return NULL; } @@ -1263,7 +1265,9 @@ pylong_from_int128(npy_extint128_t value) } Py_DECREF(val); + Py_DECREF(val_64); val = tmp; + val_64 = NULL; tmp = PyLong_FromUnsignedLongLong(value.lo); if (tmp == NULL) { diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 3b986ed04..c586a0b1d 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -4414,7 +4414,17 @@ PyArray_DescrFromType(int type) { PyArray_Descr *ret = NULL; - if (type < NPY_NTYPES) { + if (type < 0) { + /* + * It's not valid for type to be less than 0. + * If that happens, then no other branch of + * this if/else chain should be followed. + * This is effectively a no-op that ensures + * the default error is raised. + */ + ret = NULL; + } + else if (type < NPY_NTYPES) { ret = _builtin_descrs[type]; } else if (type == NPY_NOTYPE) { diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index a17d77586..3270bc20d 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -147,7 +147,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (dtype == NULL) { goto fail; } - Py_INCREF(dtype); goto promote_types; } /* Check if it's a NumPy scalar */ @@ -214,6 +213,10 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int itemsize; PyObject *temp; + /* dtype is not used in this (string discovery) branch */ + Py_DECREF(dtype); + dtype = NULL; + if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { goto fail; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 6be3170eb..c17266251 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1955,6 +1955,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (arr == NULL) { if ((flags & NPY_ARRAY_WRITEBACKIFCOPY) || (flags & NPY_ARRAY_UPDATEIFCOPY)) { + Py_DECREF(dtype); Py_XDECREF(newtype); PyErr_SetString(PyExc_TypeError, "WRITEBACKIFCOPY used for non-array input."); diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index f1e4feac2..768eb1e64 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1832,6 +1832,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, return -1; } equal_one = PyObject_RichCompareBool(event, one, Py_EQ); + Py_DECREF(one); if (equal_one == -1) { return -1; } @@ -2250,6 +2251,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (DEPRECATE( "parsing timezone aware datetimes is deprecated; " "this will raise an error in the future") < 0) { + Py_DECREF(tmp); return -1; } @@ -2266,6 +2268,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, * which contains the value we want. */ tmp = PyObject_CallMethod(offset, "total_seconds", ""); + Py_DECREF(offset); if (tmp == NULL) { return -1; } diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 620ca2104..ff85c3fcb 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -77,31 +77,51 @@ _arraydescr_from_ctypes_type(PyTypeObject *type) * and it can be converted to a dtype object. * * Returns a new reference to a dtype object, or NULL - * if this is not possible. When it returns NULL, it does - * not set a Python exception. + * if this is not possible. + * When the return value is true, the dtype attribute should have been used + * and parsed. Currently the only failure mode for a 1 return is a + * RecursionError and the descriptor is set to NULL. + * When the return value is false, no error will be set. */ -NPY_NO_EXPORT PyArray_Descr * -_arraydescr_from_dtype_attr(PyObject *obj) +int +_arraydescr_from_dtype_attr(PyObject *obj, PyArray_Descr **newdescr) { PyObject *dtypedescr; - PyArray_Descr *newdescr = NULL; int ret; /* For arbitrary objects that have a "dtype" attribute */ dtypedescr = PyObject_GetAttrString(obj, "dtype"); - PyErr_Clear(); if (dtypedescr == NULL) { - return NULL; + /* + * This can be reached due to recursion limit being hit while fetching + * the attribute (tested for py3.7). This removes the custom message. + */ + goto fail; } - ret = PyArray_DescrConverter(dtypedescr, &newdescr); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from its " + "`.dtype` attribute.") != 0) { + return 1; + } + + ret = PyArray_DescrConverter(dtypedescr, newdescr); + Py_DECREF(dtypedescr); + Py_LeaveRecursiveCall(); if (ret != NPY_SUCCEED) { - PyErr_Clear(); - return NULL; + goto fail; } - return newdescr; + return 1; + + fail: + /* Ignore all but recursion errors, to give ctypes a full try. */ + if (!PyErr_ExceptionMatches(PyExc_RecursionError)) { + PyErr_Clear(); + return 0; + } + return 1; } /* @@ -1425,11 +1445,16 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) check_num = NPY_VOID; } else { - *at = _arraydescr_from_dtype_attr(obj); - if (*at) { + if (_arraydescr_from_dtype_attr(obj, at)) { + /* + * Using dtype attribute, *at may be NULL if a + * RecursionError occurred. + */ + if (*at == NULL) { + goto error; + } return NPY_SUCCEED; } - /* * Note: this comes after _arraydescr_from_dtype_attr because the ctypes * type might override the dtype if numpy does not otherwise @@ -1521,7 +1546,8 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) /* A typecode like 'd' */ if (len == 1) { - check_num = type[0]; + /* Python byte string characters are unsigned */ + check_num = (unsigned char) type[0]; } /* A kind + size like 'f8' */ else { @@ -1608,14 +1634,16 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) goto fail; } else { - *at = _arraydescr_from_dtype_attr(obj); - if (*at) { + if (_arraydescr_from_dtype_attr(obj, at)) { + /* + * Using dtype attribute, *at may be NULL if a + * RecursionError occurred. + */ + if (*at == NULL) { + goto error; + } return NPY_SUCCEED; } - if (PyErr_Occurred()) { - return NPY_FAIL; - } - /* * Note: this comes after _arraydescr_from_dtype_attr because the ctypes * type might override the dtype if numpy does not otherwise diff --git a/numpy/core/src/multiarray/descriptor.h b/numpy/core/src/multiarray/descriptor.h index de7497363..6024c5e77 100644 --- a/numpy/core/src/multiarray/descriptor.h +++ b/numpy/core/src/multiarray/descriptor.h @@ -7,8 +7,8 @@ NPY_NO_EXPORT PyObject *arraydescr_protocol_descr_get(PyArray_Descr *self); NPY_NO_EXPORT PyObject * array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args); -NPY_NO_EXPORT PyArray_Descr * -_arraydescr_from_dtype_attr(PyObject *obj); +int +_arraydescr_from_dtype_attr(PyObject *obj, PyArray_Descr **newdescr); NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 11c45dce5..9351b5fc7 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -2236,6 +2236,7 @@ PyArray_Nonzero(PyArrayObject *self) static npy_intp const zero_dim_shape[1] = {1}; static npy_intp const zero_dim_strides[1] = {0}; + Py_INCREF(PyArray_DESCR(self)); /* array creation steals reference */ PyArrayObject *self_1d = (PyArrayObject *)PyArray_NewFromDescrAndBase( Py_TYPE(self), PyArray_DESCR(self), 1, zero_dim_shape, zero_dim_strides, PyArray_BYTES(self), @@ -2243,7 +2244,9 @@ PyArray_Nonzero(PyArrayObject *self) if (self_1d == NULL) { return NULL; } - return PyArray_Nonzero(self_1d); + ret_tuple = PyArray_Nonzero(self_1d); + Py_DECREF(self_1d); + return ret_tuple; } /* diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 64c978361..83eafaf74 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1399,6 +1399,7 @@ arraymultiter_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, } n = PySequence_Fast_GET_SIZE(fast_seq); if (n > NPY_MAXARGS) { + Py_DECREF(fast_seq); return multiiter_wrong_number_of_args(); } ret = multiiter_new_impl(n, PySequence_Fast_ITEMS(fast_seq)); diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 9e54a2c64..add1143b2 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -2516,6 +2516,7 @@ PyArray_MapIterCheckIndices(PyArrayMapIterObject *mit) indval = *((npy_intp*)data); if (check_and_adjust_index(&indval, outer_dim, outer_axis, _save) < 0) { + Py_DECREF(intp_type); return -1; } data += stride; @@ -2616,7 +2617,8 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, PyArrayObject *original_extra_op = extra_op; PyArrayObject *index_arrays[NPY_MAXDIMS]; - PyArray_Descr *dtypes[NPY_MAXDIMS]; + PyArray_Descr *intp_descr; + PyArray_Descr *dtypes[NPY_MAXDIMS]; /* borrowed references */ npy_uint32 op_flags[NPY_MAXDIMS]; npy_uint32 outer_flags; @@ -2629,9 +2631,15 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, int nops; int uses_subspace; + intp_descr = PyArray_DescrFromType(NPY_INTP); + if (intp_descr == NULL) { + return NULL; + } + /* create new MapIter object */ mit = (PyArrayMapIterObject *)PyArray_malloc(sizeof(PyArrayMapIterObject)); if (mit == NULL) { + Py_DECREF(intp_descr); return NULL; } /* set all attributes of mapiter to zero */ @@ -2661,6 +2669,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, mit->nd_fancy = fancy_ndim; if (mapiter_fill_info(mit, indices, index_num, arr) < 0) { Py_DECREF(mit); + Py_DECREF(intp_descr); return NULL; } @@ -2670,7 +2679,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, for (i=0; i < index_num; i++) { if (indices[i].type & HAS_FANCY) { index_arrays[mit->numiter] = (PyArrayObject *)indices[i].object; - dtypes[mit->numiter] = PyArray_DescrFromType(NPY_INTP); + dtypes[mit->numiter] = intp_descr; op_flags[mit->numiter] = (NPY_ITER_NBO | NPY_ITER_ALIGNED | @@ -2693,9 +2702,10 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, PyArray_DescrFromType(NPY_INTP), 0); if (index_arrays[0] == NULL) { Py_DECREF(mit); + Py_DECREF(intp_descr); return NULL; } - dtypes[0] = PyArray_DescrFromType(NPY_INTP); + dtypes[0] = intp_descr; op_flags[0] = NPY_ITER_NBO | NPY_ITER_ALIGNED | NPY_ITER_READONLY; mit->fancy_dims[0] = 1; @@ -2925,7 +2935,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, nops += 1; index_arrays[mit->numiter] = extra_op; - Py_INCREF(extra_op_dtype); dtypes[mit->numiter] = extra_op_dtype; op_flags[mit->numiter] = (extra_op_flags | NPY_ITER_ALLOCATE | @@ -2951,9 +2960,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, } /* NpyIter cleanup and information: */ - for (i=0; i < nops; i++) { - Py_DECREF(dtypes[i]); - } if (dummy_array) { Py_DECREF(index_arrays[0]); } @@ -3039,6 +3045,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, /* Can now return early if no subspace is being used */ if (!uses_subspace) { Py_XDECREF(extra_op); + Py_DECREF(intp_descr); return (PyObject *)mit; } @@ -3108,6 +3115,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, } Py_XDECREF(extra_op); + Py_DECREF(intp_descr); return (PyObject *)mit; fail: @@ -3176,6 +3184,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, finish: Py_XDECREF(extra_op); + Py_DECREF(intp_descr); Py_DECREF(mit); return NULL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index d458638ae..79c60aa2e 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -365,6 +365,7 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) npy_cache_import("numpy.core._internal", "_getfield_is_safe", &checkfunc); if (checkfunc == NULL) { + Py_DECREF(typed); return NULL; } @@ -372,6 +373,7 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) safe = PyObject_CallFunction(checkfunc, "OOi", PyArray_DESCR(self), typed, offset); if (safe == NULL) { + Py_DECREF(typed); return NULL; } Py_DECREF(safe); @@ -382,14 +384,17 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) /* check that values are valid */ if (typed_elsize > self_elsize) { PyErr_SetString(PyExc_ValueError, "new type is larger than original type"); + Py_DECREF(typed); return NULL; } if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset is negative"); + Py_DECREF(typed); return NULL; } if (offset > self_elsize - typed_elsize) { PyErr_SetString(PyExc_ValueError, "new type plus offset is larger than original type"); + Py_DECREF(typed); return NULL; } @@ -434,6 +439,7 @@ PyArray_SetField(PyArrayObject *self, PyArray_Descr *dtype, int retval = 0; if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { + Py_DECREF(dtype); return -1; } @@ -583,14 +589,13 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } if (PyBytes_Check(file) || PyUnicode_Check(file)) { - file = npy_PyFile_OpenFile(file, "wb"); + Py_SETREF(file, npy_PyFile_OpenFile(file, "wb")); if (file == NULL) { return NULL; } own = 1; } else { - Py_INCREF(file); own = 0; } @@ -2035,6 +2040,7 @@ array_setstate(PyArrayObject *self, PyObject *args) #endif npy_intp num = PyArray_NBYTES(self); if (num == 0) { + Py_DECREF(rawdata); Py_RETURN_NONE; } fa->data = PyDataMem_NEW(num); @@ -2385,7 +2391,6 @@ array_clip(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_conjugate(PyArrayObject *self, PyObject *args) { - PyArrayObject *out = NULL; if (!PyArg_ParseTuple(args, "|O&:conjugate", PyArray_OutputConverter, diff --git a/numpy/core/src/multiarray/methods.h b/numpy/core/src/multiarray/methods.h index b96a3c8a8..7a9a24a00 100644 --- a/numpy/core/src/multiarray/methods.h +++ b/numpy/core/src/multiarray/methods.h @@ -8,7 +8,10 @@ extern NPY_NO_EXPORT PyMethodDef array_methods[]; NPY_NO_EXPORT const char * npy_casting_to_string(NPY_CASTING casting); -/* Pathlib support */ +/* + * Pathlib support, takes a borrowed reference and returns a new one. + * The new object may be the same as the old. + */ static inline PyObject * NpyPath_PathlikeToFspath(PyObject *file) { @@ -24,6 +27,7 @@ NpyPath_PathlikeToFspath(PyObject *file) } if (!PyObject_IsInstance(file, os_PathLike)) { + Py_INCREF(file); return file; } return PyObject_CallFunctionObjArgs(os_fspath, file, NULL); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 915c9fcd9..e5df6bf1b 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2086,22 +2086,25 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) if (offset != 0 && strcmp(sep, "") != 0) { PyErr_SetString(PyExc_TypeError, "'offset' argument only permitted for binary files"); + Py_XDECREF(type); + Py_DECREF(file); return NULL; } if (PyString_Check(file) || PyUnicode_Check(file)) { - file = npy_PyFile_OpenFile(file, "rb"); + Py_SETREF(file, npy_PyFile_OpenFile(file, "rb")); if (file == NULL) { + Py_XDECREF(type); return NULL; } own = 1; } else { - Py_INCREF(file); own = 0; } fp = npy_PyFile_Dup2(file, "rb", &orig_pos); if (fp == NULL) { Py_DECREF(file); + Py_XDECREF(type); return NULL; } if (npy_fseek(fp, offset, SEEK_CUR) != 0) { @@ -3825,9 +3828,11 @@ _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) else { PyErr_SetString(PyExc_TypeError, "string operation on non-string array"); + Py_DECREF(type); goto err; } if (method == NULL) { + Py_DECREF(type); goto err; } diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 30a81e0ca..ffea08bb3 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -572,6 +572,7 @@ npyiter_convert_op_axes(PyObject *op_axes_in, npy_intp nop, if (*oa_ndim > NPY_MAXDIMS) { PyErr_SetString(PyExc_ValueError, "Too many dimensions in op_axes"); + Py_DECREF(a); return 0; } } @@ -602,8 +603,8 @@ npyiter_convert_op_axes(PyObject *op_axes_in, npy_intp nop, } Py_DECREF(v); } - Py_DECREF(a); } + Py_DECREF(a); } if (*oa_ndim == -1) { diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index ee847af9b..b669a3e76 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -471,12 +471,18 @@ PyArray_DescrFromTypeObject(PyObject *type) /* Do special thing for VOID sub-types */ if (PyType_IsSubtype((PyTypeObject *)type, &PyVoidArrType_Type)) { new = PyArray_DescrNewFromType(NPY_VOID); - conv = _arraydescr_from_dtype_attr(type); - if (conv) { + if (new == NULL) { + return NULL; + } + if (_arraydescr_from_dtype_attr(type, &conv)) { + if (conv == NULL) { + Py_DECREF(new); + return NULL; + } new->fields = conv->fields; - Py_INCREF(new->fields); + Py_XINCREF(new->fields); new->names = conv->names; - Py_INCREF(new->names); + Py_XINCREF(new->names); new->elsize = conv->elsize; new->subarray = conv->subarray; conv->subarray = NULL; diff --git a/numpy/core/src/umath/funcs.inc.src b/numpy/core/src/umath/funcs.inc.src index c2732f925..10ed66e50 100644 --- a/numpy/core/src/umath/funcs.inc.src +++ b/numpy/core/src/umath/funcs.inc.src @@ -161,7 +161,7 @@ npy_ObjectLogicalNot(PyObject *i1) static PyObject * npy_ObjectFloor(PyObject *obj) { - PyObject *math_floor_func = NULL; + static PyObject *math_floor_func = NULL; npy_cache_import("math", "floor", &math_floor_func); if (math_floor_func == NULL) { @@ -172,7 +172,7 @@ npy_ObjectFloor(PyObject *obj) { static PyObject * npy_ObjectCeil(PyObject *obj) { - PyObject *math_ceil_func = NULL; + static PyObject *math_ceil_func = NULL; npy_cache_import("math", "ceil", &math_ceil_func); if (math_ceil_func == NULL) { @@ -183,7 +183,7 @@ npy_ObjectCeil(PyObject *obj) { static PyObject * npy_ObjectTrunc(PyObject *obj) { - PyObject *math_trunc_func = NULL; + static PyObject *math_trunc_func = NULL; npy_cache_import("math", "trunc", &math_trunc_func); if (math_trunc_func == NULL) { @@ -228,7 +228,8 @@ npy_ObjectGCD(PyObject *i1, PyObject *i2) return NULL; } /* _gcd has some unusual behaviour regarding sign */ - return PyNumber_Absolute(gcd); + Py_SETREF(gcd, PyNumber_Absolute(gcd)); + return gcd; } } @@ -246,17 +247,19 @@ npy_ObjectLCM(PyObject *i1, PyObject *i2) * no remainder */ tmp = PyNumber_FloorDivide(i1, gcd); + Py_DECREF(gcd); if(tmp == NULL) { return NULL; } - tmp = PyNumber_Multiply(tmp, i2); + Py_SETREF(tmp, PyNumber_Multiply(tmp, i2)); if(tmp == NULL) { return NULL; } /* even though we fix gcd to be positive, we need to do it again here */ - return PyNumber_Absolute(tmp); + Py_SETREF(tmp, PyNumber_Absolute(tmp)); + return tmp; } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index f84d74efe..1a4885133 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -394,9 +394,11 @@ PyUFunc_O_O_method(char **args, npy_intp *dimensions, npy_intp *steps, void *fun i, type->tp_name, meth); npy_PyErr_ChainExceptionsCause(exc, val, tb); Py_DECREF(tup); + Py_XDECREF(func); return; } ret = PyObject_Call(func, tup, NULL); + Py_DECREF(func); if (ret == NULL) { Py_DECREF(tup); return; diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index fda2a12f6..8ae2f65e0 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -528,7 +528,9 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_ITER_ALIGNED; if (wheremask != NULL) { op[2] = wheremask; - op_dtypes[2] = PyArray_DescrFromType(NPY_BOOL); + /* wheremask is guaranteed to be NPY_BOOL, so borrow its reference */ + op_dtypes[2] = PyArray_DESCR(wheremask); + assert(op_dtypes[2]->type_num == NPY_BOOL); if (op_dtypes[2] == NULL) { goto fail; } diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index cb24f2a70..5f9a0f7f4 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -4766,6 +4766,9 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) wrapped = _apply_array_wrap(wraparr[i], mps[j], &context); mps[j] = NULL; /* Prevent fail double-freeing this */ if (wrapped == NULL) { + for (j = 0; j < i; j++) { + Py_DECREF(retobj[j]); + } goto fail; } @@ -5693,18 +5696,13 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) * Create dtypes array for either one or two input operands. * The output operand is set to the first input operand */ - dtypes[0] = PyArray_DESCR(op1_array); operands[0] = op1_array; if (op2_array != NULL) { - dtypes[1] = PyArray_DESCR(op2_array); - dtypes[2] = dtypes[0]; operands[1] = op2_array; operands[2] = op1_array; nop = 3; } else { - dtypes[1] = dtypes[0]; - dtypes[2] = NULL; operands[1] = op1_array; operands[2] = NULL; nop = 2; @@ -5861,9 +5859,10 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); - Py_XDECREF(array_operands[0]); - Py_XDECREF(array_operands[1]); - Py_XDECREF(array_operands[2]); + for (i = 0; i < 3; i++) { + Py_XDECREF(dtypes[i]); + Py_XDECREF(array_operands[i]); + } if (needs_api && PyErr_Occurred()) { return NULL; @@ -5880,9 +5879,10 @@ fail: Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); - Py_XDECREF(array_operands[0]); - Py_XDECREF(array_operands[1]); - Py_XDECREF(array_operands[2]); + for (i = 0; i < 3; i++) { + Py_XDECREF(dtypes[i]); + Py_XDECREF(array_operands[i]); + } return NULL; } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 25dd002ac..d837df117 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -548,6 +548,7 @@ PyUFunc_SimpleUniformOperationTypeResolver( } out_dtypes[0] = ensure_dtype_nbo(dtype); + Py_DECREF(dtype); if (out_dtypes[0] == NULL) { return -1; } @@ -2264,7 +2265,6 @@ PyUFunc_DivmodTypeResolver(PyUFuncObject *ufunc, out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = PyArray_DescrFromType(NPY_LONGLONG); - Py_INCREF(out_dtypes[2]); out_dtypes[3] = out_dtypes[0]; Py_INCREF(out_dtypes[3]); } diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index d24ab98e3..f60eab696 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -88,6 +88,36 @@ class TestBuiltin(object): assert_raises(TypeError, np.dtype, 'q8') assert_raises(TypeError, np.dtype, 'Q8') + @pytest.mark.parametrize( + 'value', + ['m8', 'M8', 'datetime64', 'timedelta64', + 'i4, (2,3)f8, f4', 'a3, 3u8, (3,4)a10', + '>f', '<f', '=f', '|f', + ]) + def test_dtype_bytes_str_equivalence(self, value): + bytes_value = value.encode('ascii') + from_bytes = np.dtype(bytes_value) + from_str = np.dtype(value) + assert_dtype_equal(from_bytes, from_str) + + def test_dtype_from_bytes(self): + # Empty bytes object + assert_raises(TypeError, np.dtype, b'') + # Byte order indicator, but no type + assert_raises(TypeError, np.dtype, b'|') + + # Single character with ordinal < NPY_NTYPES returns + # type by index into _builtin_descrs + assert_dtype_equal(np.dtype(bytes([0])), np.dtype('bool')) + assert_dtype_equal(np.dtype(bytes([17])), np.dtype(object)) + + # Single character where value is a valid type code + assert_dtype_equal(np.dtype(b'f'), np.dtype('float32')) + + # Bytes with non-ascii values raise errors + assert_raises(TypeError, np.dtype, b'\xff') + assert_raises(TypeError, np.dtype, b's\xff') + def test_bad_param(self): # Can't give a size that's too small assert_raises(ValueError, np.dtype, @@ -1006,6 +1036,50 @@ def test_invalid_dtype_string(): assert_raises(TypeError, np.dtype, u'Fl\xfcgel') +class TestFromDTypeAttribute(object): + def test_simple(self): + class dt: + dtype = "f8" + + assert np.dtype(dt) == np.float64 + assert np.dtype(dt()) == np.float64 + + def test_recursion(self): + class dt: + pass + + dt.dtype = dt + with pytest.raises(RecursionError): + np.dtype(dt) + + dt_instance = dt() + dt_instance.dtype = dt + with pytest.raises(RecursionError): + np.dtype(dt_instance) + + def test_void_subtype(self): + class dt(np.void): + # This code path is fully untested before, so it is unclear + # what this should be useful for. Note that if np.void is used + # numpy will think we are deallocating a base type [1.17, 2019-02]. + dtype = np.dtype("f,f") + pass + + np.dtype(dt) + np.dtype(dt(1)) + + def test_void_subtype_recursion(self): + class dt(np.void): + pass + + dt.dtype = dt + + with pytest.raises(RecursionError): + np.dtype(dt) + + with pytest.raises(RecursionError): + np.dtype(dt(1)) + class TestFromCTypes(object): @staticmethod diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 1f21c5f4d..3a5a5b939 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -4183,6 +4183,7 @@ class TestArgmax(object): assert_equal(a.argmax(out=out1, axis=0), np.argmax(a, out=out2, axis=0)) assert_equal(out1, out2) + @pytest.mark.leaks_references(reason="replaces None with NULL.") def test_object_argmax_with_NULLs(self): # See gh-6032 a = np.empty(4, dtype='O') @@ -4331,6 +4332,7 @@ class TestArgmin(object): assert_equal(a.argmin(out=out1, axis=0), np.argmin(a, out=out2, axis=0)) assert_equal(out1, out2) + @pytest.mark.leaks_references(reason="replaces None with NULL.") def test_object_argmin_with_NULLs(self): # See gh-6032 a = np.empty(4, dtype='O') @@ -7200,6 +7202,13 @@ class TestNewBufferProtocol(object): RuntimeError, "ndim", np.array, m) + # The above seems to create some deep cycles, clean them up for + # easier reference count debugging: + del c_u8_33d, m + for i in range(33): + if gc.collect() == 0: + break + def test_error_pointer_type(self): # gh-6741 m = memoryview(ctypes.pointer(ctypes.c_uint8())) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index cd2034d9c..d2ce74282 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1655,7 +1655,6 @@ class TestSpecialMethods(object): ok = np.empty(1).view(Ok) bad = np.empty(1).view(Bad) - # double-free (segfault) of "ok" if "bad" raises an exception for i in range(10): assert_raises(RuntimeError, ncu.frexp, 1, ok, bad) diff --git a/numpy/distutils/fcompiler/compaq.py b/numpy/distutils/fcompiler/compaq.py index 07d502706..671b3a55f 100644 --- a/numpy/distutils/fcompiler/compaq.py +++ b/numpy/distutils/fcompiler/compaq.py @@ -95,7 +95,7 @@ class CompaqVisualFCompiler(FCompiler): raise e except ValueError: e = get_exception() - if not "path']" in str(e): + if not "'path'" in str(e): print("Unexpected ValueError in", __file__) raise e diff --git a/numpy/doc/basics.py b/numpy/doc/basics.py index 7946c6432..1871512bf 100644 --- a/numpy/doc/basics.py +++ b/numpy/doc/basics.py @@ -314,8 +314,8 @@ compiler's ``long double`` available as ``np.longdouble`` (and ``np.clongdouble`` for the complex numbers). You can find out what your numpy provides with ``np.finfo(np.longdouble)``. -NumPy does not provide a dtype with more precision than C -``long double``\\s; in particular, the 128-bit IEEE quad precision +NumPy does not provide a dtype with more precision than C's +``long double``\\; in particular, the 128-bit IEEE quad precision data type (FORTRAN's ``REAL*16``\\) is not available. For efficient memory alignment, ``np.longdouble`` is usually stored diff --git a/numpy/doc/ufuncs.py b/numpy/doc/ufuncs.py index a112e559c..df2c455ec 100644 --- a/numpy/doc/ufuncs.py +++ b/numpy/doc/ufuncs.py @@ -13,9 +13,9 @@ example is the addition operator: :: >>> np.array([0,2,3,4]) + np.array([1,1,-1,2]) array([1, 3, 2, 6]) -The unfunc module lists all the available ufuncs in numpy. Documentation on +The ufunc module lists all the available ufuncs in numpy. Documentation on the specific ufuncs may be found in those modules. This documentation is -intended to address the more general aspects of unfuncs common to most of +intended to address the more general aspects of ufuncs common to most of them. All of the ufuncs that make use of Python operators (e.g., +, -, etc.) have equivalent functions defined (e.g. add() for +) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 93bdbce97..3bf818812 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -384,7 +384,7 @@ def _wrap_header_guess_version(header): return ret header = _wrap_header(header, (3, 0)) - warnings.warn("Stored array in format 3.0. It can only be" + warnings.warn("Stored array in format 3.0. It can only be " "read by NumPy >= 1.17", UserWarning, stacklevel=2) return header diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 1fcb6137c..9d380e67d 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -358,12 +358,12 @@ def average(a, axis=None, weights=None, returned=False): Examples -------- - >>> data = list(range(1,5)) + >>> data = np.arange(1, 5) >>> data - [1, 2, 3, 4] + array([1, 2, 3, 4]) >>> np.average(data) 2.5 - >>> np.average(range(1,11), weights=range(10,0,-1)) + >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1)) 4.0 >>> data = np.arange(6).reshape((3,2)) @@ -3102,6 +3102,7 @@ def i0(x): array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary """ + x = np.asanyarray(x) x = np.abs(x) return piecewise(x, [x <= 8.0], [_i0_1, _i0_2]) diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index e04fdc808..15cd3ad9d 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -204,14 +204,18 @@ class TestStringConverter(object): def test_upgrademapper(self): "Tests updatemapper" dateparser = _bytes_to_date - StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) - convert = StringConverter(dateparser, date(2000, 1, 1)) - test = convert('2001-01-01') - assert_equal(test, date(2001, 1, 1)) - test = convert('2009-01-01') - assert_equal(test, date(2009, 1, 1)) - test = convert('') - assert_equal(test, date(2000, 1, 1)) + _original_mapper = StringConverter._mapper[:] + try: + StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) + convert = StringConverter(dateparser, date(2000, 1, 1)) + test = convert('2001-01-01') + assert_equal(test, date(2001, 1, 1)) + test = convert('2009-01-01') + assert_equal(test, date(2009, 1, 1)) + test = convert('') + assert_equal(test, date(2000, 1, 1)) + finally: + StringConverter._mapper = _original_mapper def test_string_to_object(self): "Make sure that string-to-object functions are properly recognized" diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index c0b8ad6b8..eae52c002 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -2005,6 +2005,22 @@ class Test_I0(object): assert_equal(i0_0.shape, (1,)) assert_array_equal(np.i0([0.]), np.array([1.])) + def test_non_array(self): + a = np.arange(4) + + class array_like: + __array_interface__ = a.__array_interface__ + + def __array_wrap__(self, arr): + return self + + # E.g. pandas series survive ufunc calls through array-wrap: + assert isinstance(np.abs(array_like()), array_like) + exp = np.i0(a) + res = np.i0(array_like()) + + assert_array_equal(exp, res) + class TestKaiser(object): diff --git a/numpy/linalg/lapack_lite/fortran.py b/numpy/linalg/lapack_lite/fortran.py index 87c27aab9..dc0a5ebd9 100644 --- a/numpy/linalg/lapack_lite/fortran.py +++ b/numpy/linalg/lapack_lite/fortran.py @@ -54,7 +54,7 @@ class PushbackIterator(object): Return an iterator for which items can be pushed back into. Call the .pushback(item) method to have item returned as the next - value of .next(). + value of next(). """ def __init__(self, iterable): object.__init__(self) diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 20db2d655..f221b319a 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -800,7 +800,7 @@ class _DomainCheckInterval(object): def __init__(self, a, b): "domain_check_interval(a,b)(x) = true where x < a or y > b" - if (a > b): + if a > b: (a, b) = (b, a) self.a = a self.b = b @@ -1165,7 +1165,7 @@ class _DomainedBinaryOperation(_MaskedUFunc): if domain is not None: m |= domain(da, db) # Take care of the scalar case first - if (not m.ndim): + if not m.ndim: if m: return masked else: @@ -1743,7 +1743,7 @@ def mask_or(m1, m2, copy=False, shrink=True): if m1 is m2 and is_mask(m1): return m1 (dtype1, dtype2) = (getattr(m1, 'dtype', None), getattr(m2, 'dtype', None)) - if (dtype1 != dtype2): + if dtype1 != dtype2: raise ValueError("Incompatible dtypes '%s'<>'%s'" % (dtype1, dtype2)) if dtype1.names is not None: # Allocate an output mask array with the properly broadcast shape. @@ -2681,15 +2681,13 @@ class MaskedIterator(object): -------- >>> x = np.ma.array([3, 2], mask=[0, 1]) >>> fl = x.flat - >>> fl.next() + >>> next(fl) 3 - >>> fl.next() + >>> next(fl) masked - >>> fl.next() + >>> next(fl) Traceback (most recent call last): - File "<stdin>", line 1, in <module> - File "/home/ralf/python/numpy/numpy/ma/core.py", line 2243, in next - d = self.dataiter.next() + ... StopIteration """ @@ -3077,7 +3075,7 @@ class MaskedArray(ndarray): def view(self, dtype=None, type=None, fill_value=None): """ - Return a view of the MaskedArray data + Return a view of the MaskedArray data. Parameters ---------- @@ -3091,6 +3089,14 @@ class MaskedArray(ndarray): type : Python type, optional Type of the returned view, either ndarray or a subclass. The default None results in type preservation. + fill_value : scalar, optional + The value to use for invalid entries (None by default). + If None, then this argument is inferred from the passed `dtype`, or + in its absence the original array, as discussed in the notes below. + + See Also + -------- + numpy.ndarray.view : Equivalent method on ndarray object. Notes ----- @@ -3143,7 +3149,7 @@ class MaskedArray(ndarray): # also make the mask be a view (so attr changes to the view's # mask do no affect original object's mask) # (especially important to avoid affecting np.masked singleton) - if (getmask(output) is not nomask): + if getmask(output) is not nomask: output._mask = output._mask.view() # Make sure to reset the _fill_value if needed @@ -3156,7 +3162,6 @@ class MaskedArray(ndarray): else: output.fill_value = fill_value return output - view.__doc__ = ndarray.view.__doc__ def __getitem__(self, indx): """ @@ -3382,7 +3387,7 @@ class MaskedArray(ndarray): if mask is masked: mask = True - if (current_mask is nomask): + if current_mask is nomask: # Make sure the mask is set # Just don't do anything if there's nothing to do. if mask is nomask: @@ -5036,7 +5041,7 @@ class MaskedArray(ndarray): result = self.filled(0).sum(axis, dtype=dtype, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask return out @@ -5118,7 +5123,7 @@ class MaskedArray(ndarray): result = self.filled(1).prod(axis, dtype=dtype, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask return out @@ -5197,7 +5202,7 @@ class MaskedArray(ndarray): out.flat = result if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = getmask(result) return out @@ -5239,9 +5244,9 @@ class MaskedArray(ndarray): return m if not axis: - return (self - m) + return self - m else: - return (self - expand_dims(m, axis)) + return self - expand_dims(m, axis) def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): @@ -5656,7 +5661,7 @@ class MaskedArray(ndarray): result = self.filled(fill_value).min(axis=axis, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask else: @@ -5790,7 +5795,7 @@ class MaskedArray(ndarray): result = self.filled(fill_value).max(axis=axis, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask else: @@ -6733,7 +6738,7 @@ def power(a, b, third=None): invalid = np.logical_not(np.isfinite(result.view(ndarray))) # Add the initial mask if m is not nomask: - if not (result.ndim): + if not result.ndim: return masked result._mask = np.logical_or(m, invalid) # Fix the invalid parts diff --git a/numpy/random/generator.pyx b/numpy/random/generator.pyx index 6adf0f00b..c7432d8c1 100644 --- a/numpy/random/generator.pyx +++ b/numpy/random/generator.pyx @@ -353,7 +353,8 @@ cdef class Generator: Return random integers from `low` (inclusive) to `high` (exclusive), or if endpoint=True, `low` (inclusive) to `high` (inclusive). Replaces - randint (with endpoint=False) and random_integers (with endpoint=True) + `RandomState.randint` (with endpoint=False) and + `RandomState.random_integers` (with endpoint=True) Return random integers from the "discrete uniform" distribution of the specified dtype. If `high` is None (the default), then results are @@ -503,15 +504,8 @@ cdef class Generator: return self.integers(0, 4294967296, size=n_uint32, dtype=np.uint32).astype('<u4').tobytes()[:length] - def randint(self, low, high=None, size=None, dtype=np.int64, endpoint=False): - """ - Deprecated, renamed to ``integers`` - """ - warnings.warn("Renamed to integers", RuntimeWarning) - self.integers(low, high, size, dtype, endpoint) - @cython.wraparound(True) - def choice(self, a, size=None, replace=True, p=None, axis=0): + def choice(self, a, size=None, replace=True, p=None, axis=0, bint shuffle=True): """ choice(a, size=None, replace=True, p=None, axis=0): @@ -538,6 +532,9 @@ cdef class Generator: axis : int, optional The axis along which the selection is performed. The default, 0, selects by row. + shuffle : boolean, optional + Whether the sample is shuffled when sampling without replacement. + Default is True, False provides a speedup. Returns ------- @@ -593,14 +590,12 @@ cdef class Generator: dtype='<U11') """ - cdef char* idx_ptr - cdef int64_t buf - cdef char* buf_ptr - cdef set idx_set cdef int64_t val, t, loc, size_i, pop_size_i cdef int64_t *idx_data cdef np.npy_intp j + cdef uint64_t set_size, mask + cdef uint64_t[::1] hash_set # Format and Verify input a = np.array(a, copy=False) if a.ndim == 0: @@ -687,36 +682,45 @@ cdef class Generator: size_i = size pop_size_i = pop_size # This is a heuristic tuning. should be improvable - if pop_size_i > 200 and (size > 200 or size > (10 * pop_size // size)): + if shuffle: + cutoff = 50 + else: + cutoff = 20 + if pop_size_i > 10000 and (size_i > (pop_size_i // cutoff)): # Tail shuffle size elements - idx = np.arange(pop_size, dtype=np.int64) - idx_ptr = np.PyArray_BYTES(<np.ndarray>idx) - buf_ptr = <char*>&buf - self._shuffle_raw(pop_size_i, max(pop_size_i - size_i,1), - 8, 8, idx_ptr, buf_ptr) + idx = np.PyArray_Arange(0, pop_size_i, 1, np.NPY_INT64) + idx_data = <int64_t*>(<np.ndarray>idx).data + with self.lock, nogil: + self._shuffle_int(pop_size_i, max(pop_size_i - size_i, 1), + idx_data) # Copy to allow potentially large array backing idx to be gc idx = idx[(pop_size - size):].copy() else: - # Floyds's algorithm with precomputed indices - # Worst case, O(n**2) when size is close to pop_size + # Floyd's algorithm idx = np.empty(size, dtype=np.int64) idx_data = <int64_t*>np.PyArray_DATA(<np.ndarray>idx) - idx_set = set() - loc = 0 - # Sample indices with one pass to avoid reacquiring the lock - with self.lock: - for j in range(pop_size_i - size_i, pop_size_i): - idx_data[loc] = random_interval(&self._bitgen, j) - loc += 1 - loc = 0 - while len(idx_set) < size_i: + # smallest power of 2 larger than 1.2 * size + set_size = <uint64_t>(1.2 * size_i) + mask = _gen_mask(set_size) + set_size = 1 + mask + hash_set = np.full(set_size, <uint64_t>-1, np.uint64) + with self.lock, cython.wraparound(False), nogil: for j in range(pop_size_i - size_i, pop_size_i): - if idx_data[loc] not in idx_set: - val = idx_data[loc] - else: - idx_data[loc] = val = j - idx_set.add(val) - loc += 1 + val = random_bounded_uint64(&self._bitgen, 0, j, 0, 0) + loc = val & mask + while hash_set[loc] != <uint64_t>-1 and hash_set[loc] != <uint64_t>val: + loc = (loc + 1) & mask + if hash_set[loc] == <uint64_t>-1: # then val not in hash_set + hash_set[loc] = val + idx_data[j - pop_size_i + size_i] = val + else: # we need to insert j instead + loc = j & mask + while hash_set[loc] != <uint64_t>-1: + loc = (loc + 1) & mask + hash_set[loc] = j + idx_data[j - pop_size_i + size_i] = j + if shuffle: + self._shuffle_int(size_i, 1, idx_data) if shape is not None: idx.shape = shape @@ -3888,6 +3892,28 @@ cdef class Generator: string.memcpy(data + j * stride, data + i * stride, itemsize) string.memcpy(data + i * stride, buf, itemsize) + cdef inline void _shuffle_int(self, np.npy_intp n, np.npy_intp first, + int64_t* data) nogil: + """ + Parameters + ---------- + n + Number of elements in data + first + First observation to shuffle. Shuffles n-1, + n-2, ..., first, so that when first=1 the entire + array is shuffled + data + Location of data + """ + cdef np.npy_intp i, j + cdef int64_t temp + for i in reversed(range(first, n)): + j = random_bounded_uint64(&self._bitgen, 0, i, 0, 0) + temp = data[j] + data[j] = data[i] + data[i] = temp + def permutation(self, object x): """ permutation(x) diff --git a/numpy/random/tests/test_generator_mt19937.py b/numpy/random/tests/test_generator_mt19937.py index ef821d46f..a962fe84e 100644 --- a/numpy/random/tests/test_generator_mt19937.py +++ b/numpy/random/tests/test_generator_mt19937.py @@ -568,7 +568,10 @@ class TestRandomDist(object): def test_choice_uniform_noreplace(self): random = Generator(MT19937(self.seed)) actual = random.choice(4, 3, replace=False) - desired = np.array([0, 1, 3], dtype=np.int64) + desired = np.array([2, 0, 3], dtype=np.int64) + assert_array_equal(actual, desired) + actual = random.choice(4, 4, replace=False, shuffle=False) + desired = np.arange(4, dtype=np.int64) assert_array_equal(actual, desired) def test_choice_nonuniform_noreplace(self): @@ -688,7 +691,7 @@ class TestRandomDist(object): def test_choice_large_sample(self): import hashlib - choice_hash = '5ca163da624c938bb3bc93e89a7dec4c' + choice_hash = 'd44962a0b1e92f4a3373c23222244e21' random = Generator(MT19937(self.seed)) actual = random.choice(10000, 5000, replace=False) if sys.byteorder != 'little': |