diff options
Diffstat (limited to 'numpy')
51 files changed, 639 insertions, 214 deletions
diff --git a/numpy/_build_utils/apple_accelerate.py b/numpy/_build_utils/apple_accelerate.py index d7351f4c5..2d5bbab5e 100644 --- a/numpy/_build_utils/apple_accelerate.py +++ b/numpy/_build_utils/apple_accelerate.py @@ -1,3 +1,5 @@ +from __future__ import division, absolute_import, print_function + import os import sys import re diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index b00e229c3..c14036089 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -3826,6 +3826,45 @@ add_newdoc('numpy.core.multiarray', 'shares_memory', """) +add_newdoc('numpy.core.multiarray', 'may_share_memory', + """ + may_share_memory(a, b, max_work=None) + + Determine if two arrays might share memory + + A return of True does not necessarily mean that the two arrays + share any element. It just means that they *might*. + + Only the memory bounds of a and b are checked by default. + + Parameters + ---------- + a, b : ndarray + Input arrays + max_work : int, optional + Effort to spend on solving the overlap problem. See + `shares_memory` for details. Default for ``may_share_memory`` + is to do a bounds check. + + Returns + ------- + out : bool + + See Also + -------- + shares_memory + + Examples + -------- + >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) + False + >>> x = np.zeros([3, 4]) + >>> np.may_share_memory(x[:,0], x[:,1]) + True + + """) + + add_newdoc('numpy.core.multiarray', 'ndarray', ('newbyteorder', """ arr.newbyteorder(new_order='S') diff --git a/numpy/compat/tests/test_compat.py b/numpy/compat/tests/test_compat.py index 9822ab374..1ac24401a 100644 --- a/numpy/compat/tests/test_compat.py +++ b/numpy/compat/tests/test_compat.py @@ -1,3 +1,5 @@ +from __future__ import division, absolute_import, print_function + from os.path import join from numpy.compat import isfileobj diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 0fc572cb6..197513294 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1134,7 +1134,7 @@ def resize(a, new_shape): a = ravel(a) Na = len(a) if not Na: - return mu.zeros(new_shape, a.dtype.char) + return mu.zeros(new_shape, a.dtype) total_size = um.multiply.reduce(new_shape) n_copies = int(total_size / Na) extra = total_size % Na diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 05fea557a..c82c9bb6b 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -1,6 +1,6 @@ from __future__ import division, absolute_import, print_function -__all__ = ['logspace', 'linspace', 'may_share_memory'] +__all__ = ['logspace', 'linspace'] from . import numeric as _nx from .numeric import result_type, NaN, shares_memory, MAY_SHARE_BOUNDS, TooHardError @@ -201,46 +201,3 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): if dtype is None: return _nx.power(base, y) return _nx.power(base, y).astype(dtype) - - -def may_share_memory(a, b, max_work=None): - """Determine if two arrays can share memory - - A return of True does not necessarily mean that the two arrays - share any element. It just means that they *might*. - - Only the memory bounds of a and b are checked by default. - - Parameters - ---------- - a, b : ndarray - Input arrays - max_work : int, optional - Effort to spend on solving the overlap problem. See - `shares_memory` for details. Default for ``may_share_memory`` - is to do a bounds check. - - Returns - ------- - out : bool - - See Also - -------- - shares_memory - - Examples - -------- - >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) - False - >>> x = np.zeros([3, 4]) - >>> np.may_share_memory(x[:,0], x[:,1]) - True - - """ - if max_work is None: - max_work = MAY_SHARE_BOUNDS - try: - return shares_memory(a, b, max_work=max_work) - except (TooHardError, OverflowError): - # Unable to determine, assume yes - return True diff --git a/numpy/core/include/numpy/ndarrayobject.h b/numpy/core/include/numpy/ndarrayobject.h index fbaaeacea..c97a3a797 100644 --- a/numpy/core/include/numpy/ndarrayobject.h +++ b/numpy/core/include/numpy/ndarrayobject.h @@ -96,7 +96,7 @@ extern "C" CONFUSE_EMACS NULL) #define PyArray_FROM_OT(m,type) PyArray_FromAny(m, \ - PyArray_DescrFromType(type), 0, 0, 0, NULL); + PyArray_DescrFromType(type), 0, 0, 0, NULL) #define PyArray_FROM_OTF(m, type, flags) \ PyArray_FromAny(m, PyArray_DescrFromType(type), 0, 0, \ diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index eff5dd339..47ef94c92 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -61,6 +61,21 @@ #define NPY_UNLIKELY(x) (x) #endif +#ifdef HAVE___BUILTIN_PREFETCH +/* unlike _mm_prefetch also works on non-x86 */ +#define NPY_PREFETCH(x, rw, loc) __builtin_prefetch((x), (rw), (loc)) +#else +#ifdef HAVE__MM_PREFETCH +/* _MM_HINT_ET[01] (rw = 1) unsupported, only available in gcc >= 4.9 */ +#define NPY_PREFETCH(x, rw, loc) _mm_prefetch((x), loc == 0 ? _MM_HINT_NTA : \ + (loc == 1 ? _MM_HINT_T2 : \ + (loc == 2 ? _MM_HINT_T1 : \ + (loc == 3 ? _MM_HINT_T0 : -1)))) +#else +#define NPY_PREFETCH(x, rw,loc) +#endif +#endif + #if defined(_MSC_VER) #define NPY_INLINE __inline #elif defined(__GNUC__) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 2ece2ce8d..3b442ea78 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -41,7 +41,8 @@ __all__ = [ 'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_', 'bitwise_not', 'CLIP', 'RAISE', 'WRAP', 'MAXDIMS', 'BUFSIZE', 'ALLOW_THREADS', 'ComplexWarning', 'full', 'full_like', 'matmul', - 'shares_memory', 'MAY_SHARE_BOUNDS', 'MAY_SHARE_EXACT', 'TooHardError', + 'shares_memory', 'may_share_memory', 'MAY_SHARE_BOUNDS', 'MAY_SHARE_EXACT', + 'TooHardError', ] if sys.version_info[0] < 3: @@ -384,6 +385,7 @@ fromiter = multiarray.fromiter fromfile = multiarray.fromfile frombuffer = multiarray.frombuffer shares_memory = multiarray.shares_memory +may_share_memory = multiarray.may_share_memory if sys.version_info[0] < 3: newbuffer = multiarray.newbuffer getbuffer = multiarray.getbuffer diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index 68efd1791..d93e475e3 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -125,7 +125,10 @@ OPTIONAL_INTRINSICS = [("__builtin_isnan", '5.'), ("__builtin_expect", '5, 0'), ("__builtin_mul_overflow", '5, 5, (int*)5'), ("_mm_load_ps", '(float*)0', "xmmintrin.h"), # SSE + ("_mm_prefetch", '(float*)0, _MM_HINT_NTA', + "xmmintrin.h"), # SSE ("_mm_load_pd", '(double*)0', "emmintrin.h"), # SSE2 + ("__builtin_prefetch", "(float*)0, 0, 3"), ] # function attributes diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 7f7607e1f..5fa3ba95b 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -629,8 +629,6 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) { PyArrayObject *self; _buffer_info_t *info = NULL; - int i; - Py_ssize_t sd; self = (PyArrayObject*)obj; @@ -715,15 +713,19 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) * regenerate strides from shape. */ if (PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS) && - !((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)) { - sd = view->itemsize; + !((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)) { + Py_ssize_t sd = view->itemsize; + int i; + for (i = view->ndim-1; i >= 0; --i) { view->strides[i] = sd; sd *= view->shape[i]; } } else if (PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) { - sd = view->itemsize; + Py_ssize_t sd = view->itemsize; + int i; + for (i = 0; i < view->ndim; ++i) { view->strides[i] = sd; sd *= view->shape[i]; diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 8ffeedac2..b9db3bb8f 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -529,14 +529,15 @@ binary_search_with_guess(const npy_double key, const npy_double *arr, } /* - * It would seem that for the following code to work, 'len' should - * at least be 4. But because of the way 'guess' is normalized, it - * will always be set to 1 if len <= 4. Given that, and that keys - * outside of the 'arr' bounds have already been handled, and the - * order in which comparisons happen below, it should become obvious - * that it will work with any array of at least 2 items. + * If len <= 4 use linear search. + * From above we know key >= arr[0] when we start. */ - assert (len >= 2); + if (len <= 4) { + npy_intp i; + + for (i = 1; i < len && key >= arr[i]; ++i); + return i - 1; + } if (guess > len - 3) { guess = len - 3; @@ -546,36 +547,36 @@ binary_search_with_guess(const npy_double key, const npy_double *arr, } /* check most likely values: guess - 1, guess, guess + 1 */ - if (key <= arr[guess]) { - if (key <= arr[guess - 1]) { + if (key < arr[guess]) { + if (key < arr[guess - 1]) { imax = guess - 1; /* last attempt to restrict search to items in cache */ if (guess > LIKELY_IN_CACHE_SIZE && - key > arr[guess - LIKELY_IN_CACHE_SIZE]) { + key >= arr[guess - LIKELY_IN_CACHE_SIZE]) { imin = guess - LIKELY_IN_CACHE_SIZE; } } else { - /* key > arr[guess - 1] */ + /* key >= arr[guess - 1] */ return guess - 1; } } else { - /* key > arr[guess] */ - if (key <= arr[guess + 1]) { + /* key >= arr[guess] */ + if (key < arr[guess + 1]) { return guess; } else { - /* key > arr[guess + 1] */ - if (key <= arr[guess + 2]) { + /* key >= arr[guess + 1] */ + if (key < arr[guess + 2]) { return guess + 1; } else { - /* key > arr[guess + 2] */ + /* key >= arr[guess + 2] */ imin = guess + 2; /* last attempt to restrict search to items in cache */ if (guess < len - LIKELY_IN_CACHE_SIZE - 1 && - key <= arr[guess + LIKELY_IN_CACHE_SIZE]) { + key < arr[guess + LIKELY_IN_CACHE_SIZE]) { imax = guess + LIKELY_IN_CACHE_SIZE; } } @@ -673,7 +674,7 @@ arr_interp(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } } - /* binary_search_with_guess needs at least a 2 item long array */ + /* binary_search_with_guess needs at least a 3 item long array */ if (lenxp == 1) { const npy_double xp_val = dx[0]; const npy_double fp_val = dy[0]; diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 83cd64bdc..03a4654a0 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -806,11 +806,19 @@ _use_inherit(PyArray_Descr *type, PyObject *newobj, int *errflag) } new->elsize = conv->elsize; if (PyDataType_HASFIELDS(conv)) { + Py_XDECREF(new->fields); new->fields = conv->fields; Py_XINCREF(new->fields); + + Py_XDECREF(new->names); new->names = conv->names; Py_XINCREF(new->names); } + if (conv->metadata != NULL) { + Py_XDECREF(new->metadata); + new->metadata = conv->metadata; + Py_XINCREF(new->metadata); + } new->flags = conv->flags; Py_DECREF(conv); *errflag = 0; diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 549ea333a..c2a88e3b9 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -488,11 +488,25 @@ array_descr_set(PyArrayObject *self, PyObject *arg) if ((newtype->elsize != PyArray_DESCR(self)->elsize) && - (PyArray_NDIM(self) == 0 || !PyArray_ISONESEGMENT(self) || - PyDataType_HASSUBARRAY(newtype))) { + (PyArray_NDIM(self) == 0 || + !PyArray_ISONESEGMENT(self) || + PyDataType_HASSUBARRAY(newtype))) { goto fail; } - if (PyArray_ISCONTIGUOUS(self)) { + + /* Deprecate not C contiguous and a dimension changes */ + if (newtype->elsize != PyArray_DESCR(self)->elsize && + !PyArray_IS_C_CONTIGUOUS(self)) { + /* 11/27/2015 1.11.0 */ + if (DEPRECATE("Changing the shape of non-C contiguous array by\n" + "descriptor assignment is deprecated. To maintain\n" + "the Fortran contiguity of a multidimensional Fortran\n" + "array, use 'a.T.view(...).T' instead") < 0) { + return -1; + } + } + + if (PyArray_IS_C_CONTIGUOUS(self)) { i = PyArray_NDIM(self) - 1; } else { diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 44de1cbf2..6c56d77bb 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -169,7 +169,8 @@ prepare_index(PyArrayObject *self, PyObject *index, int new_ndim, fancy_ndim, used_ndim, index_ndim; int curr_idx, get_idx; - npy_intp i, n; + int i; + npy_intp n; npy_bool make_tuple = 0; PyObject *obj = NULL; @@ -348,14 +349,15 @@ prepare_index(PyArrayObject *self, PyObject *index, #else if (PyLong_CheckExact(obj) || !PyArray_Check(obj)) { #endif - i = PyArray_PyIntAsIntp(obj); - if ((i == -1) && PyErr_Occurred()) { + npy_intp ind = PyArray_PyIntAsIntp(obj); + + if ((ind == -1) && PyErr_Occurred()) { PyErr_Clear(); } else { index_type |= HAS_INTEGER; indices[curr_idx].object = NULL; - indices[curr_idx].value = i; + indices[curr_idx].value = ind; indices[curr_idx].type = HAS_INTEGER; used_ndim += 1; new_ndim += 0; @@ -527,15 +529,16 @@ prepare_index(PyArrayObject *self, PyObject *index, * sure that array-likes or odder arrays are always * handled right. */ - i = PyArray_PyIntAsIntp((PyObject *)arr); + npy_intp ind = PyArray_PyIntAsIntp((PyObject *)arr); + Py_DECREF(arr); - if ((i == -1) && PyErr_Occurred()) { + if ((ind == -1) && PyErr_Occurred()) { goto failed_building_indices; } else { index_type |= (HAS_INTEGER | HAS_SCALAR_ARRAY); indices[curr_idx].object = NULL; - indices[curr_idx].value = i; + indices[curr_idx].value = ind; indices[curr_idx].type = HAS_INTEGER; used_ndim += 1; new_ndim += 0; @@ -1293,7 +1296,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) PyArray_NDIM(arr), PyArray_SHAPE(arr), PyArray_STRIDES(arr), - PyArray_DATA(arr) + offset, + PyArray_BYTES(arr) + offset, PyArray_FLAGS(arr), (PyObject *)arr); if (*view == NULL) { @@ -2445,8 +2448,8 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, /* advance curr_dim for non-fancy indices */ else if (indices[i].type == HAS_ELLIPSIS) { - curr_dim += indices[i].value; - result_dim += indices[i].value; + curr_dim += (int)indices[i].value; + result_dim += (int)indices[i].value; } else if (indices[i].type != HAS_NEWAXIS){ curr_dim += 1; @@ -2891,7 +2894,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, stride = extra_op_dtype->elsize; for (i=PyArray_NDIM(subspace) - 1; i >= 0; i--) { strides[mit->nd_fancy + strideperm[i].perm] = stride; - stride *= PyArray_DIM(subspace, strideperm[i].perm); + stride *= PyArray_DIM(subspace, (int)strideperm[i].perm); } /* diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 10c22ae5a..b9d79029e 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3989,8 +3989,11 @@ test_interrupt(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * -array_shares_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +array_shares_memory_impl(PyObject *args, PyObject *kwds, Py_ssize_t default_max_work, + int raise_exceptions) { + PyObject * self_obj = NULL; + PyObject * other_obj = NULL; PyArrayObject * self = NULL; PyArrayObject * other = NULL; PyObject *max_work_obj = NULL; @@ -3998,16 +4001,40 @@ array_shares_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwd mem_overlap_t result; static PyObject *too_hard_cls = NULL; - Py_ssize_t max_work = NPY_MAY_SHARE_EXACT; + Py_ssize_t max_work; NPY_BEGIN_THREADS_DEF; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O&|O", kwlist, - PyArray_Converter, &self, - PyArray_Converter, &other, - &max_work_obj)) { + max_work = default_max_work; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist, + &self_obj, &other_obj, &max_work_obj)) { return NULL; } + if (PyArray_Check(self_obj)) { + self = (PyArrayObject*)self_obj; + Py_INCREF(self); + } + else { + /* Use FromAny to enable checking overlap for objects exposing array + interfaces etc. */ + self = (PyArrayObject*)PyArray_FromAny(self_obj, NULL, 0, 0, 0, NULL); + if (self == NULL) { + goto fail; + } + } + + if (PyArray_Check(other_obj)) { + other = (PyArrayObject*)other_obj; + Py_INCREF(other); + } + else { + other = (PyArrayObject*)PyArray_FromAny(other_obj, NULL, 0, 0, 0, NULL); + if (other == NULL) { + goto fail; + } + } + if (max_work_obj == NULL || max_work_obj == Py_None) { /* noop */ } @@ -4043,17 +4070,29 @@ array_shares_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwd Py_RETURN_TRUE; } else if (result == MEM_OVERLAP_OVERFLOW) { - PyErr_SetString(PyExc_OverflowError, - "Integer overflow in computing overlap"); - return NULL; + if (raise_exceptions) { + PyErr_SetString(PyExc_OverflowError, + "Integer overflow in computing overlap"); + return NULL; + } + else { + /* Don't know, so say yes */ + Py_RETURN_TRUE; + } } else if (result == MEM_OVERLAP_TOO_HARD) { - npy_cache_import("numpy.core._internal", "TooHardError", - &too_hard_cls); - if (too_hard_cls) { - PyErr_SetString(too_hard_cls, "Exceeded max_work"); + if (raise_exceptions) { + npy_cache_import("numpy.core._internal", "TooHardError", + &too_hard_cls); + if (too_hard_cls) { + PyErr_SetString(too_hard_cls, "Exceeded max_work"); + } + return NULL; + } + else { + /* Don't know, so say yes */ + Py_RETURN_TRUE; } - return NULL; } else { /* Doesn't happen usually */ @@ -4069,6 +4108,20 @@ fail: } +static PyObject * +array_shares_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +{ + return array_shares_memory_impl(args, kwds, NPY_MAY_SHARE_EXACT, 1); +} + + +static PyObject * +array_may_share_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +{ + return array_shares_memory_impl(args, kwds, NPY_MAY_SHARE_BOUNDS, 0); +} + + static struct PyMethodDef array_module_methods[] = { {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, @@ -4178,6 +4231,9 @@ static struct PyMethodDef array_module_methods[] = { {"shares_memory", (PyCFunction)array_shares_memory, METH_VARARGS | METH_KEYWORDS, NULL}, + {"may_share_memory", + (PyCFunction)array_may_share_memory, + METH_VARARGS | METH_KEYWORDS, NULL}, /* Datetime-related functions */ {"datetime_data", (PyCFunction)array_datetime_data, diff --git a/numpy/core/src/private/mem_overlap.c b/numpy/core/src/private/mem_overlap.c index 3cab83497..b2b80b4e6 100644 --- a/numpy/core/src/private/mem_overlap.c +++ b/numpy/core/src/private/mem_overlap.c @@ -479,6 +479,7 @@ NPY_VISIBILITY_HIDDEN mem_overlap_t solve_diophantine(unsigned int n, diophantine_term_t *E, npy_int64 b, Py_ssize_t max_work, int require_ub_nontrivial, npy_int64 *x) { + mem_overlap_t res; unsigned int j; for (j = 0; j < n; ++j) { @@ -535,15 +536,27 @@ solve_diophantine(unsigned int n, diophantine_term_t *E, npy_int64 b, return MEM_OVERLAP_NO; } else { - diophantine_term_t Ep[n]; - npy_int64 Epsilon[n], Gamma[n]; Py_ssize_t count = 0; + diophantine_term_t *Ep = NULL; + npy_int64 *Epsilon = NULL, *Gamma = NULL; - if (diophantine_precompute(n, E, Ep, Gamma, Epsilon)) { - return MEM_OVERLAP_OVERFLOW; + Ep = malloc(n * sizeof(diophantine_term_t)); + Epsilon = malloc(n * sizeof(npy_int64)); + Gamma = malloc(n * sizeof(npy_int64)); + if (Ep == NULL || Epsilon == NULL || Gamma == NULL) { + res = MEM_OVERLAP_ERROR; + } + else if (diophantine_precompute(n, E, Ep, Gamma, Epsilon)) { + res = MEM_OVERLAP_OVERFLOW; + } + else { + res = diophantine_dfs(n, n-1, E, Ep, Gamma, Epsilon, b, max_work, + require_ub_nontrivial, x, &count); } - return diophantine_dfs(n, n-1, E, Ep, Gamma, Epsilon, b, max_work, - require_ub_nontrivial, x, &count); + free(Ep); + free(Gamma); + free(Epsilon); + return res; } } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index 854c1e17a..aff6180c7 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -1444,6 +1444,8 @@ pairwise_sum_@TYPE@(@dtype@ *a, npy_uintp n, npy_intp stride) r[7] = @trf@(a[7 * stride]); for (i = 8; i < n - (n % 8); i += 8) { + /* small blocksizes seems to mess with hardware prefetch */ + NPY_PREFETCH(&a[(i + 512 / sizeof(a[0])) * stride], 0, 3); r[0] += @trf@(a[(i + 0) * stride]); r[1] += @trf@(a[(i + 1) * stride]); r[2] += @trf@(a[(i + 2) * stride]); @@ -2190,6 +2192,8 @@ pairwise_sum_@TYPE@(@ftype@ *rr, @ftype@ * ri, @ftype@ * a, npy_uintp n, r[7] = a[6 * stride + 1]; for (i = 8; i < n - (n % 8); i += 8) { + /* small blocksizes seems to mess with hardware prefetch */ + NPY_PREFETCH(&a[(i + 512 / sizeof(a[0])) * stride], 0, 3); r[0] += a[(i + 0) * stride]; r[1] += a[(i + 0) * stride + 1]; r[2] += a[(i + 2) * stride]; diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index e2542195f..8f7e55d91 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -375,7 +375,7 @@ class TestBooleanIndexShapeMismatchDeprecation(): arr.__getitem__, (slice(None), index)) -class TestFullDefaultDtype: +class TestFullDefaultDtype(object): """np.full defaults to float when dtype is not set. In the future, it will use the fill value's dtype. """ @@ -386,5 +386,19 @@ class TestFullDefaultDtype: assert_no_warnings(np.full, 1, 1, float) +class TestNonCContiguousViewDeprecation(_DeprecationTestCase): + """View of non-C-contiguous arrays deprecated in 1.11.0. + + The deprecation will not be raised for arrays that are both C and F + contiguous, as C contiguous is dominant. There are more such arrays + with relaxed stride checking than without so the deprecation is not + as visible with relaxed stride checking in force. + """ + + def test_fortran_contiguous(self): + self.assert_deprecated(np.ones((2,2)).T.view, args=(np.complex,)) + self.assert_deprecated(np.ones((2,2)).T.view, args=(np.int8,)) + + if __name__ == "__main__": run_module_suite() diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 29f2ee7bd..6d898eaa1 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -408,6 +408,10 @@ class TestMetadata(TestCase): d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))]) self.assertEqual(d['a'].metadata, {'datum': 1}) + def base_metadata_copied(self): + d = np.dtype((np.void, np.dtype('i4,i4', metadata={'datum': 1}))) + assert_equal(d.metadata, {'datum': 1}) + class TestString(TestCase): def test_complex_dtype_str(self): dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index 728cc675d..8d39fa4c0 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -482,5 +482,33 @@ def test_internal_overlap_fuzz(): no_overlap += 1 +def test_non_ndarray_inputs(): + # Regression check for gh-5604 + + class MyArray(object): + def __init__(self, data): + self.data = data + + @property + def __array_interface__(self): + return self.data.__array_interface__ + + class MyArray2(object): + def __init__(self, data): + self.data = data + + def __array__(self): + return self.data + + for cls in [MyArray, MyArray2]: + x = np.arange(5) + + assert_(np.may_share_memory(cls(x[::2]), x[1::2])) + assert_(not np.shares_memory(cls(x[::2]), x[1::2])) + + assert_(np.shares_memory(cls(x[1::3]), x[::2])) + assert_(np.may_share_memory(cls(x[1::3]), x[::2])) + + if __name__ == "__main__": run_module_suite() diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 15dd9302c..693847273 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -17,7 +17,6 @@ from decimal import Decimal import numpy as np -from nose import SkipTest from numpy.compat import asbytes, getexception, strchar, unicode, sixu from test_print import in_foreign_locale from numpy.core.multiarray_tests import ( @@ -29,7 +28,7 @@ from numpy.testing import ( TestCase, run_module_suite, assert_, assert_raises, assert_equal, assert_almost_equal, assert_array_equal, assert_array_almost_equal, assert_allclose, - assert_array_less, runstring, dec + assert_array_less, runstring, dec, SkipTest ) # Need to test an object that does not fully implement math interface diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index f5c22392a..43dad42f1 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -30,7 +30,15 @@ class TestResize(TestCase): def test_zeroresize(self): A = np.array([[1, 2], [3, 4]]) Ar = np.resize(A, (0,)) - assert_equal(Ar, np.array([])) + assert_array_equal(Ar, np.array([])) + assert_equal(A.dtype, Ar.dtype) + + def test_reshape_from_zero(self): + # See also gh-6740 + A = np.zeros(0, dtype=[('a', np.float32, 1)]) + Ar = np.resize(A, (2, 1)) + assert_array_equal(Ar, np.zeros((2, 1), Ar.dtype)) + assert_equal(A.dtype, Ar.dtype) class TestNonarrayArgs(TestCase): diff --git a/numpy/core/tests/test_print.py b/numpy/core/tests/test_print.py index f595cbe44..6234b641e 100644 --- a/numpy/core/tests/test_print.py +++ b/numpy/core/tests/test_print.py @@ -6,7 +6,7 @@ import nose import numpy as np from numpy.testing import ( - run_module_suite, assert_, assert_equal + run_module_suite, assert_, assert_equal, SkipTest ) @@ -207,7 +207,7 @@ def test_scalar_format(): def in_foreign_locale(func): """ Swap LC_NUMERIC locale to one in which the decimal point is ',' and not '.' - If not possible, raise nose.SkipTest + If not possible, raise SkipTest """ if sys.platform == 'win32': @@ -225,8 +225,8 @@ def in_foreign_locale(func): except locale.Error: pass else: - raise nose.SkipTest("Skipping locale test, because " - "French locale not found") + raise SkipTest("Skipping locale test, because " + "French locale not found") return func(*args, **kwargs) finally: locale.setlocale(locale.LC_NUMERIC, locale=curloc) diff --git a/numpy/core/tests/test_scalarinherit.py b/numpy/core/tests/test_scalarinherit.py index a2ca3e458..d8fd0acc3 100644 --- a/numpy/core/tests/test_scalarinherit.py +++ b/numpy/core/tests/test_scalarinherit.py @@ -2,6 +2,7 @@ """ Test printing of scalar types. """ +from __future__ import division, absolute_import, print_function import numpy as np from numpy.testing import TestCase, run_module_suite diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 541ad974b..2ba988b87 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1926,5 +1926,15 @@ def test_complex_nan_comparisons(): assert_equal(x == y, False, err_msg="%r == %r" % (x, y)) +def test_rint_big_int(): + # np.rint bug for large integer values on Windows 32-bit and MKL + # https://github.com/numpy/numpy/issues/6685 + val = 4607998452777363968 + # This is exactly representable in floating point + assert_equal(val, int(float(val))) + # Rint should not change the value + assert_equal(val, np.rint(val)) + + if __name__ == "__main__": run_module_suite() diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index 75d864c5a..345e60f26 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -18,6 +18,20 @@ try: except ImportError: from dummy_threading import local as tlocal +# stores temporary directory of each thread to only create one per thread +_tdata = tlocal() + +# store all created temporary directories so they can be deleted on exit +_tmpdirs = [] +def clean_up_temporary_directory(): + for d in _tmpdirs: + try: + shutil.rmtree(d) + except OSError: + pass + +atexit.register(clean_up_temporary_directory) + try: set except NameError: @@ -283,26 +297,13 @@ def gpaths(paths, local_path='', include_non_existing=True): paths = (paths,) return _fix_paths(paths, local_path, include_non_existing) - -def clean_up_temporary_directory(): - tdata = tlocal() - _temporary_directory = getattr(tdata, 'tempdir', None) - if not _temporary_directory: - return - try: - shutil.rmtree(_temporary_directory) - except OSError: - pass - _temporary_directory = None - def make_temp_file(suffix='', prefix='', text=True): - tdata = tlocal() - if not hasattr(tdata, 'tempdir'): - tdata.tempdir = tempfile.mkdtemp() - atexit.register(clean_up_temporary_directory) + if not hasattr(_tdata, 'tempdir'): + _tdata.tempdir = tempfile.mkdtemp() + _tmpdirs.append(_tdata.tempdir) fid, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, - dir=tdata.tempdir, + dir=_tdata.tempdir, text=text) fo = os.fdopen(fid, 'w') return fo, name diff --git a/numpy/distutils/msvc9compiler.py b/numpy/distutils/msvc9compiler.py index 636165bd5..c53f45531 100644 --- a/numpy/distutils/msvc9compiler.py +++ b/numpy/distutils/msvc9compiler.py @@ -1,3 +1,5 @@ +from __future__ import division, absolute_import, print_function + import os import distutils.msvc9compiler from distutils.msvc9compiler import * diff --git a/numpy/distutils/msvccompiler.py b/numpy/distutils/msvccompiler.py index 4c3658d5c..78a386d5d 100644 --- a/numpy/distutils/msvccompiler.py +++ b/numpy/distutils/msvccompiler.py @@ -1,3 +1,5 @@ +from __future__ import division, absolute_import, print_function + import os import distutils.msvccompiler from distutils.msvccompiler import * diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index a0c6f44f7..94436243e 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -1678,9 +1678,64 @@ class blas_info(system_info): info = self.check_libs(lib_dirs, blas_libs, []) if info is None: return - info['language'] = 'f77' # XXX: is it generally true? + if platform.system() == 'Windows': + # The check for windows is needed because has_cblas uses the + # same compiler that was used to compile Python and msvc is + # often not installed when mingw is being used. This rough + # treatment is not desirable, but windows is tricky. + info['language'] = 'f77' # XXX: is it generally true? + else: + lib = self.has_cblas(info) + if lib is not None: + info['language'] = 'c' + info['libraries'] = [lib] + info['define_macros'] = [('HAVE_CBLAS', None)] self.set_info(**info) + def has_cblas(self, info): + # primitive cblas check by looking for the header and trying to link + # cblas or blas + res = False + c = distutils.ccompiler.new_compiler() + tmpdir = tempfile.mkdtemp() + s = """#include <cblas.h> + int main(int argc, const char *argv[]) + { + double a[4] = {1,2,3,4}; + double b[4] = {5,6,7,8}; + return cblas_ddot(4, a, 1, b, 1) > 10; + }""" + src = os.path.join(tmpdir, 'source.c') + try: + with open(src, 'wt') as f: + f.write(s) + + try: + # check we can compile (find headers) + obj = c.compile([src], output_dir=tmpdir, + include_dirs=self.get_include_dirs()) + + # check we can link (find library) + # some systems have separate cblas and blas libs. First + # check for cblas lib, and if not present check for blas lib. + try: + c.link_executable(obj, os.path.join(tmpdir, "a.out"), + libraries=["cblas"], + library_dirs=info['library_dirs'], + extra_postargs=info.get('extra_link_args', [])) + res = "cblas" + except distutils.ccompiler.LinkError: + c.link_executable(obj, os.path.join(tmpdir, "a.out"), + libraries=["blas"], + library_dirs=info['library_dirs'], + extra_postargs=info.get('extra_link_args', [])) + res = "blas" + except distutils.ccompiler.CompileError: + res = None + finally: + shutil.rmtree(tmpdir) + return res + class openblas_info(blas_info): section = 'openblas' diff --git a/numpy/f2py/__main__.py b/numpy/f2py/__main__.py index 8f6d25619..cb8f261c1 100644 --- a/numpy/f2py/__main__.py +++ b/numpy/f2py/__main__.py @@ -1,4 +1,6 @@ # See http://cens.ioc.ee/projects/f2py2e/ +from __future__ import division, print_function + import os import sys for mode in ["g3-numpy", "2e-numeric", "2e-numarray", "2e-numpy"]: diff --git a/numpy/f2py/tests/test_array_from_pyobj.py b/numpy/f2py/tests/test_array_from_pyobj.py index 9551c099e..48bb7c0f4 100644 --- a/numpy/f2py/tests/test_array_from_pyobj.py +++ b/numpy/f2py/tests/test_array_from_pyobj.py @@ -5,13 +5,11 @@ import os import sys import copy -import nose - from numpy import ( array, alltrue, ndarray, zeros, dtype, intp, clongdouble ) from numpy.testing import ( - run_module_suite, assert_, assert_equal + run_module_suite, assert_, assert_equal, SkipTest ) from numpy.core.multiarray import typeinfo import util @@ -28,7 +26,7 @@ def setup(): # Check compiler availability first if not util.has_c_compiler(): - raise nose.SkipTest("No C compiler available") + raise SkipTest("No C compiler available") if wrap is None: config_code = """ diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index 5b4e072e7..8d06d9680 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -17,10 +17,9 @@ import textwrap import re import random -import nose - from numpy.compat import asbytes, asstr import numpy.f2py +from numpy.testing import SkipTest try: from hashlib import md5 @@ -334,7 +333,7 @@ class F2PyTest(object): # Check compiler availability first if not has_c_compiler(): - raise nose.SkipTest("No C compiler available") + raise SkipTest("No C compiler available") codes = [] if self.sources: @@ -350,9 +349,9 @@ class F2PyTest(object): elif fn.endswith('.f90'): needs_f90 = True if needs_f77 and not has_f77_compiler(): - raise nose.SkipTest("No Fortran 77 compiler available") + raise SkipTest("No Fortran 77 compiler available") if needs_f90 and not has_f90_compiler(): - raise nose.SkipTest("No Fortran 90 compiler available") + raise SkipTest("No Fortran 90 compiler available") # Build the module if self.code is not None: diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index fef69dff3..9261dba22 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -336,8 +336,12 @@ def histogram(a, bins=10, range=None, normed=False, weights=None, if (range is not None): mn, mx = range if (mn > mx): - raise AttributeError( + raise ValueError( 'max must be larger than min in range parameter.') + if not np.all(np.isfinite([mn, mx])): + raise ValueError( + 'range parameter must be finite.') + if isinstance(bins, basestring): bins = _hist_optim_numbins_estimator(a, bins) @@ -422,7 +426,7 @@ def histogram(a, bins=10, range=None, normed=False, weights=None, else: bins = asarray(bins) if (np.diff(bins) < 0).any(): - raise AttributeError( + raise ValueError( 'bins must increase monotonically.') # Initialize empty histogram @@ -533,7 +537,7 @@ def histogramdd(sample, bins=10, range=None, normed=False, weights=None): try: M = len(bins) if M != D: - raise AttributeError( + raise ValueError( 'The dimension of bins must be equal to the dimension of the ' ' sample x.') except TypeError: @@ -551,6 +555,9 @@ def histogramdd(sample, bins=10, range=None, normed=False, weights=None): smin = atleast_1d(array(sample.min(0), float)) smax = atleast_1d(array(sample.max(0), float)) else: + if not np.all(np.isfinite(range)): + raise ValueError( + 'range parameter must be finite.') smin = zeros(D) smax = zeros(D) for i in arange(D): diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index de9376300..2f677438b 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -427,7 +427,8 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): default) just the coefficients are returned, when True diagnostic information from the singular value decomposition is also returned. w : array_like, shape (M,), optional - weights to apply to the y-coordinates of the sample points. + Weights to apply to the y-coordinates of the sample points. For + gaussian uncertainties, use 1/sigma (not 1/sigma**2). cov : bool, optional Return the estimate and the covariance matrix of the estimate If full is True, then cov is not returned. diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index 615cf88f4..ffbe56721 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -799,6 +799,9 @@ def tile(A, reps): Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as (1, 1, 2, 2). + Note : Although tile may be used for broadcasting, it is strongly + recommended to use numpy's broadcasting operations and functions. + Parameters ---------- A : array_like @@ -814,6 +817,7 @@ def tile(A, reps): See Also -------- repeat : Repeat elements of an array. + broadcast_to : Broadcast an array to a new shape Examples -------- @@ -837,6 +841,12 @@ def tile(A, reps): [1, 2], [3, 4]]) + >>> c = np.array([1,2,3,4]) + >>> np.tile(c,(4,1)) + array([[1, 2, 3, 4], + [1, 2, 3, 4], + [1, 2, 3, 4], + [1, 2, 3, 4]]) """ try: tup = tuple(reps) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 090f71f67..f4bece352 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -7,7 +7,7 @@ from shutil import rmtree from numpy.compat import asbytes from numpy.testing import ( - run_module_suite, TestCase, assert_ + run_module_suite, TestCase, assert_, SkipTest ) import numpy.lib._datasource as datasource @@ -137,8 +137,7 @@ class TestDataSourceOpen(TestCase): import gzip except ImportError: # We don't have the gzip capabilities to test. - import nose - raise nose.SkipTest + raise SkipTest # Test datasource's internal file_opener for Gzip files. filepath = os.path.join(self.tmpdir, 'foobar.txt.gz') fp = gzip.open(filepath, 'w') @@ -154,8 +153,7 @@ class TestDataSourceOpen(TestCase): import bz2 except ImportError: # We don't have the bz2 capabilities to test. - import nose - raise nose.SkipTest + raise SkipTest # Test datasource's internal file_opener for BZip2 files. filepath = os.path.join(self.tmpdir, 'foobar.txt.bz2') fp = bz2.BZ2File(filepath, 'w') diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 4f8a65148..1bf65fa61 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -287,7 +287,7 @@ import numpy as np from numpy.compat import asbytes, asbytes_nested, sixu from numpy.testing import ( run_module_suite, assert_, assert_array_equal, assert_raises, raises, - dec + dec, SkipTest ) from numpy.lib import format @@ -812,7 +812,6 @@ def test_bad_header(): def test_large_file_support(): - from nose import SkipTest if (sys.platform == 'win32' or sys.platform == 'cygwin'): raise SkipTest("Unknown if Windows has sparse filesystems") # try creating a large sparse file diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index cc53c2b8e..a5ac78e33 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1267,6 +1267,13 @@ class TestHistogram(TestCase): assert_array_equal(a, np.array([0])) assert_array_equal(b, np.array([0, 1])) + def test_finite_range(self): + # Normal ranges should be fine + vals = np.linspace(0.0, 1.0, num=100) + histogram(vals, range=[0.25,0.75]) + assert_raises(ValueError, histogram, vals, range=[np.nan,0.75]) + assert_raises(ValueError, histogram, vals, range=[0.25,np.inf]) + class TestHistogramOptimBinNums(TestCase): """ @@ -1489,6 +1496,16 @@ class TestHistogramdd(TestCase): assert_(hist[0] == 0.0) assert_(hist[1] == 0.0) + def test_finite_range(self): + vals = np.random.random((100,3)) + histogramdd(vals, range=[[0.0,1.0],[0.25,0.75],[0.25,0.5]]) + assert_raises(ValueError, histogramdd, vals, + range=[[0.0,1.0],[0.25,0.75],[0.25,np.inf]]) + assert_raises(ValueError, histogramdd, vals, + range=[[0.0,1.0],[np.nan,0.75],[0.25,0.5]]) + + + class TestUnique(TestCase): @@ -1957,10 +1974,42 @@ class TestInterp(TestCase): assert_almost_equal(np.interp(x0, x, y), x0) def test_right_left_behavior(self): - assert_equal(interp([-1, 0, 1], [0], [1]), [1, 1, 1]) - assert_equal(interp([-1, 0, 1], [0], [1], left=0), [0, 1, 1]) - assert_equal(interp([-1, 0, 1], [0], [1], right=0), [1, 1, 0]) - assert_equal(interp([-1, 0, 1], [0], [1], left=0, right=0), [0, 1, 0]) + # Needs range of sizes to test different code paths. + # size ==1 is special cased, 1 < size < 5 is linear search, and + # size >= 5 goes through local search and possibly binary search. + for size in range(1, 10): + xp = np.arange(size, dtype=np.double) + yp = np.ones(size, dtype=np.double) + incpts = np.array([-1, 0, size - 1, size], dtype=np.double) + decpts = incpts[::-1] + + incres = interp(incpts, xp, yp) + decres = interp(decpts, xp, yp) + inctgt = np.array([1, 1, 1, 1], dtype=np.float) + dectgt = inctgt[::-1] + assert_equal(incres, inctgt) + assert_equal(decres, dectgt) + + incres = interp(incpts, xp, yp, left=0) + decres = interp(decpts, xp, yp, left=0) + inctgt = np.array([0, 1, 1, 1], dtype=np.float) + dectgt = inctgt[::-1] + assert_equal(incres, inctgt) + assert_equal(decres, dectgt) + + incres = interp(incpts, xp, yp, right=2) + decres = interp(decpts, xp, yp, right=2) + inctgt = np.array([1, 1, 1, 2], dtype=np.float) + dectgt = inctgt[::-1] + assert_equal(incres, inctgt) + assert_equal(decres, dectgt) + + incres = interp(incpts, xp, yp, left=0, right=2) + decres = interp(decpts, xp, yp, left=0, right=2) + inctgt = np.array([0, 1, 1, 2], dtype=np.float) + dectgt = inctgt[::-1] + assert_equal(incres, inctgt) + assert_equal(decres, dectgt) def test_scalar_interpolation_point(self): x = np.linspace(0, 1, 5) diff --git a/numpy/lib/tests/test_packbits.py b/numpy/lib/tests/test_packbits.py index 186e8960d..0de084ef9 100644 --- a/numpy/lib/tests/test_packbits.py +++ b/numpy/lib/tests/test_packbits.py @@ -1,5 +1,6 @@ -import numpy as np +from __future__ import division, absolute_import, print_function +import numpy as np from numpy.testing import assert_array_equal, assert_equal, assert_raises diff --git a/numpy/linalg/tests/test_deprecations.py b/numpy/linalg/tests/test_deprecations.py index 13d244199..9b6fe343f 100644 --- a/numpy/linalg/tests/test_deprecations.py +++ b/numpy/linalg/tests/test_deprecations.py @@ -1,6 +1,8 @@ """Test deprecation and future warnings. """ +from __future__ import division, absolute_import, print_function + import numpy as np from numpy.testing import assert_warns, run_module_suite diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 7c577d86f..afa098f12 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -17,7 +17,7 @@ from numpy.linalg.linalg import _multi_dot_matrix_chain_order from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_almost_equal, assert_allclose, run_module_suite, - dec + dec, SkipTest ) @@ -1215,7 +1215,6 @@ def test_xerbla_override(): # Check that our xerbla has been successfully linked in. If it is not, # the default xerbla routine is called, which prints a message to stdout # and may, or may not, abort the process depending on the LAPACK package. - from nose import SkipTest XERBLA_OK = 255 diff --git a/numpy/ma/core.py b/numpy/ma/core.py index b9f7da092..25e542cd6 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -1248,7 +1248,7 @@ def _recursive_make_descr(datatype, newtype=bool_): # Is this some kind of composite a la (np.float,2) elif datatype.subdtype: mdescr = list(datatype.subdtype) - mdescr[0] = newtype + mdescr[0] = _recursive_make_descr(datatype.subdtype[0], newtype) return tuple(mdescr) else: return newtype @@ -2684,6 +2684,8 @@ class MaskedArray(ndarray): _defaultmask = nomask _defaulthardmask = False _baseclass = ndarray + # Maximum number of elements per axis used when printing an array. + _print_width = 100 def __new__(cls, data=None, mask=nomask, dtype=None, copy=False, subok=True, ndmin=0, fill_value=None, @@ -2756,13 +2758,19 @@ class MaskedArray(ndarray): _data._sharedmask = True else: # Case 2. : With a mask in input. - # Read the mask with the current mdtype - try: - mask = np.array(mask, copy=copy, dtype=mdtype) - # Or assume it's a sequence of bool/int - except TypeError: - mask = np.array([tuple([m] * len(mdtype)) for m in mask], - dtype=mdtype) + # If mask is boolean, create an array of True or False + if mask is True and mdtype == MaskType: + mask = np.ones(_data.shape, dtype=mdtype) + elif mask is False and mdtype == MaskType: + mask = np.zeros(_data.shape, dtype=mdtype) + else: + # Read the mask with the current mdtype + try: + mask = np.array(mask, copy=copy, dtype=mdtype) + # Or assume it's a sequence of bool/int + except TypeError: + mask = np.array([tuple([m] * len(mdtype)) for m in mask], + dtype=mdtype) # Make sure the mask and the data have the same shape if mask.shape != _data.shape: (nd, nm) = (_data.size, mask.size) @@ -3695,7 +3703,7 @@ class MaskedArray(ndarray): if m is nomask: res = self._data else: - if m.shape == (): + if m.shape == () and m.itemsize==len(m.dtype): if m.dtype.names: m = m.view((bool, len(m.dtype))) if m.any(): @@ -3710,8 +3718,19 @@ class MaskedArray(ndarray): # convert to object array to make filled work names = self.dtype.names if names is None: - res = self._data.astype("O") - res.view(ndarray)[m] = f + data = self._data + mask = m + # For big arrays, to avoid a costly conversion to the + # object dtype, extract the corners before the conversion. + for axis in range(self.ndim): + if data.shape[axis] > self._print_width: + ind = self._print_width // 2 + arr = np.split(data, (ind, -ind), axis=axis) + data = np.concatenate((arr[0], arr[2]), axis=axis) + arr = np.split(mask, (ind, -ind), axis=axis) + mask = np.concatenate((arr[0], arr[2]), axis=axis) + res = data.astype("O") + res.view(ndarray)[mask] = f else: rdtype = _recursive_make_descr(self.dtype, "O") res = self._data.astype(rdtype) @@ -4690,7 +4709,7 @@ class MaskedArray(ndarray): See Also -------- numpy.ma.dot : equivalent function - + """ return dot(self, b, out=out, strict=strict) @@ -5850,6 +5869,18 @@ class mvoid(MaskedArray): """ m = self._mask + if isinstance(m[indx], ndarray): + # Can happen when indx is a multi-dimensional field: + # A = ma.masked_array(data=[([0,1],)], mask=[([True, + # False],)], dtype=[("A", ">i2", (2,))]) + # x = A[0]; y = x["A"]; then y.mask["A"].size==2 + # and we can not say masked/unmasked. + # The result is no longer mvoid! + # See also issue #6724. + return masked_array( + data=self._data[indx], mask=m[indx], + fill_value=self._fill_value[indx], + hard_mask=self._hardmask) if m is not nomask and m[indx]: return masked return self._data[indx] diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index e5fdfddb1..cecdedf26 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -191,6 +191,15 @@ class TestMaskedArray(TestCase): dma_3 = MaskedArray(dma_1, mask=[1, 0, 0, 0] * 6) fail_if_equal(dma_3.mask, dma_1.mask) + x = array([1, 2, 3], mask=True) + assert_equal(x._mask, [True, True, True]) + x = array([1, 2, 3], mask=False) + assert_equal(x._mask, [False, False, False]) + y = array([1, 2, 3], mask=x._mask, copy=False) + assert_(np.may_share_memory(x.mask, y.mask)) + y = array([1, 2, 3], mask=x._mask, copy=True) + assert_(not np.may_share_memory(x.mask, y.mask)) + def test_creation_with_list_of_maskedarrays(self): # Tests creaating a masked array from alist of masked arrays. x = array(np.arange(5), mask=[1, 0, 0, 0, 0]) @@ -599,6 +608,13 @@ class TestMaskedArray(TestCase): control = np.array([(0, 1), (2, 0)], dtype=a['B'].dtype) assert_equal(test, control) + # test if mask gets set correctly (see #6760) + Z = numpy.ma.zeros(2, numpy.dtype([("A", "(2,2)i1,(2,2)i1", (2,2))])) + assert_equal(Z.data.dtype, numpy.dtype([('A', [('f0', 'i1', (2, 2)), + ('f1', 'i1', (2, 2))], (2, 2))])) + assert_equal(Z.mask.dtype, numpy.dtype([('A', [('f0', '?', (2, 2)), + ('f1', '?', (2, 2))], (2, 2))])) + def test_filled_w_f_order(self): # Test filled w/ F-contiguous array a = array(np.array([(0, 1, 2), (4, 5, 6)], order='F'), @@ -625,6 +641,18 @@ class TestMaskedArray(TestCase): control = "[(--, (2, --)) (4, (--, 6.0))]" assert_equal(str(test), control) + # Test 0-d array with multi-dimensional dtype + t_2d0 = masked_array(data = (0, [[0.0, 0.0, 0.0], + [0.0, 0.0, 0.0]], + 0.0), + mask = (False, [[True, False, True], + [False, False, True]], + False), + dtype = "int, (2,3)float, float") + control = "(0, [[--, 0.0, --], [0.0, 0.0, --]], 0.0)" + assert_equal(str(t_2d0), control) + + def test_flatten_structured_array(self): # Test flatten_structured_array on arrays # On ndarray @@ -691,6 +719,14 @@ class TestMaskedArray(TestCase): self.assertTrue(f['a'] is masked) assert_equal(f[1], 4) + # exotic dtype + A = masked_array(data=[([0,1],)], + mask=[([True, False],)], + dtype=[("A", ">i2", (2,))]) + assert_equal(A[0]["A"], A["A"][0]) + assert_equal(A[0]["A"], masked_array(data=[0, 1], + mask=[True, False], dtype=">i2")) + def test_mvoid_iter(self): # Test iteration on __getitem__ ndtype = [('a', int), ('b', int)] diff --git a/numpy/random/mtrand/mtrand.pyx b/numpy/random/mtrand/mtrand.pyx index f8ae8d71b..080591e5e 100644 --- a/numpy/random/mtrand/mtrand.pyx +++ b/numpy/random/mtrand/mtrand.pyx @@ -127,6 +127,7 @@ cdef extern from "initarray.h": # Initialize numpy import_array() +cimport cython import numpy as np import operator import warnings @@ -4484,7 +4485,7 @@ cdef class RandomState: mnarr = <ndarray>multin mnix = <long*>PyArray_DATA(mnarr) sz = PyArray_SIZE(mnarr) - with self.lock, nogil: + with self.lock, nogil, cython.cdivision(True): i = 0 while i < sz: Sum = 1.0 diff --git a/numpy/testing/decorators.py b/numpy/testing/decorators.py index 56962b93c..df3d297ff 100644 --- a/numpy/testing/decorators.py +++ b/numpy/testing/decorators.py @@ -18,6 +18,7 @@ from __future__ import division, absolute_import, print_function import warnings import collections +from .utils import SkipTest def slow(t): """ @@ -141,14 +142,14 @@ def skipif(skip_condition, msg=None): def skipper_func(*args, **kwargs): """Skipper for normal test functions.""" if skip_val(): - raise nose.SkipTest(get_msg(f, msg)) + raise SkipTest(get_msg(f, msg)) else: return f(*args, **kwargs) def skipper_gen(*args, **kwargs): """Skipper for test generators.""" if skip_val(): - raise nose.SkipTest(get_msg(f, msg)) + raise SkipTest(get_msg(f, msg)) else: for x in f(*args, **kwargs): yield x @@ -166,7 +167,7 @@ def skipif(skip_condition, msg=None): def knownfailureif(fail_condition, msg=None): """ - Make function raise KnownFailureTest exception if given condition is true. + Make function raise KnownFailureException exception if given condition is true. If the condition is a callable, it is used at runtime to dynamically make the decision. This is useful for tests that may require costly @@ -178,15 +179,15 @@ def knownfailureif(fail_condition, msg=None): Flag to determine whether to mark the decorated test as a known failure (if True) or not (if False). msg : str, optional - Message to give on raising a KnownFailureTest exception. + Message to give on raising a KnownFailureException exception. Default is None. Returns ------- decorator : function - Decorator, which, when applied to a function, causes SkipTest - to be raised when `skip_condition` is True, and the function - to be called normally otherwise. + Decorator, which, when applied to a function, causes + KnownFailureException to be raised when `fail_condition` is True, + and the function to be called normally otherwise. Notes ----- @@ -207,11 +208,11 @@ def knownfailureif(fail_condition, msg=None): # Local import to avoid a hard nose dependency and only incur the # import time overhead at actual test-time. import nose - from .noseclasses import KnownFailureTest + from .noseclasses import KnownFailureException def knownfailer(*args, **kwargs): if fail_val(): - raise KnownFailureTest(msg) + raise KnownFailureException(msg) else: return f(*args, **kwargs) return nose.tools.make_decorator(f)(knownfailer) diff --git a/numpy/testing/noseclasses.py b/numpy/testing/noseclasses.py index e6cc10179..197e20bac 100644 --- a/numpy/testing/noseclasses.py +++ b/numpy/testing/noseclasses.py @@ -8,6 +8,7 @@ from __future__ import division, absolute_import, print_function import os import doctest +import inspect import nose from nose.plugins import doctests as npd @@ -16,7 +17,8 @@ from nose.plugins.base import Plugin from nose.util import src import numpy from .nosetester import get_package_name -import inspect +from .utils import KnownFailureException, KnownFailureTest + # Some of the classes in this module begin with 'Numpy' to clearly distinguish # them from the plethora of very similar names from nose/unittest/doctest @@ -298,19 +300,14 @@ class Unplugger(object): if p.name != self.to_unplug] -class KnownFailureTest(Exception): - '''Raise this exception to mark a test as a known failing test.''' - pass - - -class KnownFailure(ErrorClassPlugin): +class KnownFailurePlugin(ErrorClassPlugin): '''Plugin that installs a KNOWNFAIL error class for the - KnownFailureClass exception. When KnownFailureTest is raised, + KnownFailureClass exception. When KnownFailure is raised, the exception will be logged in the knownfail attribute of the result, 'K' or 'KNOWNFAIL' (verbose) will be output, and the exception will not be counted as an error or failure.''' enabled = True - knownfail = ErrorClass(KnownFailureTest, + knownfail = ErrorClass(KnownFailureException, label='KNOWNFAIL', isfailure=False) @@ -318,7 +315,7 @@ class KnownFailure(ErrorClassPlugin): env_opt = 'NOSE_WITHOUT_KNOWNFAIL' parser.add_option('--no-knownfail', action='store_true', dest='noKnownFail', default=env.get(env_opt, False), - help='Disable special handling of KnownFailureTest ' + help='Disable special handling of KnownFailure ' 'exceptions') def configure(self, options, conf): @@ -329,6 +326,8 @@ class KnownFailure(ErrorClassPlugin): if disable: self.enabled = False +KnownFailure = KnownFailurePlugin # backwards compat + # Class allows us to save the results of the tests in runTests - see runTests # method docstring for details diff --git a/numpy/testing/nosetester.py b/numpy/testing/nosetester.py index c9c6d10f0..551e630ec 100644 --- a/numpy/testing/nosetester.py +++ b/numpy/testing/nosetester.py @@ -121,8 +121,8 @@ def run_module_suite(file_to_run=None, argv=None): argv = argv + [file_to_run] nose = import_nose() - from .noseclasses import KnownFailure - nose.run(argv=argv, addplugins=[KnownFailure()]) + from .noseclasses import KnownFailurePlugin + nose.run(argv=argv, addplugins=[KnownFailurePlugin()]) class NoseTester(object): @@ -301,8 +301,8 @@ class NoseTester(object): '--cover-tests', '--cover-erase'] # construct list of plugins import nose.plugins.builtin - from .noseclasses import KnownFailure, Unplugger - plugins = [KnownFailure()] + from .noseclasses import KnownFailurePlugin, Unplugger + plugins = [KnownFailurePlugin()] plugins += [p() for p in nose.plugins.builtin.plugins] # add doctesting if required doctest_argv = '--with-doctest' in argv diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index f8a5be672..7dbb5a828 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -1,7 +1,7 @@ from __future__ import division, absolute_import, print_function -from numpy.testing import dec, assert_, assert_raises, run_module_suite -from numpy.testing.noseclasses import KnownFailureTest +from numpy.testing import (dec, assert_, assert_raises, run_module_suite, + SkipTest, KnownFailureException) import nose def test_slow(): @@ -40,7 +40,7 @@ def test_skip_functions_hardcoded(): f1('a') except DidntSkipException: raise Exception('Failed to skip') - except nose.SkipTest: + except SkipTest: pass @dec.skipif(False) @@ -51,7 +51,7 @@ def test_skip_functions_hardcoded(): f2('a') except DidntSkipException: pass - except nose.SkipTest: + except SkipTest: raise Exception('Skipped when not expected to') @@ -68,7 +68,7 @@ def test_skip_functions_callable(): f1('a') except DidntSkipException: raise Exception('Failed to skip') - except nose.SkipTest: + except SkipTest: pass @dec.skipif(skip_tester) @@ -80,7 +80,7 @@ def test_skip_functions_callable(): f2('a') except DidntSkipException: pass - except nose.SkipTest: + except SkipTest: raise Exception('Skipped when not expected to') @@ -93,7 +93,7 @@ def test_skip_generators_hardcoded(): try: for j in g1(10): pass - except KnownFailureTest: + except KnownFailureException: pass else: raise Exception('Failed to mark as known failure') @@ -107,7 +107,7 @@ def test_skip_generators_hardcoded(): try: for j in g2(10): pass - except KnownFailureTest: + except KnownFailureException: raise Exception('Marked incorretly as known failure') except DidntSkipException: pass @@ -126,7 +126,7 @@ def test_skip_generators_callable(): skip_flag = 'skip me!' for j in g1(10): pass - except KnownFailureTest: + except KnownFailureException: pass else: raise Exception('Failed to mark as known failure') @@ -141,7 +141,7 @@ def test_skip_generators_callable(): skip_flag = 'do not skip' for j in g2(10): pass - except KnownFailureTest: + except KnownFailureException: raise Exception('Marked incorretly as known failure') except DidntSkipException: pass diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index a31fce4af..13aeffe02 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -9,7 +9,8 @@ from numpy.testing import ( assert_array_almost_equal, build_err_msg, raises, assert_raises, assert_warns, assert_no_warnings, assert_allclose, assert_approx_equal, assert_array_almost_equal_nulp, assert_array_max_ulp, - clear_and_catch_warnings, run_module_suite + clear_and_catch_warnings, run_module_suite, + assert_string_equal ) import unittest @@ -715,6 +716,22 @@ class TestULP(unittest.TestCase): lambda: assert_array_max_ulp(nan, nzero, maxulp=maxulp)) +class TestStringEqual(unittest.TestCase): + def test_simple(self): + assert_string_equal("hello", "hello") + assert_string_equal("hello\nmultiline", "hello\nmultiline") + + try: + assert_string_equal("foo\nbar", "hello\nbar") + except AssertionError as exc: + assert_equal(str(exc), "Differences in strings:\n- foo\n+ hello") + else: + raise AssertionError("exception not raised") + + self.assertRaises(AssertionError, + lambda: assert_string_equal("foo", "hello")) + + def assert_warn_len_equal(mod, n_in_context): mod_warns = mod.__warningregistry__ # Python 3.4 appears to clear any pre-existing warnings of the same type, diff --git a/numpy/testing/utils.py b/numpy/testing/utils.py index 099b75bdf..00f7ce4d1 100644 --- a/numpy/testing/utils.py +++ b/numpy/testing/utils.py @@ -13,6 +13,7 @@ from functools import partial import shutil import contextlib from tempfile import mkdtemp + from .nosetester import import_nose from numpy.core import float32, empty, arange, array_repr, ndarray @@ -28,11 +29,27 @@ __all__ = ['assert_equal', 'assert_almost_equal', 'assert_approx_equal', 'raises', 'rand', 'rundocs', 'runstring', 'verbose', 'measure', 'assert_', 'assert_array_almost_equal_nulp', 'assert_raises_regex', 'assert_array_max_ulp', 'assert_warns', 'assert_no_warnings', - 'assert_allclose', 'IgnoreException', 'clear_and_catch_warnings'] + 'assert_allclose', 'IgnoreException', 'clear_and_catch_warnings', + 'SkipTest', 'KnownFailureException'] -verbose = 0 +class KnownFailureException(Exception): + '''Raise this exception to mark a test as a known failing test.''' + pass + +KnownFailureTest = KnownFailureException # backwards compat + +# nose.SkipTest is unittest.case.SkipTest +# import it into the namespace, so that it's available as np.testing.SkipTest +try: + from unittest.case import SkipTest +except ImportError: + # on py2.6 unittest.case is not available. Ask nose for a replacement. + SkipTest = import_nose().SkipTest + + +verbose = 0 def assert_(val, msg=''): """ @@ -1018,11 +1035,12 @@ def assert_string_equal(actual, desired): if not d2.startswith('+ '): raise AssertionError(repr(d2)) l.append(d2) - d3 = diff.pop(0) - if d3.startswith('? '): - l.append(d3) - else: - diff.insert(0, d3) + if diff: + d3 = diff.pop(0) + if d3.startswith('? '): + l.append(d3) + else: + diff.insert(0, d3) if re.match(r'\A'+d2[2:]+r'\Z', d1[2:]): continue diff_list.extend(l) diff --git a/numpy/tests/test_scripts.py b/numpy/tests/test_scripts.py index c7bb125b3..552383d77 100644 --- a/numpy/tests/test_scripts.py +++ b/numpy/tests/test_scripts.py @@ -12,6 +12,7 @@ import numpy as np from numpy.compat.py3k import basestring, asbytes from nose.tools import assert_equal from numpy.testing.decorators import skipif +from numpy.testing import assert_ skipif_inplace = skipif(isfile(pathjoin(dirname(np.__file__), '..', 'setup.py'))) @@ -63,7 +64,18 @@ def test_f2py(): if sys.platform == 'win32': f2py_cmd = r"%s\Scripts\f2py.py" % dirname(sys.executable) code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) + assert_equal(stdout.strip(), asbytes('2')) else: - f2py_cmd = 'f2py' + basename(sys.executable)[6:] - code, stdout, stderr = run_command([f2py_cmd, '-v']) - assert_equal(stdout.strip(), asbytes('2')) + # unclear what f2py cmd was installed as, check plain (f2py) and + # current python version specific one (f2py3.4) + f2py_cmds = ['f2py', 'f2py' + basename(sys.executable)[6:]] + success = False + for f2py_cmd in f2py_cmds: + try: + code, stdout, stderr = run_command([f2py_cmd, '-v']) + assert_equal(stdout.strip(), asbytes('2')) + success = True + break + except FileNotFoundError: + pass + assert_(success, "wasn't able to find f2py or %s on commandline" % f2py_cmds[1]) |