diff options
26 files changed, 460 insertions, 150 deletions
diff --git a/doc/release/2.0.0-notes.rst b/doc/release/2.0.0-notes.rst index 3b61afc31..7a9d43fa1 100644 --- a/doc/release/2.0.0-notes.rst +++ b/doc/release/2.0.0-notes.rst @@ -8,6 +8,41 @@ Highlights ========== +Compatibility notes +=================== + +In a future version of numpy, the functions np.diag, np.diagonal, and +the diagonal method of ndarrays will return a view onto the original +array, instead of producing a copy as they do now. This makes a +difference if you write to the array returned by any of these +functions. To facilitate this transition, numpy 1.7 produces a +DeprecationWarning if it detects that you may be attempting to write +to such an array. See the documentation for np.diagonal for details. + +The default casting rule for UFunc out= parameters has been changed from +'unsafe' to 'same_kind'. Most usages which violate the 'same_kind' +rule are likely bugs, so this change may expose previously undetected +errors in projects that depend on NumPy. + +Full-array boolean indexing used to allow boolean arrays with a size +non-broadcastable to the array size. Now it forces this to be broadcastable. +Since this affects some legacy code, this change will require discussion +during alpha or early beta testing, and a decision to either keep the +stricter behavior, or add in a hack to allow the previous behavior to +work. + +Attempting to write to a read-only array (one with +``arr.flags.writeable`` set to ``False``) used to raise either a +RuntimeError, ValueError, or TypeError inconsistently, depending on +which code path was taken. It now consistently raises a ValueError. + +The <ufunc>.reduce functions evaluate some reductions in a different +order than in previous versions of NumPy, generally providing higher +performance. Because of the nature of floating-point arithmetic, this +may subtly change some results, just as linking NumPy to a different +BLAS implementations such as MKL can. + + New features ============ @@ -148,36 +183,20 @@ New argument to searchsorted The function searchsorted now accepts a 'sorter' argument that is a permuation array that sorts the array to search. +C API +----- + +New function ``PyArray_RequireWriteable`` provides a consistent +interface for checking array writeability -- any C code which works +with arrays whose WRITEABLE flag is not known to be True a priori, +should make sure to call this function before writing. + Changes ======= General ------- -The default casting rule for UFunc out= parameters has been changed from -'unsafe' to 'same_kind'. Most usages which violate the 'same_kind' -rule are likely bugs, so this change may expose previously undetected -errors in projects that depend on NumPy. - -Full-array boolean indexing used to allow boolean arrays with a size -non-broadcastable to the array size. Now it forces this to be broadcastable. -Since this affects some legacy code, this change will require discussion -during alpha or early beta testing, and a decision to either keep the -stricter behavior, or add in a hack to allow the previous behavior to -work. - -The functions np.diag, np.diagonal, and <ndarray>.diagonal now return a -view into the original array instead of making a copy. This makes these -functions more consistent with NumPy's general approach of taking views -where possible, and performs much faster as well. This has the -potential to break code that assumes a copy is made instead of a view. - -The <ufunc>.reduce functions evaluates some reductions in a different -order than in previous versions of NumPy, generally providing higher -performance. Because of the nature of floating-point arithmetic, this -may subtly change some results, just as linking NumPy to a different -BLAS implementations such as MKL can. - The function np.concatenate tries to match the layout of its input arrays. Previously, the layout did not follow any particular reason, and depended in an undesirable way on the particular axis chosen for diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 6330d4ae6..3599f47c7 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -9,5 +9,5 @@ # Version 6 (NumPy 1.6) added new iterator, half float and casting functions, # PyArray_CountNonzero, PyArray_NewLikeArray and PyArray_MatrixProduct2. 0x00000006 = e61d5dc51fa1c6459328266e215d6987 -# Version 7 (NumPy 1.7) added API for NA, improved datetime64. -0x00000007 = eb54c77ff4149bab310324cd7c0cb176 +# Version 7 (NumPy 1.7) added API for NA, improved datetime64, misc utilities. +0x00000007 = 280023b3ecfc2ad0326874917f6f16f9 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index ca89c28ec..15b868e23 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -344,6 +344,8 @@ multiarray_funcs_api = { 'NpyNA_FromDTypeAndPayload': 304, 'PyArray_AllowNAConverter': 305, 'PyArray_OutputAllowNAConverter': 306, + 'PyArray_FailUnlessWriteable': 307, + 'PyArray_SetUpdateIfCopyBase': 308, } ufunc_types_api = { diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index dae109a98..2b108bcf9 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -932,7 +932,27 @@ def diagonal(a, offset=0, axis1=0, axis2=1): removing `axis1` and `axis2` and appending an index to the right equal to the size of the resulting diagonals. - As of NumPy 1.7, this function always returns a view into `a`. + In versions of NumPy prior to 1.7, this function always returned a new, + independent array containing a copy of the values in the diagonal. + + In NumPy 1.7, it continues to return a copy of the diagonal, but depending + on this fact is deprecated. Writing to the resulting array continues to + work as it used to, but a DeprecationWarning will be issued. + + In NumPy 1.8, it will switch to returning a read-only view on the original + array. Attempting to write to the resulting array will produce an error. + + In NumPy 1.9, it will still return a view, but this view will no longer be + marked read-only. Writing to the returned array will alter your original + array as well. + + If you don't write to the array returned by this function, then you can + just ignore all of the above. + + If you depend on the current behavior, then we suggest copying the + returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of + just ``np.diagonal(a)``. This will work with both past and future versions + of NumPy. Parameters ---------- diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index db5257761..087dd2398 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -917,6 +917,10 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); */ #define NPY_ARRAY_ALLOWNA 0x8000 +/* + * NOTE: there are also internal flags defined in multiarray/arrayobject.h, + * which start at bit 31 and work down. + */ #define NPY_ARRAY_BEHAVED (NPY_ARRAY_ALIGNED | \ NPY_ARRAY_WRITEABLE) @@ -1550,7 +1554,7 @@ static NPY_INLINE PyObject * PyArray_GETITEM(const PyArrayObject *arr, const char *itemptr) { return ((PyArrayObject_fields *)arr)->descr->f->getitem( - (void *)itemptr, (PyArrayObject *)arr); + (void *)itemptr, (PyArrayObject *)arr); } static NPY_INLINE int diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 9a51229a1..aa7d2c29b 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -1956,9 +1956,8 @@ def identity(n, dtype=None, maskna=False): [ 0., 0., 1.]]) """ - a = zeros((n,n), dtype=dtype, maskna=maskna) - a.diagonal()[...] = 1 - return a + from numpy import eye + return eye(n, dtype=dtype, maskna=maskna) def allclose(a, b, rtol=1.e-5, atol=1.e-8): """ diff --git a/numpy/core/src/multiarray/array_assign_array.c b/numpy/core/src/multiarray/array_assign_array.c index 03231a995..a9f3a35fe 100644 --- a/numpy/core/src/multiarray/array_assign_array.c +++ b/numpy/core/src/multiarray/array_assign_array.c @@ -449,10 +449,7 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, return 0; } - /* Check that 'dst' is writeable */ - if (!PyArray_ISWRITEABLE(dst)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot assign to a read-only array"); + if (PyArray_FailUnlessWriteable(dst, "assignment destination") < 0) { goto fail; } diff --git a/numpy/core/src/multiarray/array_assign_scalar.c b/numpy/core/src/multiarray/array_assign_scalar.c index bf0676c96..60658ed4d 100644 --- a/numpy/core/src/multiarray/array_assign_scalar.c +++ b/numpy/core/src/multiarray/array_assign_scalar.c @@ -336,10 +336,7 @@ PyArray_AssignRawScalar(PyArrayObject *dst, int allocated_src_data = 0, dst_has_maskna = PyArray_HASMASKNA(dst); npy_longlong scalarbuffer[4]; - /* Check that 'dst' is writeable */ - if (!PyArray_ISWRITEABLE(dst)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot assign a scalar value to a read-only array"); + if (PyArray_FailUnlessWriteable(dst, "assignment destination") < 0) { return -1; } diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 11b04989f..4f0181b4a 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -67,6 +67,57 @@ PyArray_Size(PyObject *op) } /*NUMPY_API + * + * Precondition: 'arr' is a copy of 'base' (though possibly with different + * strides, ordering, etc.). This function sets the UPDATEIFCOPY flag and the + * ->base pointer on 'arr', so that when 'arr' is destructed, it will copy any + * changes back to 'base'. + * + * Steals a reference to 'base'. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +PyArray_SetUpdateIfCopyBase(PyArrayObject *arr, PyArrayObject *base) +{ + if (base == NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot UPDATEIFCOPY to NULL array"); + return -1; + } + if (PyArray_BASE(arr) != NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot set array with existing base to UPDATEIFCOPY"); + goto fail; + } + if (PyArray_FailUnlessWriteable(base, "UPDATEIFCOPY base") < 0) { + goto fail; + } + + /* + * Any writes to 'arr' will magicaly turn into writes to 'base', so we + * should warn if necessary. + */ + if (PyArray_FLAGS(base) & NPY_ARRAY_WARN_ON_WRITE) { + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_WARN_ON_WRITE); + } + + /* + * Unlike PyArray_SetBaseObject, we do not compress the chain of base + * references. + */ + ((PyArrayObject_fields *)arr)->base = (PyObject *)base; + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(base, NPY_ARRAY_WRITEABLE); + + return 0; + + fail: + Py_DECREF(base); + return -1; +} + +/*NUMPY_API * Sets the 'base' attribute of the array. This steals a reference * to 'obj'. * @@ -104,6 +155,11 @@ PyArray_SetBaseObject(PyArrayObject *arr, PyObject *obj) PyArrayObject *obj_arr = (PyArrayObject *)obj; PyObject *tmp; + /* Propagate WARN_ON_WRITE through views. */ + if (PyArray_FLAGS(obj_arr) & NPY_ARRAY_WARN_ON_WRITE) { + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_WARN_ON_WRITE); + } + /* If this array owns its own data, stop collapsing */ if (PyArray_CHKFLAGS(obj_arr, NPY_ARRAY_OWNDATA)) { break; @@ -704,6 +760,58 @@ PyArray_CompareString(char *s1, char *s2, size_t len) } +/* Call this from contexts where an array might be written to, but we have no + * way to tell. (E.g., when converting to a read-write buffer.) + */ +NPY_NO_EXPORT int +array_might_be_written(PyArrayObject *obj) +{ + const char *msg = + "Numpy has detected that you (may be) writing to an array returned\n" + "by numpy.diagonal. This code will likely break in the next numpy\n" + "release -- see numpy.diagonal docs for details. The quick fix is\n" + "to make an explicit copy (e.g., do arr.diagonal().copy())."; + if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { + if (DEPRECATE(msg) < 0) { + return -1; + } + /* Only warn once per array */ + while (1) { + PyArray_CLEARFLAGS(obj, NPY_ARRAY_WARN_ON_WRITE); + if (!PyArray_BASE(obj) || !PyArray_Check(PyArray_BASE(obj))) { + break; + } + obj = (PyArrayObject *)PyArray_BASE(obj); + } + } + return 0; +} + +/*NUMPY_API + * + * This function does nothing if obj is writeable, and raises an exception + * (and returns -1) if obj is not writeable. It may also do other + * house-keeping, such as issuing warnings on arrays which are transitioning + * to become views. Always call this function at some point before writing to + * an array. + * + * 'name' is a name for the array, used to give better error + * messages. Something like "assignment destination", "output array", or even + * just "array". + */ +NPY_NO_EXPORT int +PyArray_FailUnlessWriteable(PyArrayObject *obj, const char *name) +{ + if (!PyArray_ISWRITEABLE(obj)) { + PyErr_Format(PyExc_ValueError, "%s is read-only", name); + return -1; + } + if (array_might_be_written(obj) < 0) { + return -1; + } + return 0; +} + /* This also handles possibly mis-aligned data */ /* Compare s1 and s2 which are not necessarily NULL-terminated. s1 is of length len1 diff --git a/numpy/core/src/multiarray/arrayobject.h b/numpy/core/src/multiarray/arrayobject.h index ec3361435..9b74944ff 100644 --- a/numpy/core/src/multiarray/arrayobject.h +++ b/numpy/core/src/multiarray/arrayobject.h @@ -12,4 +12,18 @@ _strings_richcompare(PyArrayObject *self, PyArrayObject *other, int cmp_op, NPY_NO_EXPORT PyObject * array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op); +NPY_NO_EXPORT int +array_might_be_written(PyArrayObject *obj); + +/* + * This flag is used to mark arrays which we would like to, in the future, + * turn into views. It causes a warning to be issued on the first attempt to + * write to the array (but the write is allowed to succeed). + * + * This flag is for internal use only, and may be removed in a future release, + * which is why the #define is not exposed to user code. Currently it is set + * on arrays returned by ndarray.diagonal. + */ +static const int NPY_ARRAY_WARN_ON_WRITE = (1 << 31); + #endif diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 6dcc7d8e9..3ad8d1a75 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -18,6 +18,7 @@ #include "usertypes.h" #include "_datetime.h" #include "na_object.h" +#include "arrayobject.h" #include "numpyos.h" @@ -649,6 +650,9 @@ VOID_getitem(char *ip, PyArrayObject *ap) * current item a view of it */ if (PyArray_ISWRITEABLE(ap)) { + if (array_might_be_written(ap) < 0) { + return NULL; + } u = (PyArrayObject *)PyBuffer_FromReadWriteMemory(ip, itemsize); } else { diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index ac94d270e..a1ad23911 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -13,6 +13,7 @@ #include "buffer.h" #include "numpyos.h" +#include "arrayobject.h" /************************************************************************* **************** Implement Buffer Protocol **************************** @@ -57,14 +58,10 @@ array_getreadbuf(PyArrayObject *self, Py_ssize_t segment, void **ptrptr) static Py_ssize_t array_getwritebuf(PyArrayObject *self, Py_ssize_t segment, void **ptrptr) { - if (PyArray_CHKFLAGS(self, NPY_ARRAY_WRITEABLE)) { - return array_getreadbuf(self, segment, (void **) ptrptr); - } - else { - PyErr_SetString(PyExc_ValueError, "array cannot be " - "accessed as a writeable buffer"); + if (PyArray_FailUnlessWriteable(self, "buffer source array") < 0) { return -1; } + return array_getreadbuf(self, segment, (void **) ptrptr); } static Py_ssize_t @@ -632,10 +629,21 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous"); goto fail; } - if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE && - !PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_ValueError, "ndarray is not writeable"); - goto fail; + if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) { + if (PyArray_FailUnlessWriteable(self, "buffer source array") < 0) { + goto fail; + } + } + /* + * If a read-only buffer is requested on a read-write array, we return a + * read-write buffer, which is dubious behavior. But that's why this call + * is guarded by PyArray_ISWRITEABLE rather than (flags & + * PyBUF_WRITEABLE). + */ + if (PyArray_ISWRITEABLE(self)) { + if (array_might_be_written(self) < 0) { + goto fail; + } } if (view == NULL) { diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index c8548f894..eac9ba437 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1310,7 +1310,9 @@ _array_from_buffer_3118(PyObject *obj, PyObject **out) r = PyArray_NewFromDescr(&PyArray_Type, descr, nd, shape, strides, view->buf, flags, NULL); - ((PyArrayObject_fields *)r)->base = memoryview; + if (PyArray_SetBaseObject((PyArrayObject *)r, memoryview) < 0) { + goto fail; + } PyArray_UpdateFlags((PyArrayObject *)r, NPY_ARRAY_UPDATE_ALL); *out = r; @@ -1348,9 +1350,8 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, /* If op is an array */ if (PyArray_Check(op)) { - if (writeable && !PyArray_ISWRITEABLE((PyArrayObject *)op)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to array"); + if (writeable + && PyArray_FailUnlessWriteable((PyArrayObject *)op, "array") < 0) { return -1; } Py_INCREF(op); @@ -1419,9 +1420,8 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, /* If op supports the PEP 3118 buffer interface */ if (!PyBytes_Check(op) && !PyUnicode_Check(op) && _array_from_buffer_3118(op, (PyObject **)out_arr) == 0) { - if (writeable && !PyArray_ISWRITEABLE(*out_arr)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to PEP 3118 buffer"); + if (writeable + && PyArray_FailUnlessWriteable(*out_arr, "PEP 3118 buffer") < 0) { Py_DECREF(*out_arr); return -1; } @@ -1440,9 +1440,9 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, } } if (tmp != Py_NotImplemented) { - if (writeable && !PyArray_ISWRITEABLE((PyArrayObject *)tmp)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to array interface object"); + if (writeable + && PyArray_FailUnlessWriteable((PyArrayObject *)tmp, + "array interface object") < 0) { Py_DECREF(tmp); return -1; } @@ -1462,9 +1462,9 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, if (!writeable) { tmp = PyArray_FromArrayAttr(op, requested_dtype, context); if (tmp != Py_NotImplemented) { - if (writeable && !PyArray_ISWRITEABLE((PyArrayObject *)tmp)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to array interface object"); + if (writeable + && PyArray_FailUnlessWriteable((PyArrayObject *)tmp, + "array interface object") < 0) { Py_DECREF(tmp); return -1; } @@ -2032,13 +2032,6 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) order = NPY_CORDER; } - if ((flags & NPY_ARRAY_UPDATEIFCOPY) && - (!PyArray_ISWRITEABLE(arr))) { - Py_DECREF(newtype); - PyErr_SetString(PyExc_ValueError, - "cannot copy back to a read-only array"); - return NULL; - } if ((flags & NPY_ARRAY_ENSUREARRAY)) { subok = 0; } @@ -2078,14 +2071,11 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) } if (flags & NPY_ARRAY_UPDATEIFCOPY) { - /* - * Don't use PyArray_SetBaseObject, because that compresses - * the chain of bases. - */ Py_INCREF(arr); - ((PyArrayObject_fields *)ret)->base = (PyObject *)arr; - PyArray_ENABLEFLAGS(ret, NPY_ARRAY_UPDATEIFCOPY); - PyArray_CLEARFLAGS(arr, NPY_ARRAY_WRITEABLE); + if (PyArray_SetUpdateIfCopyBase(ret, arr) < 0) { + Py_DECREF(ret); + return NULL; + } } } /* @@ -2599,9 +2589,7 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) NPY_BEGIN_THREADS_DEF; - if (!PyArray_ISWRITEABLE(dst)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to array"); + if (PyArray_FailUnlessWriteable(dst, "destination array") < 0) { return -1; } diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index a414f1f37..a8470e326 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -16,6 +16,7 @@ #include "scalartypes.h" #include "descriptor.h" #include "getset.h" +#include "arrayobject.h" /******************* array attribute get and set routines ******************/ @@ -260,6 +261,10 @@ array_interface_get(PyArrayObject *self) return NULL; } + if (array_might_be_written(self) < 0) { + return NULL; + } + /* dataptr */ obj = array_dataptr_get(self); PyDict_SetItemString(dict, "data", obj); @@ -355,9 +360,12 @@ array_data_set(PyArrayObject *self, PyObject *op) PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); } Py_DECREF(PyArray_BASE(self)); + ((PyArrayObject_fields *)self)->base = NULL; } Py_INCREF(op); - ((PyArrayObject_fields *)self)->base = op; + if (PyArray_SetBaseObject(self, op) < 0) { + return -1; + } ((PyArrayObject_fields *)self)->data = buf; ((PyArrayObject_fields *)self)->flags = NPY_ARRAY_CARRAY; if (!writeable) { @@ -554,6 +562,11 @@ array_struct_get(PyArrayObject *self) PyArrayInterface *inter; PyObject *ret; + if (PyArray_ISWRITEABLE(self)) { + if (array_might_be_written(self) < 0) { + return NULL; + } + } inter = (PyArrayInterface *)PyArray_malloc(sizeof(PyArrayInterface)); if (inter==NULL) { return PyErr_NoMemory(); diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 77faa1eb2..010aa8aa5 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -1114,9 +1114,7 @@ PyArray_Sort(PyArrayObject *op, int axis, NPY_SORTKIND which) PyErr_Format(PyExc_ValueError, "axis(=%d) out of bounds", axis); return -1; } - if (!PyArray_ISWRITEABLE(op)) { - PyErr_SetString(PyExc_RuntimeError, - "attempted sort on unwriteable array."); + if (PyArray_FailUnlessWriteable(op, "sort array") < 0) { return -1; } @@ -1843,7 +1841,11 @@ PyArray_SearchSorted(PyArrayObject *op1, PyObject *op2, /*NUMPY_API * Diagonal * - * As of NumPy 1.7, this function always returns a view into 'self'. + * In NumPy versions prior to 1.7, this function always returned a copy of + * the diagonal array. In 1.7, the code has been updated to compute a view + * onto 'self', but it still copies this array before returning, as well as + * setting the internal WARN_ON_WRITE flag. In a future version, it will + * simply return a view onto self. */ NPY_NO_EXPORT PyObject * PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) @@ -1859,6 +1861,7 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) PyArrayObject *ret; PyArray_Descr *dtype; npy_intp ret_shape[NPY_MAXDIMS], ret_strides[NPY_MAXDIMS]; + PyObject *copy; if (ndim < 2) { PyErr_SetString(PyExc_ValueError, @@ -1989,7 +1992,13 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) fret->flags |= NPY_ARRAY_MASKNA; } - return (PyObject *)ret; + /* For backwards compatibility, during the deprecation period: */ + copy = PyArray_NewCopy(ret, NPY_KEEPORDER); + if (!copy) { + return NULL; + } + PyArray_ENABLEFLAGS((PyArrayObject *)copy, NPY_ARRAY_WARN_ON_WRITE); + return copy; } /*NUMPY_API diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 262be6443..e8064fefd 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1260,14 +1260,11 @@ iter_array(PyArrayIterObject *it, PyObject *NPY_UNUSED(op)) Py_DECREF(ret); return NULL; } - /* - * Don't use PyArray_SetBaseObject, because that compresses - * the chain of bases. - */ Py_INCREF(it->ao); - ((PyArrayObject_fields *)ret)->base = (PyObject *)it->ao; - PyArray_ENABLEFLAGS(ret, NPY_ARRAY_UPDATEIFCOPY); - PyArray_CLEARFLAGS(it->ao, NPY_ARRAY_WRITEABLE); + if (PyArray_SetUpdateIfCopyBase(ret, it->ao) < 0) { + Py_DECREF(ret); + return NULL; + } } return ret; diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index ec746481c..f260cf8d9 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -176,9 +176,7 @@ array_ass_big_item(PyArrayObject *self, npy_intp i, PyObject *v) return -1; } - if (!PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_RuntimeError, - "array is not writeable"); + if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { return -1; } @@ -1502,9 +1500,7 @@ array_ass_sub(PyArrayObject *self, PyObject *ind, PyObject *op) "cannot delete array elements"); return -1; } - if (!PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_RuntimeError, - "array is not writeable"); + if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { return -1; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 8129a7021..6d1a52edc 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -532,10 +532,7 @@ PyArray_Byteswap(PyArrayObject *self, npy_bool inplace) copyswapn = PyArray_DESCR(self)->f->copyswapn; if (inplace) { - if (!PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_RuntimeError, - "Cannot byte-swap in-place on a " \ - "read-only array"); + if (PyArray_FailUnlessWriteable(self, "array to be byte-swapped") < 0) { return NULL; } size = PyArray_SIZE(self); @@ -748,9 +745,7 @@ array_setscalar(PyArrayObject *self, PyObject *args) "itemset must have at least one argument"); return NULL; } - if (!PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_RuntimeError, - "array is not writeable"); + if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { return NULL; } @@ -1697,6 +1692,7 @@ array_setstate(PyArrayObject *self, PyObject *args) PyArray_CLEARFLAGS(self, NPY_ARRAY_OWNDATA); } Py_XDECREF(PyArray_BASE(self)); + fa->base = NULL; PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); @@ -1769,7 +1765,9 @@ array_setstate(PyArrayObject *self, PyObject *args) Py_DECREF(rawdata); } else { - fa->base = rawdata; + if (PyArray_SetBaseObject(self, rawdata) < 0) { + return NULL; + } } } else { diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index f1d0c5c38..180c55063 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1098,11 +1098,9 @@ npyiter_prepare_one_operand(PyArrayObject **op, if (PyArray_Check(*op)) { npy_uint32 tmp; - if (((*op_itflags) & NPY_OP_ITFLAG_WRITE) && - (!PyArray_CHKFLAGS(*op, NPY_ARRAY_WRITEABLE))) { - PyErr_SetString(PyExc_ValueError, - "Operand was a non-writeable array, but " - "flagged as writeable"); + if ((*op_itflags) & NPY_OP_ITFLAG_WRITE + && PyArray_FailUnlessWriteable(*op, "operand array with iterator " + "write flag set") < 0) { return 0; } if (!(flags & NPY_ITER_ZEROSIZE_OK) && PyArray_SIZE(*op) == 0) { @@ -2984,15 +2982,11 @@ npyiter_allocate_arrays(NpyIter *iter, } /* If the data will be written to, set UPDATEIFCOPY */ if (op_itflags[iop] & NPY_OP_ITFLAG_WRITE) { - /* - * Don't use PyArray_SetBaseObject, because that compresses - * the chain of bases. - */ Py_INCREF(op[iop]); - ((PyArrayObject_fields *)temp)->base = - (PyObject *)op[iop]; - PyArray_ENABLEFLAGS(temp, NPY_ARRAY_UPDATEIFCOPY); - PyArray_CLEARFLAGS(op[iop], NPY_ARRAY_WRITEABLE); + if (PyArray_SetUpdateIfCopyBase(temp, op[iop]) < 0) { + Py_DECREF(temp); + return 0; + } } Py_DECREF(op[iop]); diff --git a/numpy/core/src/multiarray/sequence.c b/numpy/core/src/multiarray/sequence.c index 004aa2d78..cb3b30b3a 100644 --- a/numpy/core/src/multiarray/sequence.c +++ b/numpy/core/src/multiarray/sequence.c @@ -119,9 +119,7 @@ array_ass_slice(PyArrayObject *self, Py_ssize_t ilow, "cannot delete array elements"); return -1; } - if (!PyArray_ISWRITEABLE(self)) { - PyErr_SetString(PyExc_RuntimeError, - "array is not writeable"); + if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { return -1; } tmp = (PyArrayObject *)array_slice(self, ilow, ihigh); diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 93f63038a..3b62e150f 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -795,9 +795,8 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, } /* If it's an array, can use it */ if (PyArray_Check(obj)) { - if (!PyArray_ISWRITEABLE((PyArrayObject *)obj)) { - PyErr_SetString(PyExc_ValueError, - "return array is not writeable"); + if (PyArray_FailUnlessWriteable((PyArrayObject *)obj, + "output array") < 0) { return -1; } Py_INCREF(obj); @@ -894,9 +893,9 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, } if (PyArray_Check(value)) { - if (!PyArray_ISWRITEABLE((PyArrayObject *)value)) { - PyErr_SetString(PyExc_ValueError, - "return array is not writeable"); + const char *name = "output array"; + PyArrayObject *value_arr = (PyArrayObject *)value; + if (PyArray_FailUnlessWriteable(value_arr, name) < 0) { goto fail; } Py_INCREF(value); diff --git a/numpy/core/tests/test_maskna.py b/numpy/core/tests/test_maskna.py index 4efd5abcf..c19cd70c2 100644 --- a/numpy/core/tests/test_maskna.py +++ b/numpy/core/tests/test_maskna.py @@ -1417,11 +1417,9 @@ def test_array_maskna_diagonal(): a.shape = (2,3) a[0,1] = np.NA - # Should produce a view into a res = a.diagonal() - assert_(res.base is a) assert_(res.flags.maskna) - assert_(not res.flags.ownmaskna) + assert_(res.flags.ownmaskna) assert_equal(res, [0, 4]) res = a.diagonal(-1) @@ -1593,6 +1591,8 @@ def test_array_maskna_linspace_logspace(): assert_(b.flags.maskna) +from numpy.testing import dec +@dec.knownfailureif(True, "eye is not implemented for maskna") def test_array_maskna_eye_identity(): # np.eye diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c00930f6d..056749c19 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -29,8 +29,8 @@ class TestFlags(TestCase): def test_writeable(self): mydict = locals() self.a.flags.writeable = False - self.assertRaises(RuntimeError, runstring, 'self.a[0] = 3', mydict) - self.assertRaises(RuntimeError, runstring, 'self.a[0:1].itemset(3)', mydict) + self.assertRaises(ValueError, runstring, 'self.a[0] = 3', mydict) + self.assertRaises(ValueError, runstring, 'self.a[0:1].itemset(3)', mydict) self.a.flags.writeable = True self.a[0] = 5 self.a[0] = 0 @@ -792,6 +792,148 @@ class TestMethods(TestCase): assert_equal(np.dot(a, b), a.dot(b)) assert_equal(np.dot(np.dot(a, b), c), a.dot(b).dot(c)) + def test_diagonal(self): + a = np.arange(12).reshape((3, 4)) + assert_equal(a.diagonal(), [0, 5, 10]) + assert_equal(a.diagonal(0), [0, 5, 10]) + assert_equal(a.diagonal(1), [1, 6, 11]) + assert_equal(a.diagonal(-1), [4, 9]) + + b = np.arange(8).reshape((2, 2, 2)) + assert_equal(b.diagonal(), [[0, 6], [1, 7]]) + assert_equal(b.diagonal(0), [[0, 6], [1, 7]]) + assert_equal(b.diagonal(1), [[2], [3]]) + assert_equal(b.diagonal(-1), [[4], [5]]) + assert_raises(ValueError, b.diagonal, axis1=0, axis2=0) + assert_equal(b.diagonal(0, 1, 2), [[0, 3], [4, 7]]) + assert_equal(b.diagonal(0, 0, 1), [[0, 6], [1, 7]]) + assert_equal(b.diagonal(offset=1, axis1=0, axis2=2), [[1], [3]]) + # Order of axis argument doesn't matter: + assert_equal(b.diagonal(0, 2, 1), [[0, 3], [4, 7]]) + + def test_diagonal_deprecation(self): + import warnings + from numpy.testing.utils import WarningManager + def collect_warning_types(f, *args, **kwargs): + ctx = WarningManager(record=True) + warning_log = ctx.__enter__() + warnings.simplefilter("always") + try: + f(*args, **kwargs) + finally: + ctx.__exit__() + return [w.category for w in warning_log] + a = np.arange(9).reshape(3, 3) + # All the different functions raise a warning, but not an error, and + # 'a' is not modified: + assert_equal(collect_warning_types(a.diagonal().__setitem__, 0, 10), + [DeprecationWarning]) + assert_equal(a, np.arange(9).reshape(3, 3)) + assert_equal(collect_warning_types(np.diagonal(a).__setitem__, 0, 10), + [DeprecationWarning]) + assert_equal(a, np.arange(9).reshape(3, 3)) + assert_equal(collect_warning_types(np.diag(a).__setitem__, 0, 10), + [DeprecationWarning]) + assert_equal(a, np.arange(9).reshape(3, 3)) + # Views also warn + d = np.diagonal(a) + d_view = d.view() + assert_equal(collect_warning_types(d_view.__setitem__, 0, 10), + [DeprecationWarning]) + # But the write goes through: + assert_equal(d[0], 10) + # Only one warning per call to diagonal, though (even if there are + # multiple views involved): + assert_equal(collect_warning_types(d.__setitem__, 0, 10), + []) + + # Other ways of accessing the data also warn: + # .data goes via the C buffer API, gives a read-write + # buffer/memoryview. We don't warn until tp_getwritebuf is actually + # called, which is not until the buffer is written to. + have_memoryview = (hasattr(__builtins__, "memoryview") + or "memoryview" in __builtins__) + def get_data_and_write(getter): + buf_or_memoryview = getter(a.diagonal()) + if (have_memoryview and isinstance(buf_or_memoryview, memoryview)): + buf_or_memoryview[0] = np.array(1) + else: + buf_or_memoryview[0] = "x" + assert_equal(collect_warning_types(get_data_and_write, + lambda d: d.data), + [DeprecationWarning]) + if hasattr(np, "getbuffer"): + assert_equal(collect_warning_types(get_data_and_write, + np.getbuffer), + [DeprecationWarning]) + # PEP 3118: + if have_memoryview: + assert_equal(collect_warning_types(get_data_and_write, memoryview), + [DeprecationWarning]) + # Void dtypes can give us a read-write buffer, but only in Python 2: + import sys + if sys.version_info[0] < 3: + aV = np.empty((3, 3), dtype="V10") + assert_equal(collect_warning_types(aV.diagonal().item, 0), + [DeprecationWarning]) + # XX it seems that direct indexing of a void object returns a void + # scalar, which ignores not just WARN_ON_WRITE but even WRITEABLE. + # i.e. in this: + # a = np.empty(10, dtype="V10") + # a.flags.writeable = False + # buf = a[0].item() + # 'buf' ends up as a writeable buffer. I guess no-one actually + # uses void types like this though... + # __array_interface also lets a data pointer get away from us + log = collect_warning_types(getattr, a.diagonal(), + "__array_interface__") + assert_equal(log, [DeprecationWarning]) + # ctypeslib goes via __array_interface__: + try: + # may not exist in python 2.4: + import ctypes + except ImportError: + pass + else: + log = collect_warning_types(np.ctypeslib.as_ctypes, a.diagonal()) + assert_equal(log, [DeprecationWarning]) + # __array_struct__ + log = collect_warning_types(getattr, a.diagonal(), "__array_struct__") + assert_equal(log, [DeprecationWarning]) + + # Make sure that our recommendation to silence the warning by copying + # the array actually works: + diag_copy = a.diagonal().copy() + assert_equal(collect_warning_types(diag_copy.__setitem__, 0, 10), + []) + # There might be people who get a spurious warning because they are + # extracting a buffer, but then use that buffer in a read-only + # fashion. And they might get cranky at having to create a superfluous + # copy just to work around this spurious warning. A reasonable + # solution would be for them to mark their usage as read-only, and + # thus safe for both past and future PyArray_Diagonal + # semantics. So let's make sure that setting the diagonal array to + # non-writeable will suppress these warnings: + ro_diag = a.diagonal() + ro_diag.flags.writeable = False + assert_equal(collect_warning_types(getattr, ro_diag, "data"), []) + # __array_interface__ has no way to communicate read-onlyness -- + # effectively all __array_interface__ arrays are assumed to be + # writeable :-( + # ro_diag = a.diagonal() + # ro_diag.flags.writeable = False + # assert_equal(collect_warning_types(getattr, ro_diag, + # "__array_interface__"), []) + if hasattr(__builtins__, "memoryview"): + ro_diag = a.diagonal() + ro_diag.flags.writeable = False + assert_equal(collect_warning_types(memoryview, ro_diag), []) + ro_diag = a.diagonal() + ro_diag.flags.writeable = False + assert_equal(collect_warning_types(getattr, ro_diag, + "__array_struct__"), []) + + def test_ravel(self): a = np.array([[0,1],[2,3]]) assert_equal(a.ravel(), [0,1,2,3]) diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 3d18a9b98..7a316ac3a 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -865,7 +865,6 @@ def test_iter_array_cast(): i = None assert_equal(a[2,1,1], -12.5) - # Unsafe cast 'f4' -> 'i4' a = np.arange(6, dtype='i4')[::-2] i = nditer(a, [], [['writeonly','updateifcopy']], diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index 58d8250a1..eab8f867a 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -210,19 +210,28 @@ def eye(N, M=None, k=0, dtype=float, maskna=False): if M is None: M = N m = zeros((N, M), dtype=dtype, maskna=maskna) - diagonal(m, k)[...] = 1 + if k >= M: + return m + if k >= 0: + i = k + else: + i = (-k) * M + m[:M-k].flat[i::M+1] = 1 return m def diag(v, k=0): """ Extract a diagonal or construct a diagonal array. - As of NumPy 1.7, extracting a diagonal always returns a view into `v`. + See the more detailed documentation for ``numpy.diagonal`` if you use this + function to extract a diagonal and wish to write to the resulting array; + whether it returns a copy or a view depends on what version of numpy you + are using. Parameters ---------- v : array_like - If `v` is a 2-D array, return a view of its `k`-th diagonal. + If `v` is a 2-D array, return a copy of its `k`-th diagonal. If `v` is a 1-D array, return a 2-D array with `v` on the `k`-th diagonal. k : int, optional diff --git a/numpy/numarray/_capi.c b/numpy/numarray/_capi.c index fee07d79d..78187c50e 100644 --- a/numpy/numarray/_capi.c +++ b/numpy/numarray/_capi.c @@ -1077,9 +1077,12 @@ NA_OutputArray(PyObject *a, NumarrayType t, int requires) PyArray_Descr *dtype; PyArrayObject *ret; - if (!PyArray_Check(a) || !PyArray_ISWRITEABLE((PyArrayObject *)a)) { + if (!PyArray_Check(a)) { PyErr_Format(PyExc_TypeError, - "NA_OutputArray: only writeable arrays work for output."); + "NA_OutputArray: only arrays work for output."); + return NULL; + } + if (PyArray_FailUnlessWriteable((PyArrayObject *)a, "output array") < 0) { return NULL; } @@ -1098,12 +1101,10 @@ NA_OutputArray(PyObject *a, NumarrayType t, int requires) PyArray_DIMS((PyArrayObject *)a), dtype, 0); Py_INCREF(a); - if (PyArray_SetBaseObject(ret, a) < 0) { + if (PyArray_SetUpdateIfCopyBase(ret, a) < 0) { Py_DECREF(ret); return NULL; } - PyArray_ENABLEFLAGS(ret, NPY_ARRAY_UPDATEIFCOPY); - PyArray_CLEARFLAGS((PyArrayObject *)a, NPY_ARRAY_WRITEABLE); return ret; } @@ -1127,9 +1128,7 @@ NA_IoArray(PyObject *a, NumarrayType t, int requires) /* Guard against non-writable, but otherwise satisfying requires. In this case, shadow == a. */ - if (!PyArray_ISWRITABLE(shadow)) { - PyErr_Format(PyExc_TypeError, - "NA_IoArray: I/O array must be writable array"); + if (!PyArray_FailUnlessWriteable(shadow, "input/output array")) { PyArray_XDECREF_ERR(shadow); return NULL; } @@ -2488,13 +2487,10 @@ _setFromPythonScalarCore(PyArrayObject *a, long offset, PyObject*value, int entr static int NA_setFromPythonScalar(PyArrayObject *a, long offset, PyObject *value) { - if (PyArray_FLAGS(a) & NPY_ARRAY_WRITEABLE) - return _setFromPythonScalarCore(a, offset, value, 0); - else { - PyErr_Format( - PyExc_ValueError, "NA_setFromPythonScalar: assigment to readonly array buffer"); + if (PyArray_FailUnlessWriteable(a, "array") < 0) { return -1; } + return _setFromPythonScalarCore(a, offset, value, 0); } |