diff options
42 files changed, 692 insertions, 173 deletions
diff --git a/doc/CAPI.rst.txt b/doc/CAPI.rst.txt index 9656abf5c..f38815e2a 100644 --- a/doc/CAPI.rst.txt +++ b/doc/CAPI.rst.txt @@ -274,19 +274,27 @@ array, or part of some larger record array. But, they may have other uses... ``NPY_WRITEABLE`` True only if the data buffer can be "written" to. -``NPY_UPDATEIFCOPY`` +``NPY_WRITEBACKIFCOPY`` This is a special flag that is set if this array represents a copy made because a user required certain flags in ``PyArray_FromAny`` and a copy had to be made of some other array (and the user asked for this flag to be set in such a situation). The base attribute then - points to the "misbehaved" array (which is set read_only). When - the array with this flag set is deallocated, it will copy its - contents back to the "misbehaved" array (casting if necessary) and - will reset the "misbehaved" array to ``WRITEABLE``. If the - "misbehaved" array was not ``WRITEABLE`` to begin with then - ``PyArray_FromAny`` would have returned an error because ``UPDATEIFCOPY`` - would not have been possible. + points to the "misbehaved" array (which is set read_only). If you use + this flag, you are must call ``PyArray_ResolveWritebackIfCopy`` before + deallocating this array (i.e. before calling ``Py_DECREF`` the last time) + which will write the data contents back to the "misbehaved" array (casting + if necessary) and will reset the "misbehaved" array to ``WRITEABLE``. If + the "misbehaved" array was not ``WRITEABLE`` to begin with then + ``PyArray_FromAny`` would have returned an error because ``WRITEBACKIFCOPY`` + would not have been possible. In error conditions, call + ``PyArray_DiscardWritebackIfCopy`` to throw away the scratch buffer, then + ``Py_DECREF`` or ``Py_XDECREF``. +``NPY_UPDATEIFCOPY`` + Similar to ``NPY_WRITEBACKIFCOPY``, but deprecated since it copied the + contents back when the array is deallocated, which is not explicit and + relies on refcount semantics. Refcount semantics are unreliable on + alternative implementations of python such as PyPy. ``PyArray_UpdateFlags(obj, flags)`` will update the ``obj->flags`` for ``flags`` which can be any of ``NPY_CONTIGUOUS``, ``NPY_FORTRAN``, ``NPY_ALIGNED``, or diff --git a/doc/release/1.14.0-notes.rst b/doc/release/1.14.0-notes.rst index c1e632c80..100aaee72 100644 --- a/doc/release/1.14.0-notes.rst +++ b/doc/release/1.14.0-notes.rst @@ -187,6 +187,42 @@ When indexed with a float, the dtype object used to raise ``ValueError``. C API changes ============= +``UPDATEIFCOPY`` semantics deprecated in favor of ``WRITEBACKIFCOPY`` +--------------------------------------------------------------------- + +The ``UPDATEIFCOPY`` flag is deprecated, and replaced by a new flag +``WRITEBACKIFCOPY``. Use of the flag requires a call to ``PyArray_ResolveWritebackIfCopy``. + +Code using ``UPDATEIFCOPY`` must be updated to account for changed semantics, as +follows:. When ndarrays are created with either the deprecated ``UPDATEIFCOPY`` +or new ``WRITEBACKIFCOPY`` flags, a temporary copy of the data is created which +is eventually written back to the original array. For ``UPDATEIFCOPY``, the +writeback to the original array occurs at ndarray deallocation. For +``WRITEBACKIFCOPY``, a new ``PyArray_ResolveWritebackIfCopy`` function must be +explicitly called before deallocation (or, if an error occurred, +``PyArray_DiscardWritebackIfCopy``). Numpy now mostly uses the +``WRITEBACKIFCOPY`` mechanism internally, but still uses ``UPDATEIFCOPY`` in +nditer use for backward-compatibility with the python nditer interface. + +In python code, if numpy is compiled with ``-DDEPRECATE_UPDATEIFCOPY`` or if +run on PyPy, ``DeprecationWarning`` warnings will be issued on use of +``UPDATEIFCOPY`` (eg, when nditer is used) if writeback resolution has not +occurred before calling ``array_dealloc``. In the future this warning will +always be issued, once nditer stops using ``UPDATEIFCOPY``. + +In C code, calling ``PyArray_SetUpdateIfCopyBase`` will now always issue a +``DeprecationWarning`` and should be replaced by +``PyArray_SetWritebackIfCopyBase``. Similarly, calls to ``PyArray_XDECREF_ERR`` +should be replaced by ``PyArray_DiscardWritebackIfCopy``. Calling ndarray +creation functions such as ``PyArray_FromArray`` where flags uses +``NPY_ARRAY_UPDATEIFCOPY`` will also raise a ``DeprecationWarning``. In all +these cases the code must be further modified to call +``PyArray_ResolveWritebackIfCopy`` before deallocation. + +Note that during the deprecation period, the ``NPY_ARRAY_INOUT_ARRAY`` and +``NPY_ARRAY_INOUT_FARRAY`` flags should be replaced by +``NPY_ARRAY_INOUT_ARRAY2`` and ``NPY_ARRAY_INOUT_FARRAY2``, which behave +similarly but require use of the new ``WRITEBACKIFCOPY`` semantics. New Features ============ diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index ae95517b6..c41a8df56 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -170,7 +170,7 @@ concepts to remember include: .. data:: newaxis - The :const:`newaxis` object can be used in all slicing operations to + The :const:`newaxis` object can be used in all slicing operations to create an axis of length one. :const:`newaxis` is an alias for 'None', and 'None' can be used in place of this with the same result. @@ -503,7 +503,7 @@ dictionary-like. Indexing ``x['field-name']`` returns a new :term:`view` to the array, which is of the same shape as *x* (except when the field is a sub-array) but of data type ``x.dtype['field-name']`` and contains -only the part of the data in the specified field. Also +only the part of the data in the specified field. Also :ref:`record array <arrays.classes.rec>` scalars can be "indexed" this way. Indexing into a structured array can also be done with a list of field names, diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api.array.rst index 7e42d3c5f..d3aa7cafb 100644 --- a/doc/source/reference/c-api.array.rst +++ b/doc/source/reference/c-api.array.rst @@ -76,9 +76,10 @@ sub-types). your own memory, you should use the function :c:func:`PyArray_SetBaseObject` to set the base to an object which owns the memory. - If the :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag is set, it has a different + If the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or the + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flags are set, it has a different meaning, namely base is the array into which the current array will - be copied upon destruction. This overloading of the base property + be copied upon copy resolution. This overloading of the base property for two functions is likely to change in a future version of NumPy. .. c:function:: PyArray_Descr *PyArray_DESCR(PyArrayObject* arr) @@ -217,9 +218,9 @@ From scratch can be non-zero to indicate a Fortran-style contiguous array. If *data* is not ``NULL``, then it is assumed to point to the memory to be used for the array and the *flags* argument is used as the - new flags for the array (except the state of :c:data:`NPY_OWNDATA` - and :c:data:`NPY_ARRAY_UPDATEIFCOPY` flags of the new array will - be reset). + new flags for the array (except the state of :c:data:`NPY_OWNDATA`, + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` and :c:data:`NPY_ARRAY_UPDATEIFCOPY` + flags of the new array will be reset). In addition, if *data* is non-NULL, then *strides* can also be provided. If *strides* is ``NULL``, then the array strides @@ -444,19 +445,26 @@ From other objects safely. Without this flag, a data cast will occur only if it can be done safely, otherwise an error is raised. - .. c:var:: NPY_ARRAY_UPDATEIFCOPY + .. c:var:: NPY_ARRAY_WRITEBACKIFCOPY If *op* is already an array, but does not satisfy the requirements, then a copy is made (which will satisfy the requirements). If this flag is present and a copy (of an object that is already an array) must be made, then the corresponding - :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag is set in the returned - copy and *op* is made to be read-only. When the returned copy - is deleted (presumably after your calculations are complete), - its contents will be copied back into *op* and the *op* array + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag is set in the returned + copy and *op* is made to be read-only. You must be sure to call + :c:func:`PyArray_ResolveWritebackIfCopy` to copy the contents + back into *op* and the *op* array will be made writeable again. If *op* is not writeable to begin with, or if it is not already an array, then an error is raised. + .. c:var:: NPY_ARRAY_UPDATEIFCOPY + + Deprecated. Use :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, which is similar. + This flag "automatically" copies the data back when the returned + array is deallocated, which is not supported in all python + implementations. + .. c:var:: NPY_ARRAY_BEHAVED :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE` @@ -502,12 +510,14 @@ From other objects .. c:var:: NPY_ARRAY_INOUT_ARRAY :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_UPDATEIFCOPY` + :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` \| + :c:data:`NPY_ARRAY_UPDATEIFCOPY` .. c:var:: NPY_ARRAY_INOUT_FARRAY :c:data:`NPY_ARRAY_F_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_UPDATEIFCOPY` + :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` \| + :c:data:`NPY_ARRAY_UPDATEIFCOPY` .. c:function:: int PyArray_GetArrayParamsFromObject( \ PyObject* op, PyArray_Descr* requested_dtype, npy_bool writeable, \ @@ -752,7 +762,8 @@ From other objects :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_NOTSWAPPED`, :c:data:`NPY_ARRAY_ENSURECOPY`, - :c:data:`NPY_ARRAY_UPDATEIFCOPY`, :c:data:`NPY_ARRAY_FORCECAST`, and + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, :c:data:`NPY_ARRAY_UPDATEIFCOPY`, + :c:data:`NPY_ARRAY_FORCECAST`, and :c:data:`NPY_ARRAY_ENSUREARRAY`. Standard combinations of flags can also be used: @@ -1336,6 +1347,21 @@ Special functions for NPY_OBJECT decrement all the items in the object array prior to calling this function. +.. c:function:: int PyArray_SetUpdateIfCopyBase(PyArrayObject* arr, PyArrayObject* base) + + Precondition: ``arr`` is a copy of ``base`` (though possibly with different + strides, ordering, etc.) Set the UPDATEIFCOPY flag and ``arr->base`` so + that when ``arr`` is destructed, it will copy any changes back to ``base``. + DEPRECATED, use :c:func:`PyArray_SetWritebackIfCopyBase``. + +.. c:function:: int PyArray_SetWritebackIfCopyBase(PyArrayObject* arr, PyArrayObject* base) + + Precondition: ``arr`` is a copy of ``base`` (though possibly with different + strides, ordering, etc.) Sets the WRITEBACKIFCOPY flag and ``arr->base``, and + set ``base`` to READONLY. Call PyArray_ResolveWritebackIfCopy before calling + :c:func:`PyArray_DECREF`` in order copy any changes back to ``base`` and + reset the READONLY flag. + Array flags ----------- @@ -1415,24 +1441,33 @@ of the constant names is deprecated in 1.7. Notice that the above 3 flags are defined so that a new, well- behaved array has these flags defined as true. -.. c:var:: NPY_ARRAY_UPDATEIFCOPY +.. c:var:: NPY_ARRAY_WRITEBACKIFCOPY The data area represents a (well-behaved) copy whose information - should be transferred back to the original when this array is deleted. + should be transferred back to the original when + :c:func:`PyArray_ResolveWritebackIfCopy` is called. This is a special flag that is set if this array represents a copy made because a user required certain flags in :c:func:`PyArray_FromAny` and a copy had to be made of some other array (and the user asked for this flag to be set in such a situation). The base attribute then points to the "misbehaved" - array (which is set read_only). When the array with this flag set - is deallocated, it will copy its contents back to the "misbehaved" + array (which is set read_only). :c:func`PyArray_ResolveWritebackIfCopy` + will copy its contents back to the "misbehaved" array (casting if necessary) and will reset the "misbehaved" array to :c:data:`NPY_ARRAY_WRITEABLE`. If the "misbehaved" array was not :c:data:`NPY_ARRAY_WRITEABLE` to begin with then :c:func:`PyArray_FromAny` - would have returned an error because :c:data:`NPY_ARRAY_UPDATEIFCOPY` + would have returned an error because :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` would not have been possible. +.. c:var:: NPY_ARRAY_UPDATEIFCOPY + + A deprecated version of :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` which + depends upon ``dealloc`` to trigger the writeback. For backwards + compatibility, :c:func:`PyArray_ResolveWritebackIfCopy` is called at + ``dealloc`` but relying + on that behavior is deprecated and not supported in PyPy. + :c:func:`PyArray_UpdateFlags` (obj, flags) will update the ``obj->flags`` for ``flags`` which can be any of :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_ALIGNED`, or @@ -1513,7 +1548,8 @@ For all of these macros *arr* must be an instance of a (subclass of) combinations of the possible flags an array can have: :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_ALIGNED`, - :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_UPDATEIFCOPY`. + :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, + :c:data:`NPY_ARRAY_UPDATEIFCOPY`. .. c:function:: PyArray_IS_C_CONTIGUOUS(arr) @@ -3432,13 +3468,25 @@ Miscellaneous Macros Returns the reference count of any Python object. -.. c:function:: PyArray_XDECREF_ERR(PyObject \*obj) +.. c:function:: PyArray_DiscardWritebackIfCopy(PyObject* obj) + + Reset the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` and deprecated + :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag. Also resets the + :c:data:`NPY_ARRAY_WRITEABLE` flag on the base object. This is + useful for recovering from an error condition when + writeback semantics are used, but will lead to wrong results. + +.. c:function:: PyArray_XDECREF_ERR(PyObject* obj) + + Deprecated in 1.14, use :c:func:`PyArray_DiscardWritebackIfCopy` + followed by ``Py_XDECREF`` - DECREF's an array object which may have the :c:data:`NPY_ARRAY_UPDATEIFCOPY` + DECREF's an array object which may have the (deprecated) + :c:data:`NPY_ARRAY_UPDATEIFCOPY` or :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set without causing the contents to be copied back into the original array. Resets the :c:data:`NPY_ARRAY_WRITEABLE` flag on the base object. This is useful for recovering from an error condition when - :c:data:`NPY_ARRAY_UPDATEIFCOPY` is used. + writeback semantics are used, but will lead to wrong results. Enumerated Types diff --git a/doc/source/reference/c-api.types-and-structures.rst b/doc/source/reference/c-api.types-and-structures.rst index 255c348f9..dcebd1ede 100644 --- a/doc/source/reference/c-api.types-and-structures.rst +++ b/doc/source/reference/c-api.types-and-structures.rst @@ -133,10 +133,11 @@ PyArray_Type is related to this array. There are two use cases: 1) If this array does not own its own memory, then base points to the Python object that owns it (perhaps another array object), 2) If this array has - the :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag set, then this array is - a working copy of a "misbehaved" array. As soon as this array is - deleted, the array pointed to by base will be updated with the - contents of this array. + the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or + :c:data:NPY_ARRAY_WRITEBACKIFCOPY`: flag set, then this array is + a working copy of a "misbehaved" array. When + ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to by base + will be updated with the contents of this array. .. c:member:: PyArray_Descr *PyArrayObject.descr @@ -153,8 +154,8 @@ PyArray_Type Flags indicating how the memory pointed to by data is to be interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, - :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, and - :c:data:`NPY_ARRAY_UPDATEIFCOPY`. + :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, and :c:data:`NPY_ARRAY_UPDATEIFCOPY`. .. c:member:: PyObject *PyArrayObject.weakreflist diff --git a/doc/source/reference/internals.code-explanations.rst b/doc/source/reference/internals.code-explanations.rst index 94e827429..ca81e1676 100644 --- a/doc/source/reference/internals.code-explanations.rst +++ b/doc/source/reference/internals.code-explanations.rst @@ -368,8 +368,9 @@ The output arguments (if any) are then processed and any missing return arrays are constructed. If any provided output array doesn't have the correct type (or is mis-aligned) and is smaller than the buffer size, then a new output array is constructed with the special -UPDATEIFCOPY flag set so that when it is DECREF'd on completion of the -function, its contents will be copied back into the output array. +:c:data:`WRITEBACKIFCOPY` flag set. At the end of the function, +:c:func:`PyArray_ResolveWritebackIfCopy` is called so that +its contents will be copied back into the output array. Iterators for the output arguments are then processed. Finally, the decision is made about how to execute the looping @@ -508,10 +509,11 @@ of a different shape depending on whether the method is reduce, accumulate, or reduceat. If an output array is already provided, then it's shape is checked. If the output array is not C-contiguous, aligned, and of the correct data type, then a temporary copy is made -with the UPDATEIFCOPY flag set. In this way, the methods will be able +with the WRITEBACKIFCOPY flag set. In this way, the methods will be able to work with a well-behaved output array but the result will be copied -back into the true output array when the method computation is -complete. Finally, iterators are set up to loop over the correct axis +back into the true output array when :c:func:`PyArray_ResolveWritebackIfCopy` +is called at function completion. +Finally, iterators are set up to loop over the correct axis (depending on the value of axis provided to the method) and the setup routine returns to the actual computation routine. diff --git a/doc/source/reference/routines.io.rst b/doc/source/reference/routines.io.rst index 00cafebc2..5df590f17 100644 --- a/doc/source/reference/routines.io.rst +++ b/doc/source/reference/routines.io.rst @@ -14,7 +14,7 @@ NumPy binary files (NPY, NPZ) savez_compressed The format of these binary file types is documented in -http://docs.scipy.org/doc/numpy/neps/npy-format.html +http://docs.scipy.org/doc/numpy/neps/npy-format.html Text files ---------- diff --git a/doc/source/reference/routines.set.rst b/doc/source/reference/routines.set.rst index 0089fb3e9..b12d3d5f5 100644 --- a/doc/source/reference/routines.set.rst +++ b/doc/source/reference/routines.set.rst @@ -17,7 +17,7 @@ Boolean operations in1d intersect1d - isin + isin setdiff1d setxor1d union1d diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index f36cc493b..0c7f27e44 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -468,18 +468,20 @@ writeable). The syntax is Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE` \| + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` \| :c:data:`NPY_ARRAY_UPDATEIFCOPY`. This combination of flags is useful to specify an array that will be used for both - input and output. If a copy is needed, then when the - temporary is deleted (by your use of :c:func:`Py_DECREF` at - the end of the interface routine), the temporary array - will be copied back into the original array passed in. Use - of the :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag requires that the input + input and output. :c:func:`PyArray_ResolveWritebackIfCopy` + must be called before :func:`Py_DECREF` at + the end of the interface routine to write back the temporary data + into the original array passed in. Use + of the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or + :c:data:`NPY_ARRAY_UPDATEIFCOPY` flags requires that the input object is already an array (because other objects cannot be automatically updated in this fashion). If an error - occurs use :c:func:`PyArray_DECREF_ERR` (obj) on an array - with the :c:data:`NPY_ARRAY_UPDATEIFCOPY` flag set. This will - delete the array without causing the contents to be copied + occurs use :c:func:`PyArray_DiscardWritebackIfCopy` (obj) on an + array with these flags set. This will set the underlying base array + writable without causing the contents to be copied back into the original array. @@ -635,6 +637,7 @@ updates the output array. Py_DECREF(arr1); Py_DECREF(arr2); + PyArray_ResolveWritebackIfCopy(oarr); Py_DECREF(oarr); Py_INCREF(Py_None); return Py_None; @@ -642,6 +645,7 @@ updates the output array. fail: Py_XDECREF(arr1); Py_XDECREF(arr2); - PyArray_XDECREF_ERR(oarr); + PyArray_DiscardWritebackIfCopy(oarr); + Py_XDECREF(oarr); return NULL; } diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 6982bf689..595bede06 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -2792,8 +2792,13 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('flags', array raises a RuntimeError exception. ALIGNED (A) The data and all elements are aligned appropriately for the hardware. + WRITEBACKIFCOPY (X) + This array is a copy of some other array. The C-API function + PyArray_ResolveWritebackIfCopy must be called before deallocating + to the base array will be updated with the contents of this array. UPDATEIFCOPY (U) - This array is a copy of some other array. When this array is + (Deprecated, use WRITEBACKIFCOPY) This array is a copy of some other array. + When this array is deallocated, the base array will be updated with the contents of this array. FNC @@ -2813,13 +2818,14 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('flags', or by using lowercased attribute names (as in ``a.flags.writeable``). Short flag names are only supported in dictionary access. - Only the UPDATEIFCOPY, WRITEABLE, and ALIGNED flags can be changed by - the user, via direct assignment to the attribute or dictionary entry, - or by calling `ndarray.setflags`. + Only the WRITEBACKIFCOPY, UPDATEIFCOPY, WRITEABLE, and ALIGNED flags can be + changed by the user, via direct assignment to the attribute or dictionary + entry, or by calling `ndarray.setflags`. The array flags cannot be set arbitrarily: - UPDATEIFCOPY can only be set ``False``. + - WRITEBACKIFCOPY can only be set ``False``. - ALIGNED can only be set ``True`` if the data is truly aligned. - WRITEABLE can only be set ``True`` if the array owns its own memory or the ultimate owner of the memory exposes a writeable buffer @@ -4322,16 +4328,17 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', """ a.setflags(write=None, align=None, uic=None) - Set array flags WRITEABLE, ALIGNED, and UPDATEIFCOPY, respectively. + Set array flags WRITEABLE, ALIGNED, (WRITEBACKIFCOPY and UPDATEIFCOPY), + respectively. These Boolean-valued flags affect how numpy interprets the memory area used by `a` (see Notes below). The ALIGNED flag can only be set to True if the data is actually aligned according to the type. - The UPDATEIFCOPY flag can never be set to True. The flag WRITEABLE - can only be set to True if the array owns its own memory, or the - ultimate owner of the memory exposes a writeable buffer interface, - or is a string. (The exception for string is made so that unpickling - can be done without copying memory.) + The WRITEBACKIFCOPY and (deprecated) UPDATEIFCOPY flags can never be set + to True. The flag WRITEABLE can only be set to True if the array owns its + own memory, or the ultimate owner of the memory exposes a writeable buffer + interface, or is a string. (The exception for string is made so that + unpickling can be done without copying memory.) Parameters ---------- @@ -4345,20 +4352,22 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', Notes ----- Array flags provide information about how the memory area used - for the array is to be interpreted. There are 6 Boolean flags - in use, only three of which can be changed by the user: - UPDATEIFCOPY, WRITEABLE, and ALIGNED. + for the array is to be interpreted. There are 7 Boolean flags + in use, only four of which can be changed by the user: + WRITEBACKIFCOPY, UPDATEIFCOPY, WRITEABLE, and ALIGNED. WRITEABLE (W) the data area can be written to; ALIGNED (A) the data and strides are aligned appropriately for the hardware (as determined by the compiler); - UPDATEIFCOPY (U) this array is a copy of some other array (referenced - by .base). When this array is deallocated, the base array will be - updated with the contents of this array. + UPDATEIFCOPY (U) (deprecated), replaced by WRITEBACKIFCOPY; - All flags can be accessed using their first (upper case) letter as well + WRITEBACKIFCOPY (X) this array is a copy of some other array (referenced + by .base). When the C-API function PyArray_ResolveWritebackIfCopy is + called, the base array will be updated with the contents of this array. + + All flags can be accessed using the single (upper case) letter as well as the full name. Examples @@ -4373,6 +4382,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', OWNDATA : True WRITEABLE : True ALIGNED : True + WRITEBACKIFCOPY : False UPDATEIFCOPY : False >>> y.setflags(write=0, align=0) >>> y.flags @@ -4381,11 +4391,12 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', OWNDATA : True WRITEABLE : False ALIGNED : False + WRITEBACKIFCOPY : False UPDATEIFCOPY : False >>> y.setflags(uic=1) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: cannot set UPDATEIFCOPY flag to True + ValueError: cannot set WRITEBACKIFCOPY flag to True """)) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 6e6547129..b603b6df8 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -36,5 +36,6 @@ 0x0000000a = 9b8bce614655d3eb02acddcb508203cb # Version 11 (NumPy 1.13) Added PyArray_MapIterArrayCopyIfOverlap -# Version 11 (NumPy 1.14) No Change +# Version 11 (NumPy 1.14) Added PyArray_ResolveWritebackIfCopy, +# PyArray_SetWritebackIfCopyBase and deprecate PyArray_SetUpdateIfCopyBase. 0x0000000b = edb1ba83730c650fd9bc5772a919cda7 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index d1406e3b2..a454d95b0 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -346,6 +346,9 @@ multiarray_funcs_api = { # End 1.10 API 'PyArray_MapIterArrayCopyIfOverlap': (301,), # End 1.13 API + 'PyArray_ResolveWritebackIfCopy': (302,), + 'PyArray_SetWritebackIfCopyBase': (303,), + # End 1.14 API } ufunc_types_api = { diff --git a/numpy/core/include/numpy/ndarrayobject.h b/numpy/core/include/numpy/ndarrayobject.h index f26d64efb..4e63868f3 100644 --- a/numpy/core/include/numpy/ndarrayobject.h +++ b/numpy/core/include/numpy/ndarrayobject.h @@ -171,15 +171,16 @@ extern "C" CONFUSE_EMACS (l)*PyArray_STRIDES(obj)[3])) static NPY_INLINE void -PyArray_XDECREF_ERR(PyArrayObject *arr) +PyArray_DiscardWritebackIfCopy(PyArrayObject *arr) { if (arr != NULL) { - if (PyArray_FLAGS(arr) & NPY_ARRAY_UPDATEIFCOPY) { + if ((PyArray_FLAGS(arr) & NPY_ARRAY_WRITEBACKIFCOPY) || + (PyArray_FLAGS(arr) & NPY_ARRAY_UPDATEIFCOPY)) { PyArrayObject *base = (PyArrayObject *)PyArray_BASE(arr); PyArray_ENABLEFLAGS(base, NPY_ARRAY_WRITEABLE); + PyArray_CLEARFLAGS(arr, NPY_ARRAY_WRITEBACKIFCOPY); PyArray_CLEARFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); } - Py_DECREF(arr); } } @@ -238,6 +239,19 @@ PyArray_XDECREF_ERR(PyArrayObject *arr) #define DEPRECATE(msg) PyErr_WarnEx(PyExc_DeprecationWarning,msg,1) #define DEPRECATE_FUTUREWARNING(msg) PyErr_WarnEx(PyExc_FutureWarning,msg,1) +#if !defined(NPY_NO_DEPRECATED_API) || \ + (NPY_NO_DEPRECATED_API < NPY_1_14_API_VERSION) +static NPY_INLINE void +PyArray_XDECREF_ERR(PyArrayObject *arr) +{ + /* 2017-Nov-10 1.14 */ + DEPRECATE("PyArray_XDECREF_ERR is deprecated, call " + "PyArray_DiscardWritebackIfCopy then Py_XDECREF instead"); + PyArray_DiscardWritebackIfCopy(arr); + Py_XDECREF(arr); +} +#endif + #ifdef __cplusplus } diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 8c5d855df..19bbc7435 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -677,7 +677,7 @@ typedef struct tagPyArrayObject_fields { /* * This object is decref'd upon * deletion of array. Except in the - * case of UPDATEIFCOPY which has + * case of WRITEBACKIFCOPY which has * special handling. * * For views it points to the original @@ -688,9 +688,9 @@ typedef struct tagPyArrayObject_fields { * points to an object that should be * decref'd on deletion * - * For UPDATEIFCOPY flag this is an - * array to-be-updated upon deletion - * of this one + * For WRITEBACKIFCOPY flag this is an + * array to-be-updated upon calling + * PyArray_ResolveWritebackIfCopy */ PyObject *base; /* Pointer to type structure */ @@ -865,12 +865,13 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); /* * If this flag is set, then base contains a pointer to an array of * the same size that should be updated with the current contents of - * this array when this array is deallocated + * this array when PyArray_ResolveWritebackIfCopy is called. * * This flag may be requested in constructor functions. * This flag may be tested for in PyArray_FLAGS(arr). */ -#define NPY_ARRAY_UPDATEIFCOPY 0x1000 +#define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ +#define NPY_ARRAY_WRITEBACKIFCOPY 0x2000 /* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, @@ -895,10 +896,14 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_OUT_ARRAY (NPY_ARRAY_CARRAY) #define NPY_ARRAY_INOUT_ARRAY (NPY_ARRAY_CARRAY | \ NPY_ARRAY_UPDATEIFCOPY) +#define NPY_ARRAY_INOUT_ARRAY2 (NPY_ARRAY_CARRAY | \ + NPY_ARRAY_WRITEBACKIFCOPY) #define NPY_ARRAY_IN_FARRAY (NPY_ARRAY_FARRAY_RO) #define NPY_ARRAY_OUT_FARRAY (NPY_ARRAY_FARRAY) #define NPY_ARRAY_INOUT_FARRAY (NPY_ARRAY_FARRAY | \ NPY_ARRAY_UPDATEIFCOPY) +#define NPY_ARRAY_INOUT_FARRAY2 (NPY_ARRAY_FARRAY | \ + NPY_ARRAY_WRITEBACKIFCOPY) #define NPY_ARRAY_UPDATE_ALL (NPY_ARRAY_C_CONTIGUOUS | \ NPY_ARRAY_F_CONTIGUOUS | \ @@ -1044,7 +1049,7 @@ typedef void (NpyIter_GetMultiIndexFunc)(NpyIter *iter, #define NPY_ITER_CONTIG 0x00200000 /* The operand may be copied to satisfy requirements */ #define NPY_ITER_COPY 0x00400000 -/* The operand may be copied with UPDATEIFCOPY to satisfy requirements */ +/* The operand may be copied with WRITEBACKIFCOPY to satisfy requirements */ #define NPY_ITER_UPDATEIFCOPY 0x00800000 /* Allocate the operand if it is NULL */ #define NPY_ITER_ALLOCATE 0x01000000 diff --git a/numpy/core/include/numpy/noprefix.h b/numpy/core/include/numpy/noprefix.h index 45130d16e..041f30192 100644 --- a/numpy/core/include/numpy/noprefix.h +++ b/numpy/core/include/numpy/noprefix.h @@ -166,6 +166,7 @@ #define NOTSWAPPED NPY_NOTSWAPPED #define WRITEABLE NPY_WRITEABLE #define UPDATEIFCOPY NPY_UPDATEIFCOPY +#define WRITEBACKIFCOPY NPY_ARRAY_WRITEBACKIFCOPY #define ARR_HAS_DESCR NPY_ARR_HAS_DESCR #define BEHAVED NPY_BEHAVED #define BEHAVED_NS NPY_BEHAVED_NS diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index bf3f43444..25f4d6c35 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -666,6 +666,7 @@ def require(a, dtype=None, requirements=None): OWNDATA : False WRITEABLE : True ALIGNED : True + WRITEBACKIFCOPY : False UPDATEIFCOPY : False >>> y = np.require(x, dtype=np.float32, requirements=['A', 'O', 'W', 'F']) @@ -675,6 +676,7 @@ def require(a, dtype=None, requirements=None): OWNDATA : True WRITEABLE : True ALIGNED : True + WRITEBACKIFCOPY : False UPDATEIFCOPY : False """ diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 1d4816d96..1b4c2ee8d 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -76,7 +76,7 @@ PyArray_Size(PyObject *op) * Precondition: 'arr' is a copy of 'base' (though possibly with different * strides, ordering, etc.). This function sets the UPDATEIFCOPY flag and the * ->base pointer on 'arr', so that when 'arr' is destructed, it will copy any - * changes back to 'base'. + * changes back to 'base'. DEPRECATED, use PyArray_SetWritebackIfCopyBase * * Steals a reference to 'base'. * @@ -85,17 +85,59 @@ PyArray_Size(PyObject *op) NPY_NO_EXPORT int PyArray_SetUpdateIfCopyBase(PyArrayObject *arr, PyArrayObject *base) { + int ret; +#ifdef PYPY_VERSION + #ifndef DEPRECATE_UPDATEIFCOPY + #define DEPRECATE_UPDATEIFCOPY + #endif +#endif + +#ifdef DEPRECATE_UPDATEIFCOPY + /* TODO: enable this once a solution for UPDATEIFCOPY + * and nditer are resolved, also pending the fix for GH7054 + */ + /* 2017-Nov-10 1.14 */ + if (DEPRECATE("PyArray_SetUpdateIfCopyBase is deprecated, use " + "PyArray_SetWritebackIfCopyBase instead, and be sure to call " + "PyArray_ResolveWritebackIfCopy before the array is deallocated, " + "i.e. before the last call to Py_DECREF. If cleaning up from an " + "error, PyArray_DiscardWritebackIfCopy may be called instead to " + "throw away the scratch buffer.") < 0) + return -1; +#endif + ret = PyArray_SetWritebackIfCopyBase(arr, base); + if (ret >=0) { + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(arr, NPY_ARRAY_WRITEBACKIFCOPY); + } + return ret; +} + +/*NUMPY_API + * + * Precondition: 'arr' is a copy of 'base' (though possibly with different + * strides, ordering, etc.). This function sets the WRITEBACKIFCOPY flag and the + * ->base pointer on 'arr', call PyArray_ResolveWritebackIfCopy to copy any + * changes back to 'base' before deallocating the array. + * + * Steals a reference to 'base'. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +PyArray_SetWritebackIfCopyBase(PyArrayObject *arr, PyArrayObject *base) +{ if (base == NULL) { PyErr_SetString(PyExc_ValueError, - "Cannot UPDATEIFCOPY to NULL array"); + "Cannot WRITEBACKIFCOPY to NULL array"); return -1; } if (PyArray_BASE(arr) != NULL) { PyErr_SetString(PyExc_ValueError, - "Cannot set array with existing base to UPDATEIFCOPY"); + "Cannot set array with existing base to WRITEBACKIFCOPY"); goto fail; } - if (PyArray_FailUnlessWriteable(base, "UPDATEIFCOPY base") < 0) { + if (PyArray_FailUnlessWriteable(base, "WRITEBACKIFCOPY base") < 0) { goto fail; } @@ -112,7 +154,7 @@ PyArray_SetUpdateIfCopyBase(PyArrayObject *arr, PyArrayObject *base) * references. */ ((PyArrayObject_fields *)arr)->base = (PyObject *)base; - PyArray_ENABLEFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_WRITEBACKIFCOPY); PyArray_CLEARFLAGS(base, NPY_ARRAY_WRITEABLE); return 0; @@ -370,6 +412,54 @@ PyArray_TypeNumFromName(char *str) return NPY_NOTYPE; } +/*NUMPY_API + * + * If WRITEBACKIFCOPY and self has data, reset the base WRITEABLE flag, + * copy the local data to base, release the local data, and set flags + * appropriately. Return 0 if not relevant, 1 if success, < 0 on failure + */ +NPY_NO_EXPORT int +PyArray_ResolveWritebackIfCopy(PyArrayObject * self) +{ + PyArrayObject_fields *fa = (PyArrayObject_fields *)self; + if (fa && fa->base) { + if ((fa->flags & NPY_ARRAY_UPDATEIFCOPY) || (fa->flags & NPY_ARRAY_WRITEBACKIFCOPY)) { + /* + * UPDATEIFCOPY or WRITEBACKIFCOPY means that fa->base's data + * should be updated with the contents + * of self. + * fa->base->flags is not WRITEABLE to protect the relationship + * unlock it. + */ + int retval = 0; + PyArray_ENABLEFLAGS(((PyArrayObject *)fa->base), + NPY_ARRAY_WRITEABLE); + PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(self, NPY_ARRAY_WRITEBACKIFCOPY); + retval = PyArray_CopyAnyInto((PyArrayObject *)fa->base, self); + if (retval < 0) { + /* this should never happen, how did the two copies of data + * get out of sync? + */ + return retval; + } + if ((fa->flags & NPY_ARRAY_OWNDATA) && fa->data) { + /* Free internal references if an Object array */ + if (PyDataType_FLAGCHK(fa->descr, NPY_ITEM_REFCOUNT)) { + Py_INCREF(self); /*hold on to self */ + PyArray_XDECREF(self); + Py_DECREF(self); + } + npy_free_cache(fa->data, PyArray_NBYTES(self)); + fa->data = NULL; + PyArray_CLEARFLAGS(self, NPY_ARRAY_OWNDATA); + } + return 1; + } + } + return 0; + } + /*********************** end C-API functions **********************/ /* array object functions */ @@ -385,26 +475,51 @@ array_dealloc(PyArrayObject *self) PyObject_ClearWeakRefs((PyObject *)self); } if (fa->base) { - /* - * UPDATEIFCOPY means that base points to an - * array that should be updated with the contents - * of this array upon destruction. - * fa->base->flags must have been WRITEABLE - * (checked previously) and it was locked here - * thus, unlock it. - */ - if (fa->flags & NPY_ARRAY_UPDATEIFCOPY) { - PyArray_ENABLEFLAGS(((PyArrayObject *)fa->base), - NPY_ARRAY_WRITEABLE); - Py_INCREF(self); /* hold on to self in next call */ - if (PyArray_CopyAnyInto((PyArrayObject *)fa->base, self) < 0) { + int retval; + if (PyArray_FLAGS(self) & NPY_ARRAY_WRITEBACKIFCOPY) + { + char * msg = "WRITEBACKIFCOPY requires a call to " + "PyArray_ResolveWritebackIfCopy or " + "PyArray_DiscardWritebackIfCopy before array_dealloc is " + "called."; + /* 2017-Nov-10 1.14 */ + if (DEPRECATE(msg) < 0) { + /* dealloc must not raise an error, best effort try to write + to stderr and clear the error + */ + PyObject * s; +#if PY_MAJOR_VERSION < 3 + s = PyString_FromString(msg); +#else + s = PyUnicode_FromString(msg); +#endif + if (s) { + PyErr_WriteUnraisable(s); + Py_DECREF(s); + } + else { + PyErr_WriteUnraisable(Py_None); + } + } + retval = PyArray_ResolveWritebackIfCopy(self); + if (retval < 0) + { + PyErr_Print(); + PyErr_Clear(); + } + } + if (PyArray_FLAGS(self) & NPY_ARRAY_UPDATEIFCOPY) { + /* DEPRECATED, remove once the flag is removed */ + Py_INCREF(self); /* hold on to self in next call since if + * refcount == 0 it will recurse back into + *array_dealloc + */ + retval = PyArray_ResolveWritebackIfCopy(self); + if (retval < 0) + { PyErr_Print(); PyErr_Clear(); } - /* - * Don't need to DECREF -- because we are deleting - *self already... - */ } /* * In any case base is pointing to something that we need @@ -482,6 +597,8 @@ PyArray_DebugPrint(PyArrayObject *obj) printf(" NPY_WRITEABLE"); if (fobj->flags & NPY_ARRAY_UPDATEIFCOPY) printf(" NPY_UPDATEIFCOPY"); + if (fobj->flags & NPY_ARRAY_WRITEBACKIFCOPY) + printf(" NPY_WRITEBACKIFCOPY"); printf("\n"); if (fobj->base != NULL && PyArray_Check(fobj->base)) { @@ -505,8 +622,6 @@ PyArray_SetDatetimeParseFunction(PyObject *op) { } - - /*NUMPY_API */ NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/calculation.c b/numpy/core/src/multiarray/calculation.c index 379e5c3d2..e24ac2b57 100644 --- a/numpy/core/src/multiarray/calculation.c +++ b/numpy/core/src/multiarray/calculation.c @@ -118,7 +118,7 @@ PyArray_ArgMax(PyArrayObject *op, int axis, PyArrayObject *out) } rp = (PyArrayObject *)PyArray_FromArray(out, PyArray_DescrFromType(NPY_INTP), - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY); if (rp == NULL) { goto fail; } @@ -134,8 +134,9 @@ PyArray_ArgMax(PyArrayObject *op, int axis, PyArrayObject *out) NPY_END_THREADS_DESCR(PyArray_DESCR(ap)); Py_DECREF(ap); - /* Trigger the UPDATEIFCOPY if necessary */ + /* Trigger the UPDATEIFCOPY/WRTIEBACKIFCOPY if necessary */ if (out != NULL && out != rp) { + PyArray_ResolveWritebackIfCopy(rp); Py_DECREF(rp); rp = out; Py_INCREF(rp); @@ -233,7 +234,7 @@ PyArray_ArgMin(PyArrayObject *op, int axis, PyArrayObject *out) } rp = (PyArrayObject *)PyArray_FromArray(out, PyArray_DescrFromType(NPY_INTP), - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY); if (rp == NULL) { goto fail; } @@ -249,8 +250,9 @@ PyArray_ArgMin(PyArrayObject *op, int axis, PyArrayObject *out) NPY_END_THREADS_DESCR(PyArray_DESCR(ap)); Py_DECREF(ap); - /* Trigger the UPDATEIFCOPY if necessary */ + /* Trigger the UPDATEIFCOPY/WRITEBACKIFCOPY if necessary */ if (out != NULL && out != rp) { + PyArray_ResolveWritebackIfCopy(rp); Py_DECREF(rp); rp = out; Py_INCREF(rp); @@ -1117,7 +1119,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o oflags = NPY_ARRAY_FARRAY; else oflags = NPY_ARRAY_CARRAY; - oflags |= NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_FORCECAST; + oflags |= NPY_ARRAY_WRITEBACKIFCOPY | NPY_ARRAY_FORCECAST; Py_INCREF(indescr); newout = (PyArrayObject*)PyArray_FromArray(out, indescr, oflags); if (newout == NULL) { @@ -1153,6 +1155,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o Py_XDECREF(maxa); Py_DECREF(newin); /* Copy back into out if out was not already a nice array. */ + PyArray_ResolveWritebackIfCopy(newout); Py_DECREF(newout); return (PyObject *)out; @@ -1162,7 +1165,8 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o Py_XDECREF(maxa); Py_XDECREF(mina); Py_XDECREF(newin); - PyArray_XDECREF_ERR(newout); + PyArray_DiscardWritebackIfCopy(newout); + Py_XDECREF(newout); return NULL; } diff --git a/numpy/core/src/multiarray/cblasfuncs.c b/numpy/core/src/multiarray/cblasfuncs.c index 7cb1652bb..c941bb29b 100644 --- a/numpy/core/src/multiarray/cblasfuncs.c +++ b/numpy/core/src/multiarray/cblasfuncs.c @@ -412,7 +412,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, /* set copy-back */ Py_INCREF(out); - if (PyArray_SetUpdateIfCopyBase(out_buf, out) < 0) { + if (PyArray_SetWritebackIfCopyBase(out_buf, out) < 0) { Py_DECREF(out); goto fail; } @@ -772,6 +772,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, Py_DECREF(ap2); /* Trigger possible copyback into `result` */ + PyArray_ResolveWritebackIfCopy(out_buf); Py_DECREF(out_buf); return PyArray_Return(result); diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 95b1d241a..951c4d3ba 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -339,7 +339,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } array = (PyArrayObject *)PyArray_FromArray((PyArrayObject *)array0, NULL, - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY); if (array == NULL) { goto fail; } @@ -414,6 +414,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) Py_XDECREF(values); Py_XDECREF(mask); + PyArray_ResolveWritebackIfCopy(array); Py_DECREF(array); Py_RETURN_NONE; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index d31a9cf74..606c3e81f 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1009,6 +1009,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, } } else { + fa->flags = (flags & ~NPY_ARRAY_WRITEBACKIFCOPY); fa->flags = (flags & ~NPY_ARRAY_UPDATEIFCOPY); } fa->descr = descr; @@ -1703,10 +1704,11 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* If we got dimensions and dtype instead of an array */ if (arr == NULL) { - if (flags & NPY_ARRAY_UPDATEIFCOPY) { + if ((flags & NPY_ARRAY_WRITEBACKIFCOPY) || + (flags & NPY_ARRAY_UPDATEIFCOPY)) { Py_XDECREF(newtype); PyErr_SetString(PyExc_TypeError, - "UPDATEIFCOPY used for non-array input."); + "WRITEBACKIFCOPY used for non-array input."); return NULL; } else if (min_depth != 0 && ndim < min_depth) { @@ -1810,6 +1812,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * NPY_ARRAY_NOTSWAPPED, * NPY_ARRAY_ENSURECOPY, * NPY_ARRAY_UPDATEIFCOPY, + * NPY_ARRAY_WRITEBACKIFCOPY, * NPY_ARRAY_FORCECAST, * NPY_ARRAY_ENSUREARRAY, * NPY_ARRAY_ELEMENTSTRIDES @@ -1834,10 +1837,13 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * Fortran arrays are always behaved (aligned, * notswapped, and writeable) and not (C) CONTIGUOUS (if > 1d). * - * NPY_ARRAY_UPDATEIFCOPY flag sets this flag in the returned array if a copy - * is made and the base argument points to the (possibly) misbehaved array. - * When the new array is deallocated, the original array held in base - * is updated with the contents of the new array. + * NPY_ARRAY_UPDATEIFCOPY is deprecated in favor of + * NPY_ARRAY_WRITEBACKIFCOPY in 1.14 + + * NPY_ARRAY_WRITEBACKIFCOPY flag sets this flag in the returned + * array if a copy is made and the base argument points to the (possibly) + * misbehaved array. Before returning to python, PyArray_ResolveWritebackIfCopy + * must be called to update the contents of the orignal array from the copy. * * NPY_ARRAY_FORCECAST will cause a cast to occur regardless of whether or not * it is safe. @@ -2001,9 +2007,30 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) return NULL; } - if (flags & NPY_ARRAY_UPDATEIFCOPY) { + if (flags & NPY_ARRAY_UPDATEIFCOPY) { + /* This is the ONLY place the NPY_ARRAY_UPDATEIFCOPY flag + * is still used. + * Can be deleted once the flag itself is removed + */ + + /* 2017-Nov-10 1.14 */ + if (DEPRECATE("NPY_ARRAY_UPDATEIFCOPY, NPY_ARRAY_INOUT_ARRAY, and " + "NPY_ARRAY_INOUT_FARRAY are deprecated, use NPY_WRITEBACKIFCOPY, " + "NPY_ARRAY_INOUT_ARRAY2, or NPY_ARRAY_INOUT_FARRAY2 respectively " + "instead, and call PyArray_ResolveWritebackIfCopy before the " + "array is deallocated, i.e. before the last call to Py_DECREF.") < 0) + return NULL; + Py_INCREF(arr); + if (PyArray_SetWritebackIfCopyBase(ret, arr) < 0) { + Py_DECREF(ret); + return NULL; + } + PyArray_ENABLEFLAGS(ret, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(ret, NPY_ARRAY_WRITEBACKIFCOPY); + } + else if (flags & NPY_ARRAY_WRITEBACKIFCOPY) { Py_INCREF(arr); - if (PyArray_SetUpdateIfCopyBase(ret, arr) < 0) { + if (PyArray_SetWritebackIfCopyBase(ret, arr) < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index 7f56ddb03..af4a17def 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -208,11 +208,10 @@ arrayflags_dealloc(PyArrayFlagsObject *self) _define_get(NPY_ARRAY_C_CONTIGUOUS, contiguous) _define_get(NPY_ARRAY_F_CONTIGUOUS, fortran) -_define_get(NPY_ARRAY_UPDATEIFCOPY, updateifcopy) +_define_get(NPY_ARRAY_WRITEBACKIFCOPY, writebackifcopy) _define_get(NPY_ARRAY_OWNDATA, owndata) _define_get(NPY_ARRAY_ALIGNED, aligned) _define_get(NPY_ARRAY_WRITEABLE, writeable) - _define_get(NPY_ARRAY_ALIGNED| NPY_ARRAY_WRITEABLE, behaved) _define_get(NPY_ARRAY_ALIGNED| @@ -220,6 +219,25 @@ _define_get(NPY_ARRAY_ALIGNED| NPY_ARRAY_C_CONTIGUOUS, carray) static PyObject * +arrayflags_updateifcopy_get(PyArrayFlagsObject *self) +{ + PyObject *item; + /* 2017-Nov-10 1.14 */ + if(DEPRECATE("UPDATEIFCOPY deprecated, use WRITEBACKIFCOPY instead") < 0) { + return NULL; + } + if ((self->flags & (NPY_ARRAY_UPDATEIFCOPY)) == (NPY_ARRAY_UPDATEIFCOPY)) { + item = Py_True; + } + else { + item = Py_False; + } + Py_INCREF(item); + return item; +} + + +static PyObject * arrayflags_forc_get(PyArrayFlagsObject *self) { PyObject *item; @@ -291,6 +309,35 @@ arrayflags_updateifcopy_set(PyArrayFlagsObject *self, PyObject *obj) "Cannot set flags on array scalars."); return -1; } + /* 2017-Nov-10 1.14 */ + if(DEPRECATE("UPDATEIFCOPY deprecated, use WRITEBACKIFCOPY instead") < 0) { + return -1; + } + res = PyObject_CallMethod(self->arr, "setflags", "OOO", Py_None, Py_None, + (PyObject_IsTrue(obj) ? Py_True : Py_False)); + if (res == NULL) { + return -1; + } + Py_DECREF(res); + return 0; +} + +/* relies on setflags order being write, align, uic */ +static int +arrayflags_writebackifcopy_set(PyArrayFlagsObject *self, PyObject *obj) +{ + PyObject *res; + + if (obj == NULL) { + PyErr_SetString(PyExc_AttributeError, + "Cannot delete flags writebackifcopy attribute"); + return -1; + } + if (self->arr == NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot set flags on array scalars."); + return -1; + } res = PyObject_CallMethod(self->arr, "setflags", "OOO", Py_None, Py_None, (PyObject_IsTrue(obj) ? Py_True : Py_False)); if (res == NULL) { @@ -372,6 +419,10 @@ static PyGetSetDef arrayflags_getsets[] = { (getter)arrayflags_updateifcopy_get, (setter)arrayflags_updateifcopy_set, NULL, NULL}, + {"writebackifcopy", + (getter)arrayflags_writebackifcopy_get, + (setter)arrayflags_writebackifcopy_set, + NULL, NULL}, {"owndata", (getter)arrayflags_owndata_get, NULL, @@ -455,6 +506,8 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) return arrayflags_owndata_get(self); case 'A': return arrayflags_aligned_get(self); + case 'X': + return arrayflags_writebackifcopy_get(self); case 'U': return arrayflags_updateifcopy_get(self); default: @@ -522,6 +575,11 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) return arrayflags_fortran_get(self); } break; + case 14: + if (strncmp(key, "WRITEBACKIFCOPY", n) == 0) { + return arrayflags_writebackifcopy_get(self); + } + break; } fail: @@ -564,6 +622,10 @@ arrayflags_setitem(PyArrayFlagsObject *self, PyObject *ind, PyObject *item) ((n==1) && (strncmp(key, "U", n) == 0))) { return arrayflags_updateifcopy_set(self, item); } + else if (((n==14) && (strncmp(key, "WRITEBACKIFCOPY", n) == 0)) || + ((n==1) && (strncmp(key, "X", n) == 0))) { + return arrayflags_writebackifcopy_set(self, item); + } fail: PyErr_SetString(PyExc_KeyError, "Unknown flag"); @@ -589,16 +651,17 @@ arrayflags_print(PyArrayFlagsObject *self) return PyUString_FromFormat( " %s : %s\n %s : %s\n" " %s : %s\n %s : %s\n" - " %s : %s\n %s : %s", - "C_CONTIGUOUS", _torf_(fl, NPY_ARRAY_C_CONTIGUOUS), - "F_CONTIGUOUS", _torf_(fl, NPY_ARRAY_F_CONTIGUOUS), - "OWNDATA", _torf_(fl, NPY_ARRAY_OWNDATA), - "WRITEABLE", _torf_(fl, NPY_ARRAY_WRITEABLE), - "ALIGNED", _torf_(fl, NPY_ARRAY_ALIGNED), - "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY)); + " %s : %s\n %s : %s\n" + " %s : %s", + "C_CONTIGUOUS", _torf_(fl, NPY_ARRAY_C_CONTIGUOUS), + "F_CONTIGUOUS", _torf_(fl, NPY_ARRAY_F_CONTIGUOUS), + "OWNDATA", _torf_(fl, NPY_ARRAY_OWNDATA), + "WRITEABLE", _torf_(fl, NPY_ARRAY_WRITEABLE), + "ALIGNED", _torf_(fl, NPY_ARRAY_ALIGNED), + "WRITEBACKIFCOPY", _torf_(fl, NPY_ARRAY_WRITEBACKIFCOPY), + "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY)); } - static int arrayflags_compare(PyArrayFlagsObject *self, PyArrayFlagsObject *other) { diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index a43675040..825363f19 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -365,9 +365,11 @@ array_data_set(PyArrayObject *self, PyObject *op) PyDataMem_FREE(PyArray_DATA(self)); } if (PyArray_BASE(self)) { - if (PyArray_FLAGS(self) & NPY_ARRAY_UPDATEIFCOPY) { + if ((PyArray_FLAGS(self) & NPY_ARRAY_WRITEBACKIFCOPY) || + (PyArray_FLAGS(self) & NPY_ARRAY_UPDATEIFCOPY)) { PyArray_ENABLEFLAGS((PyArrayObject *)PyArray_BASE(self), NPY_ARRAY_WRITEABLE); + PyArray_CLEARFLAGS(self, NPY_ARRAY_WRITEBACKIFCOPY); PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); } Py_DECREF(PyArray_BASE(self)); @@ -614,7 +616,7 @@ array_struct_get(PyArrayObject *self) inter->itemsize = PyArray_DESCR(self)->elsize; inter->flags = PyArray_FLAGS(self); /* reset unused flags */ - inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_OWNDATA); + inter->flags &= ~(NPY_ARRAY_WRITEBACKIFCOPY | NPY_ARRAY_UPDATEIFCOPY |NPY_ARRAY_OWNDATA); if (PyArray_ISNOTSWAPPED(self)) inter->flags |= NPY_ARRAY_NOTSWAPPED; /* * Copy shape and strides over since these can be reset diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 3b5d76362..486eb43ce 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -87,8 +87,7 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, } else { - int flags = NPY_ARRAY_CARRAY | - NPY_ARRAY_UPDATEIFCOPY; + int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY; if ((PyArray_NDIM(out) != nd) || !PyArray_CompareLists(PyArray_DIMS(out), shape, nd)) { @@ -235,13 +234,15 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, Py_XDECREF(self); if (out != NULL && out != obj) { Py_INCREF(out); + PyArray_ResolveWritebackIfCopy(obj); Py_DECREF(obj); obj = out; } return (PyObject *)obj; fail: - PyArray_XDECREF_ERR(obj); + PyArray_DiscardWritebackIfCopy(obj); + Py_XDECREF(obj); Py_XDECREF(indices); Py_XDECREF(self); return NULL; @@ -273,7 +274,7 @@ PyArray_PutTo(PyArrayObject *self, PyObject* values0, PyObject *indices0, if (!PyArray_ISCONTIGUOUS(self)) { PyArrayObject *obj; - int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY; + int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY; if (clipmode == NPY_RAISE) { flags |= NPY_ARRAY_ENSURECOPY; @@ -407,6 +408,7 @@ PyArray_PutTo(PyArrayObject *self, PyObject* values0, PyObject *indices0, Py_XDECREF(values); Py_XDECREF(indices); if (copied) { + PyArray_ResolveWritebackIfCopy(self); Py_DECREF(self); } Py_RETURN_NONE; @@ -415,7 +417,8 @@ PyArray_PutTo(PyArrayObject *self, PyObject* values0, PyObject *indices0, Py_XDECREF(indices); Py_XDECREF(values); if (copied) { - PyArray_XDECREF_ERR(self); + PyArray_DiscardWritebackIfCopy(self); + Py_XDECREF(self); } return NULL; } @@ -448,7 +451,7 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) dtype = PyArray_DESCR(self); Py_INCREF(dtype); obj = (PyArrayObject *)PyArray_FromArray(self, dtype, - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEBACKIFCOPY); if (obj != self) { copied = 1; } @@ -524,6 +527,7 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) Py_XDECREF(values); Py_XDECREF(mask); if (copied) { + PyArray_ResolveWritebackIfCopy(self); Py_DECREF(self); } Py_RETURN_NONE; @@ -532,7 +536,8 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) Py_XDECREF(mask); Py_XDECREF(values); if (copied) { - PyArray_XDECREF_ERR(self); + PyArray_DiscardWritebackIfCopy(self); + Py_XDECREF(self); } return NULL; } @@ -694,7 +699,7 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out, } else { int flags = NPY_ARRAY_CARRAY | - NPY_ARRAY_UPDATEIFCOPY | + NPY_ARRAY_WRITEBACKIFCOPY | NPY_ARRAY_FORCECAST; if ((PyArray_NDIM(out) != multi->nd) @@ -769,6 +774,7 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out, npy_free_cache(mps, n * sizeof(mps[0])); if (out != NULL && out != obj) { Py_INCREF(out); + PyArray_ResolveWritebackIfCopy(obj); Py_DECREF(obj); obj = out; } @@ -781,7 +787,8 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out, } Py_XDECREF(ap); npy_free_cache(mps, n * sizeof(mps[0])); - PyArray_XDECREF_ERR(obj); + PyArray_DiscardWritebackIfCopy(obj); + Py_XDECREF(obj); return NULL; } diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 9e6ed712c..1be2eee35 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1149,6 +1149,7 @@ iter_richcompare(PyArrayIterObject *self, PyObject *other, int cmp_op) return NULL; } ret = array_richcompare(new, other, cmp_op); + PyArray_ResolveWritebackIfCopy(new); Py_DECREF(new); return ret; } diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 1a92365c8..18fa7b986 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -3269,7 +3269,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, * If copy_if_overlap != 0, check if `a` has memory overlap with any of the * arrays in `index` and with `extra_op`. If yes, make copies as appropriate * to avoid problems if `a` is modified during the iteration. - * `iter->array` may contain a copied array (with UPDATEIFCOPY set). + * `iter->array` may contain a copied array (UPDATEIFCOPY/WRITEBACKIFCOPY set). */ NPY_NO_EXPORT PyObject * PyArray_MapIterArrayCopyIfOverlap(PyArrayObject * a, PyObject * index, @@ -3303,7 +3303,7 @@ PyArray_MapIterArrayCopyIfOverlap(PyArrayObject * a, PyObject * index, } Py_INCREF(a); - if (PyArray_SetUpdateIfCopyBase(a_copy, a) < 0) { + if (PyArray_SetWritebackIfCopyBase(a_copy, a) < 0) { goto fail; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 6a121574b..572352304 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1782,6 +1782,7 @@ array_setstate(PyArrayObject *self, PyObject *args) Py_XDECREF(PyArray_BASE(self)); fa->base = NULL; + PyArray_CLEARFLAGS(self, NPY_ARRAY_WRITEBACKIFCOPY); PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); if (PyArray_DIMS(self) != NULL) { @@ -2323,11 +2324,12 @@ array_setflags(PyArrayObject *self, PyObject *args, PyObject *kwds) if (PyObject_IsTrue(uic)) { fa->flags = flagback; PyErr_SetString(PyExc_ValueError, - "cannot set UPDATEIFCOPY " \ + "cannot set WRITEBACKIFCOPY " \ "flag to True"); return NULL; } else { + PyArray_CLEARFLAGS(self, NPY_ARRAY_WRITEBACKIFCOPY); PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); Py_XDECREF(fa->base); fa->base = NULL; diff --git a/numpy/core/src/multiarray/multiarray_tests.c.src b/numpy/core/src/multiarray/multiarray_tests.c.src index a20cf6257..75ff7c8fd 100644 --- a/numpy/core/src/multiarray/multiarray_tests.c.src +++ b/numpy/core/src/multiarray/multiarray_tests.c.src @@ -619,6 +619,25 @@ npy_char_deprecation(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) return (PyObject *)descr; } +/* used to test UPDATEIFCOPY usage emits deprecation warning */ +static PyObject* +npy_updateifcopy_deprecation(PyObject* NPY_UNUSED(self), PyObject* args) +{ + int flags; + PyObject* array; + if (!PyArray_Check(args)) { + PyErr_SetString(PyExc_TypeError, "test needs ndarray input"); + return NULL; + } + flags = NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY; + array = PyArray_FromArray((PyArrayObject*)args, NULL, flags); + if (array == NULL) + return NULL; + PyArray_ResolveWritebackIfCopy((PyArrayObject*)array); + Py_DECREF(array); + Py_RETURN_NONE; +} + #if !defined(NPY_PY3K) static PyObject * int_subclass(PyObject *dummy, PyObject *args) @@ -1708,6 +1727,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"npy_char_deprecation", npy_char_deprecation, METH_NOARGS, NULL}, + {"npy_updateifcopy_deprecation", + npy_updateifcopy_deprecation, + METH_O, NULL}, #if !defined(NPY_PY3K) {"test_int_subclass", int_subclass, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 9ee53362e..f062a30f5 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -830,7 +830,7 @@ new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, /* set copy-back */ Py_INCREF(out); - if (PyArray_SetUpdateIfCopyBase(out_buf, out) < 0) { + if (PyArray_SetWritebackIfCopyBase(out_buf, out) < 0) { Py_DECREF(out); Py_DECREF(out_buf); return NULL; @@ -1102,6 +1102,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) Py_DECREF(ap2); /* Trigger possible copy-back into `result` */ + PyArray_ResolveWritebackIfCopy(out_buf); Py_DECREF(out_buf); return (PyObject *)result; @@ -4673,6 +4674,7 @@ set_flaginfo(PyObject *d) _addnew(CONTIGUOUS, NPY_ARRAY_C_CONTIGUOUS, C); _addnew(ALIGNED, NPY_ARRAY_ALIGNED, A); _addnew(UPDATEIFCOPY, NPY_ARRAY_UPDATEIFCOPY, U); + _addnew(WRITEBACKIFCOPY, NPY_ARRAY_WRITEBACKIFCOPY, X); _addnew(WRITEABLE, NPY_ARRAY_WRITEABLE, W); _addone(C_CONTIGUOUS, NPY_ARRAY_C_CONTIGUOUS); _addone(F_CONTIGUOUS, NPY_ARRAY_F_CONTIGUOUS); diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 1af396821..0d318178f 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -694,7 +694,7 @@ npyiter_convert_ops(PyObject *op_in, PyObject *op_flags_in, int fromanyflags = 0; if (op_flags[iop]&(NPY_ITER_READWRITE|NPY_ITER_WRITEONLY)) { - fromanyflags |= NPY_ARRAY_UPDATEIFCOPY; + fromanyflags |= NPY_ARRAY_WRITEBACKIFCOPY; } ao = (PyArrayObject *)PyArray_FROM_OF((PyObject *)op[iop], fromanyflags); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 306944d11..836df82fa 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -1090,7 +1090,8 @@ gentype_struct_get(PyObject *self) inter->two = 2; inter->nd = 0; inter->flags = PyArray_FLAGS(arr); - inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_OWNDATA); + inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_WRITEBACKIFCOPY | + NPY_ARRAY_OWNDATA); inter->flags |= NPY_ARRAY_NOTSWAPPED; inter->typekind = PyArray_DESCR(arr)->kind; inter->itemsize = PyArray_DESCR(arr)->elsize; diff --git a/numpy/core/src/multiarray/temp_elide.c b/numpy/core/src/multiarray/temp_elide.c index b8fa4c0ae..e5175f162 100644 --- a/numpy/core/src/multiarray/temp_elide.c +++ b/numpy/core/src/multiarray/temp_elide.c @@ -287,6 +287,7 @@ can_elide_temp(PyArrayObject * alhs, PyObject * orhs, int * cannot) !PyArray_CHKFLAGS(alhs, NPY_ARRAY_OWNDATA) || !PyArray_ISWRITEABLE(alhs) || PyArray_CHKFLAGS(alhs, NPY_ARRAY_UPDATEIFCOPY) || + PyArray_CHKFLAGS(alhs, NPY_ARRAY_WRITEBACKIFCOPY) || PyArray_NBYTES(alhs) < NPY_MIN_ELIDE_BYTES) { return 0; } diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index 390b28c31..04f5cc1d3 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -189,7 +189,7 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, } Py_INCREF(ret); - if (PyArray_SetUpdateIfCopyBase(ret_copy, (PyArrayObject *)ret) < 0) { + if (PyArray_SetWritebackIfCopyBase(ret_copy, (PyArrayObject *)ret) < 0) { Py_DECREF(ret); Py_DECREF(ret_copy); return NULL; @@ -485,7 +485,7 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, * This either conforms 'out' to the ndim of 'operand', or allocates * a new array appropriate for this reduction. * - * A new array with UPDATEIFCOPY is allocated if operand and out have memory + * A new array with WRITEBACKIFCOPY is allocated if operand and out have memory * overlap. */ Py_INCREF(result_dtype); @@ -611,6 +611,7 @@ finish: } } else { + PyArray_ResolveWritebackIfCopy(result); /* prevent spurious warnings */ Py_DECREF(result); result = out; Py_INCREF(result); @@ -619,6 +620,7 @@ finish: return result; fail: + PyArray_ResolveWritebackIfCopy(result); /* prevent spurious warnings */ Py_XDECREF(result); Py_XDECREF(op_view); if (iter != NULL) { diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 16693b366..35c7724b1 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1177,7 +1177,7 @@ iterator_loop(PyUFuncObject *ufunc, PyUFuncGenericFunction innerloop, void *innerloopdata) { - npy_intp i, nin = ufunc->nin, nout = ufunc->nout; + npy_intp i, iop, nin = ufunc->nin, nout = ufunc->nout; npy_intp nop = nin + nout; npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; @@ -1263,6 +1263,12 @@ iterator_loop(PyUFuncObject *ufunc, /* Call the __array_prepare__ functions for the new array */ if (prepare_ufunc_output(ufunc, &op[nin+i], arr_prep[i], arr_prep_args, i) < 0) { + for(iop = 0; iop < nin+i; ++iop) { + if (op_it[iop] != op[iop]) { + /* ignore errrors */ + PyArray_ResolveWritebackIfCopy(op_it[iop]); + } + } NpyIter_Deallocate(iter); return -1; } @@ -1315,7 +1321,11 @@ iterator_loop(PyUFuncObject *ufunc, NPY_END_THREADS; } - + for(iop = 0; iop < nop; ++iop) { + if (op_it[iop] != op[iop]) { + PyArray_ResolveWritebackIfCopy(op_it[iop]); + } + } NpyIter_Deallocate(iter); return 0; } @@ -1502,7 +1512,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyObject **arr_prep, PyObject *arr_prep_args) { - int i, nin = ufunc->nin, nout = ufunc->nout; + int retval, i, nin = ufunc->nin, nout = ufunc->nout; int nop = nin + nout; npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; @@ -1688,8 +1698,16 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, NPY_AUXDATA_FREE(innerloopdata); } + retval = 0; + nop = NpyIter_GetNOp(iter); + for(i=0; i< nop; ++i) { + if (PyArray_ResolveWritebackIfCopy(NpyIter_GetOperandArray(iter)[i]) < 0) { + retval = -1; + } + } + NpyIter_Deallocate(iter); - return 0; + return retval; } static PyObject * @@ -2286,6 +2304,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, goto fail; } + /* Write back any temporary data from PyArray_SetWritebackIfCopyBase */ + for(i=nin; i< nop; ++i) + if (PyArray_ResolveWritebackIfCopy(NpyIter_GetOperandArray(iter)[i]) < 0) + goto fail; + PyArray_free(inner_strides); NpyIter_Deallocate(iter); /* The caller takes ownership of all the references in op */ @@ -3218,6 +3241,9 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, } finish: + /* Write back any temporary data from PyArray_SetWritebackIfCopyBase */ + if (PyArray_ResolveWritebackIfCopy(op[0]) < 0) + goto fail; Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); NpyIter_Deallocate(iter_inner); @@ -3600,6 +3626,9 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, } finish: + if (op[0] && PyArray_ResolveWritebackIfCopy(op[0]) < 0) { + goto fail; + } Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); @@ -4114,7 +4143,8 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) } else if (override) { for (i = 0; i < ufunc->nargs; i++) { - PyArray_XDECREF_ERR(mps[i]); + PyArray_DiscardWritebackIfCopy(mps[i]); + Py_XDECREF(mps[i]); } return override; } @@ -4122,7 +4152,8 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) errval = PyUFunc_GenericFunction(ufunc, args, kwds, mps); if (errval < 0) { for (i = 0; i < ufunc->nargs; i++) { - PyArray_XDECREF_ERR(mps[i]); + PyArray_DiscardWritebackIfCopy(mps[i]); + Py_XDECREF(mps[i]); } if (errval == -1) { return NULL; @@ -5215,6 +5246,9 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) NpyIter_Deallocate(iter_buffer); + if (op1_array != (PyArrayObject*)op1) { + PyArray_ResolveWritebackIfCopy(op1_array); + } Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); @@ -5231,6 +5265,9 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) fail: + if (op1_array != (PyArrayObject*)op1) { + PyArray_ResolveWritebackIfCopy(op1_array); + } Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 1c1851fc7..fe0c7cc5f 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -434,6 +434,18 @@ class TestNPY_CHAR(_DeprecationTestCase): assert_(npy_char_deprecation() == 'S1') +class Test_UPDATEIFCOPY(_DeprecationTestCase): + """ + v1.14 deprecates creating an array with the UPDATEIFCOPY flag, use + WRITEBACKIFCOPY instead + """ + def test_npy_updateifcopy_deprecation(self): + from numpy.core.multiarray_tests import npy_updateifcopy_deprecation + arr = np.arange(9).reshape(3, 3) + v = arr.T + self.assert_deprecated(npy_updateifcopy_deprecation, args=(v,)) + + class TestDatetimeEvent(_DeprecationTestCase): # 2017-08-11, 1.14.0 def test_3_tuple(self): diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index bf2f8e6da..90cc473bc 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -18,11 +18,11 @@ if sys.version_info[0] >= 3: else: import __builtin__ as builtins from decimal import Decimal - +from unittest import TestCase import numpy as np from numpy.compat import strchar, unicode -from .test_print import in_foreign_locale +from numpy.core.tests.test_print import in_foreign_locale from numpy.core.multiarray_tests import ( test_neighborhood_iterator, test_neighborhood_iterator_oob, test_pydatamem_seteventhook_start, test_pydatamem_seteventhook_end, @@ -89,6 +89,7 @@ class TestFlags(object): def test_otherflags(self): assert_equal(self.a.flags.carray, True) + assert_equal(self.a.flags['C'], True) assert_equal(self.a.flags.farray, False) assert_equal(self.a.flags.behaved, True) assert_equal(self.a.flags.fnc, False) @@ -96,7 +97,13 @@ class TestFlags(object): assert_equal(self.a.flags.owndata, True) assert_equal(self.a.flags.writeable, True) assert_equal(self.a.flags.aligned, True) - assert_equal(self.a.flags.updateifcopy, False) + with assert_warns(DeprecationWarning): + assert_equal(self.a.flags.updateifcopy, False) + with assert_warns(DeprecationWarning): + assert_equal(self.a.flags['U'], False) + assert_equal(self.a.flags.writebackifcopy, False) + assert_equal(self.a.flags['X'], False) + def test_string_align(self): a = np.zeros(4, dtype=np.dtype('|S4')) @@ -4498,12 +4505,19 @@ class TestFlat(object): # UPDATEIFCOPY array returned for non-contiguous arrays. assert_(e.flags.writeable is True) assert_(f.flags.writeable is False) - - assert_(c.flags.updateifcopy is False) - assert_(d.flags.updateifcopy is False) - assert_(e.flags.updateifcopy is False) - # UPDATEIFCOPY is removed. - assert_(f.flags.updateifcopy is False) + with assert_warns(DeprecationWarning): + assert_(c.flags.updateifcopy is False) + with assert_warns(DeprecationWarning): + assert_(d.flags.updateifcopy is False) + with assert_warns(DeprecationWarning): + assert_(e.flags.updateifcopy is False) + with assert_warns(DeprecationWarning): + # UPDATEIFCOPY is removed. + assert_(f.flags.updateifcopy is False) + assert_(c.flags.writebackifcopy is False) + assert_(d.flags.writebackifcopy is False) + assert_(e.flags.writebackifcopy is False) + assert_(f.flags.writebackifcopy is False) class TestResize(object): @@ -6481,7 +6495,7 @@ class TestArrayAttributeDeletion(object): def test_multiarray_flags_writable_attribute_deletion(self): a = np.ones(2).flags - attr = ['updateifcopy', 'aligned', 'writeable'] + attr = ['writebackifcopy', 'updateifcopy', 'aligned', 'writeable'] for s in attr: assert_raises(AttributeError, delattr, a, s) @@ -7102,6 +7116,72 @@ class TestCTypes(object): _internal.ctypes = ctypes +class TestUpdateIfCopy(TestCase): + # all these tests use the WRITEBACKIFCOPY mechanism + def test_argmax_with_out(self): + mat = np.eye(5) + out = np.empty(5, dtype='i2') + res = np.argmax(mat, 0, out=out) + assert_equal(res, range(5)) + + def test_argmin_with_out(self): + mat = -np.eye(5) + out = np.empty(5, dtype='i2') + res = np.argmin(mat, 0, out=out) + assert_equal(res, range(5)) + + def test_clip_with_out(self): + mat = np.eye(5) + out = np.eye(5, dtype='i2') + res = np.clip(mat, a_min=-10, a_max=0, out=out) + assert_equal(np.sum(out), 0) + + def test_insert_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + # uses arr_insert + np.place(a, a>2, [44, 55]) + assert_equal(a, np.array([[0, 44], [1, 55], [2, 44]])) + + def test_put_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + np.put(a, [0, 2], [44, 55]) + assert_equal(a, np.array([[44, 3], [55, 4], [2, 5]])) + + def test_putmask_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + # uses arr_putmask + np.putmask(a, a>2, a**2) + assert_equal(a, np.array([[0, 9], [1, 16], [2, 25]])) + + def test_take_mode_raise(self): + a = np.arange(6, dtype='int') + out = np.empty(2, dtype='int') + np.take(a, [0, 2], out=out, mode='raise') + assert_equal(out, np.array([0, 2])) + + def test_choose_mod_raise(self): + a = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]]) + out = np.empty((3,3), dtype='int') + choices = [-10, 10] + np.choose(a, choices, out=out, mode='raise') + assert_equal(out, np.array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]])) + + def test_flatiter__array__(self): + a = np.arange(9).reshape(3,3) + b = a.T.flat + c = b.__array__() + # triggers the WRITEBACKIFCOPY resolution, assuming refcount semantics + del c + + def test_dot_out(self): + # if HAVE_CBLAS, will use WRITEBACKIFCOPY + a = np.arange(9, dtype=float).reshape(3,3) + b = np.dot(a, a, out=a) + assert_equal(b, np.array([[15, 18, 21], [42, 54, 66], [69, 90, 111]])) + + def test_orderconverter_with_nonASCII_unicode_ordering(): # gh-7475 a = np.arange(5) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index aaf01999c..2cbcab2d1 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -904,7 +904,7 @@ class TestTypes(object): fi = np.finfo(dt) assert_(np.can_cast(fi.min, dt)) assert_(np.can_cast(fi.max, dt)) - + # Custom exception class to test exception propagation in fromiter class NIterError(Exception): @@ -1319,7 +1319,7 @@ def assert_array_strict_equal(x, y): assert_(x.flags.writeable == y.flags.writeable) assert_(x.flags.c_contiguous == y.flags.c_contiguous) assert_(x.flags.f_contiguous == y.flags.f_contiguous) - assert_(x.flags.updateifcopy == y.flags.updateifcopy) + assert_(x.flags.writebackifcopy == y.flags.writebackifcopy) # check endianness assert_(x.dtype.isnative == y.dtype.isnative) diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 77aace249..b8457c78b 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -164,7 +164,7 @@ def _num_fromflags(flaglist): return num _flagnames = ['C_CONTIGUOUS', 'F_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', - 'OWNDATA', 'UPDATEIFCOPY'] + 'OWNDATA', 'UPDATEIFCOPY', 'WRITEBACKIFCOPY'] def _flags_fromnum(num): res = [] for key in _flagnames: @@ -244,6 +244,7 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): - OWNDATA / O - WRITEABLE / W - ALIGNED / A + - WRITEBACKIFCOPY / X - UPDATEIFCOPY / U Returns diff --git a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c index 2da6a2c5d..22801abdc 100644 --- a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c +++ b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c @@ -199,6 +199,7 @@ PyMODINIT_FUNC inittest_array_from_pyobj_ext(void) { PyDict_SetItemString(d, "ALIGNED", PyInt_FromLong(NPY_ARRAY_ALIGNED)); PyDict_SetItemString(d, "WRITEABLE", PyInt_FromLong(NPY_ARRAY_WRITEABLE)); PyDict_SetItemString(d, "UPDATEIFCOPY", PyInt_FromLong(NPY_ARRAY_UPDATEIFCOPY)); + PyDict_SetItemString(d, "WRITEBACKIFCOPY", PyInt_FromLong(NPY_ARRAY_WRITEBACKIFCOPY)); PyDict_SetItemString(d, "BEHAVED", PyInt_FromLong(NPY_ARRAY_BEHAVED)); PyDict_SetItemString(d, "BEHAVED_NS", PyInt_FromLong(NPY_ARRAY_BEHAVED_NS)); diff --git a/numpy/f2py/tests/test_array_from_pyobj.py b/numpy/f2py/tests/test_array_from_pyobj.py index 663fead6a..776ec3471 100644 --- a/numpy/f2py/tests/test_array_from_pyobj.py +++ b/numpy/f2py/tests/test_array_from_pyobj.py @@ -51,7 +51,7 @@ def flags2names(flags): info = [] for flagname in ['CONTIGUOUS', 'FORTRAN', 'OWNDATA', 'ENSURECOPY', 'ENSUREARRAY', 'ALIGNED', 'NOTSWAPPED', 'WRITEABLE', - 'UPDATEIFCOPY', 'BEHAVED', 'BEHAVED_RO', + 'WRITEBACKIFCOPY', 'UPDATEIFCOPY', 'BEHAVED', 'BEHAVED_RO', 'CARRAY', 'FARRAY' ]: if abs(flags) & getattr(wrap, flagname, 0): diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 605f37a02..b71e8fa06 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -4663,6 +4663,7 @@ class MaskedArray(ndarray): OWNDATA : False WRITEABLE : True ALIGNED : True + WRITEBACKIFCOPY : False UPDATEIFCOPY : False """ diff --git a/numpy/random/mtrand/numpy.pxd b/numpy/random/mtrand/numpy.pxd index db6265238..9092fa113 100644 --- a/numpy/random/mtrand/numpy.pxd +++ b/numpy/random/mtrand/numpy.pxd @@ -41,7 +41,7 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_ALIGNED NPY_ARRAY_NOTSWAPPED NPY_ARRAY_WRITEABLE - NPY_ARRAY_UPDATEIFCOPY + NPY_ARRAY_WRITEBACKIFCOPY NPY_ARR_HAS_DESCR NPY_ARRAY_BEHAVED |