diff options
Diffstat (limited to 'numpy')
29 files changed, 2286 insertions, 1290 deletions
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index f25159d8e..01741cd1a 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -645,3 +645,15 @@ class AxisError(ValueError, IndexError): msg = "{}: {}".format(msg_prefix, msg) super(AxisError, self).__init__(msg) + + +def array_ufunc_errmsg_formatter(ufunc, method, *inputs, **kwargs): + """ Format the error message for when __array_ufunc__ gives up. """ + args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] + + ['{}={!r}'.format(k, v) + for k, v in kwargs.items()]) + args = inputs + kwargs.get('out', ()) + types_string = ', '.join(repr(type(arg).__name__) for arg in args) + return ('operand type(s) do not implement __array_ufunc__' + '({!r}, {!r}, {}): {}' + .format(ufunc, method, args_string, types_string)) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 20d4c7792..e057c5614 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -748,6 +748,8 @@ def configuration(parent_package='',top_path=None): join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h'), join('src', 'private', 'npy_extint128.h'), join('include', 'numpy', 'arrayobject.h'), join('include', 'numpy', '_neighborhood_iterator_imp.h'), @@ -818,6 +820,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'vdot.c'), join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c'), ] blas_info = get_info('blas_opt', 0) @@ -871,7 +874,9 @@ def configuration(parent_package='',top_path=None): join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'scalarmath.c.src'), join('src', 'umath', 'ufunc_type_resolution.c'), - join('src', 'private', 'mem_overlap.c')] + join('src', 'umath', 'override.c'), + join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c')] umath_deps = [ generate_umath_py, @@ -880,10 +885,12 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'common.h'), join('src', 'private', 'templ_common.h.src'), join('src', 'umath', 'simd.inc.src'), + join('src', 'umath', 'override.h'), join(codegen_dir, 'generate_ufunc_api.py'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), - join('src', 'private', 'ufunc_override.h')] + npymath_sources + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h')] + npymath_sources config.add_extension('umath', sources=umath_src + diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 8946ff255..df3890201 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -54,6 +54,8 @@ maintainer email: oliphant.travis@ieee.org #include "mem_overlap.h" #include "numpyos.h" +#include "binop_override.h" + /*NUMPY_API Compute the size of an array (in number of items) */ @@ -1335,23 +1337,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) switch (cmp_op) { case Py_LT: - if (needs_right_binop_forward(obj_self, other, "__gt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - /* See discussion in number.c */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less); break; case Py_LE: - if (needs_right_binop_forward(obj_self, other, "__ge__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less_equal); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less_equal); break; case Py_EQ: /* @@ -1401,11 +1392,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__eq__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.equal); @@ -1478,11 +1465,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__ne__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.not_equal); if (result == NULL) { @@ -1502,20 +1485,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) } break; case Py_GT: - if (needs_right_binop_forward(obj_self, other, "__lt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater); break; case Py_GE: - if (needs_right_binop_forward(obj_self, other, "__le__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater_equal); break; diff --git a/numpy/core/src/multiarray/cblasfuncs.c b/numpy/core/src/multiarray/cblasfuncs.c index 4b11be947..3b0b2f4f6 100644 --- a/numpy/core/src/multiarray/cblasfuncs.c +++ b/numpy/core/src/multiarray/cblasfuncs.c @@ -237,7 +237,7 @@ _bad_strides(PyArrayObject *ap) * This is for use by PyArray_MatrixProduct2. It is assumed on entry that * the arrays ap1 and ap2 have a common data type given by typenum that is * float, double, cfloat, or cdouble and have dimension <= 2. The - * __numpy_ufunc__ nonsense is also assumed to have been taken care of. + * __array_ufunc__ nonsense is also assumed to have been taken care of. */ NPY_NO_EXPORT PyObject * cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index dc9b2edec..1df3b8b48 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -14,6 +14,8 @@ #include "common.h" #include "buffer.h" +#include "get_attr_string.h" + /* * The casting to use for implicit assignment operations resulting from * in-place operations (like +=) and out= arguments. (Notice that this @@ -29,61 +31,6 @@ * warning (that people's code will be broken in a future release.) */ -/* - * PyArray_GetAttrString_SuppressException: - * - * Stripped down version of PyObject_GetAttrString, - * avoids lookups for None, tuple, and List objects, - * and doesn't create a PyErr since this code ignores it. - * - * This can be much faster then PyObject_GetAttrString where - * exceptions are not used by caller. - * - * 'obj' is the object to search for attribute. - * - * 'name' is the attribute to search for. - * - * Returns attribute value on success, 0 on failure. - */ -PyObject * -PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) -{ - PyTypeObject *tp = Py_TYPE(obj); - PyObject *res = (PyObject *)NULL; - - /* We do not need to check for special attributes on trivial types */ - if (_is_basic_python_type(obj)) { - return NULL; - } - - /* Attribute referenced by (char *)name */ - if (tp->tp_getattr != NULL) { - res = (*tp->tp_getattr)(obj, name); - if (res == NULL) { - PyErr_Clear(); - } - } - /* Attribute referenced by (PyObject *)name */ - else if (tp->tp_getattro != NULL) { -#if defined(NPY_PY3K) - PyObject *w = PyUnicode_InternFromString(name); -#else - PyObject *w = PyString_InternFromString(name); -#endif - if (w == NULL) { - return (PyObject *)NULL; - } - res = (*tp->tp_getattro)(obj, w); - Py_DECREF(w); - if (res == NULL) { - PyErr_Clear(); - } - } - return res; -} - - - NPY_NO_EXPORT NPY_CASTING NPY_DEFAULT_ASSIGN_CASTING = NPY_SAME_KIND_CASTING; diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 8da317856..ae9b960c8 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -40,9 +40,6 @@ NPY_NO_EXPORT int PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_status); -NPY_NO_EXPORT PyObject * -PyArray_GetAttrString_SuppressException(PyObject *v, char *name); - /* * Returns NULL without setting an exception if no scalar is matched, a * new dtype reference otherwise. @@ -255,34 +252,6 @@ npy_memchr(char * haystack, char needle, return p; } -static NPY_INLINE int -_is_basic_python_type(PyObject * obj) -{ - if (obj == Py_None || - PyBool_Check(obj) || - /* Basic number types */ -#if !defined(NPY_PY3K) - PyInt_CheckExact(obj) || - PyString_CheckExact(obj) || -#endif - PyLong_CheckExact(obj) || - PyFloat_CheckExact(obj) || - PyComplex_CheckExact(obj) || - /* Basic sequence types */ - PyList_CheckExact(obj) || - PyTuple_CheckExact(obj) || - PyDict_CheckExact(obj) || - PyAnySet_CheckExact(obj) || - PyUnicode_CheckExact(obj) || - PyBytes_CheckExact(obj) || - PySlice_Check(obj)) { - - return 1; - } - - return 0; -} - /* * Convert NumPy stride to BLAS stride. Returns 0 if conversion cannot be done * (BLAS won't handle negative or zero strides the way we want). diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0506d3dee..f7a5f3deb 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -29,6 +29,8 @@ #include "alloc.h" #include <assert.h> +#include "get_attr_string.h" + /* * Reading from a file or a string. * diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index dc02b29f4..946dc542f 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1008,6 +1008,48 @@ array_getarray(PyArrayObject *self, PyObject *args) static PyObject * +array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *ufunc, *method_name, *normal_args, *ufunc_method; + PyObject *result = NULL; + + if (PyTuple_Size(args) < 2) { + PyErr_SetString(PyExc_TypeError, + "__array_ufunc__ requires at least 2 arguments"); + return NULL; + } + normal_args = PyTuple_GetSlice(args, 2, PyTuple_GET_SIZE(args)); + if (normal_args == NULL) { + return NULL; + } + /* ndarray cannot handle overrides itself */ + if (PyUFunc_WithOverride(normal_args, kwds, NULL)) { + result = Py_NotImplemented; + Py_INCREF(Py_NotImplemented); + goto cleanup; + } + + ufunc = PyTuple_GET_ITEM(args, 0); + method_name = PyTuple_GET_ITEM(args, 1); + /* + * TODO(?): call into UFunc code at a later point, since here arguments are + * already normalized and we do not have to look for __array_ufunc__ again. + */ + ufunc_method = PyObject_GetAttr(ufunc, method_name); + if (ufunc_method == NULL) { + goto cleanup; + } + result = PyObject_Call(ufunc_method, normal_args, kwds); + Py_DECREF(ufunc_method); + +cleanup: + Py_DECREF(normal_args); + /* no need to DECREF borrowed references ufunc and method_name */ + return result; +} + + +static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) { NPY_ORDER order = NPY_CORDER; @@ -2030,11 +2072,7 @@ array_cumprod(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; - PyObject *a = (PyObject *)self, *b, *o = Py_None; - PyObject *newargs; + PyObject *a = (PyObject *)self, *b, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"b", "out", NULL }; @@ -2043,36 +2081,15 @@ array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - if ((newargs = PyTuple_Pack(3, a, b, o)) == NULL) { - return NULL; - } - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", - newargs, NULL, &override, 2); - Py_DECREF(newargs); - - if (errval) { - return NULL; - } - else if (override) { - return override; - } - - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o != NULL) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, + "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, b, (PyArrayObject *)o); return PyArray_Return(ret); @@ -2471,6 +2488,9 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { {"__array_wrap__", (PyCFunction)array_wraparray, METH_VARARGS, NULL}, + {"__array_ufunc__", + (PyCFunction)array_ufunc, + METH_VARARGS | METH_KEYWORDS, NULL}, /* for the sys module */ {"__sizeof__", diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 2fdabb187..73ba8c5c4 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -61,6 +61,8 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "compiled_base.h" #include "mem_overlap.h" +#include "get_attr_string.h" + /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -2177,41 +2179,22 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args) static PyObject * array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; PyObject *v, *a, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"a", "b", "out", NULL }; - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwds, - &override, 2); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", kwlist, &a, &v, &o)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", + kwlist, &a, &v, &o)) { return NULL; } - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o != NULL) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, v, (PyArrayObject *)o); return PyArray_Return(ret); @@ -2344,14 +2327,10 @@ fail: * out: Either NULL, or an array into which the output should be placed. * * Returns NULL on error. - * Returns NotImplemented on priority override. */ static PyObject * array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) { - static PyObject *matmul = NULL; - int errval; - PyObject *override = NULL; PyObject *in1, *in2, *out = NULL; char* kwlist[] = {"a", "b", "out", NULL }; PyArrayObject *ap1, *ap2, *ret = NULL; @@ -2362,39 +2341,25 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) char *subscripts; PyArrayObject *ops[2]; - npy_cache_import("numpy.core.multiarray", "matmul", &matmul); - if (matmul == NULL) { - return NULL; - } - - errval = PyUFunc_CheckOverride((PyUFuncObject*)matmul, "__call__", - args, kwds, &override, 2); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matmul", kwlist, &in1, &in2, &out)) { return NULL; } - if (out == Py_None) { - out = NULL; - } - if (out != NULL && !PyArray_Check(out)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (out != NULL) { + if (out == Py_None) { + out = NULL; + } + else if (!PyArray_Check(out)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } dtype = PyArray_DescrFromObject(in1, NULL); dtype = PyArray_DescrFromObject(in2, dtype); if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, - "Cannot find a common data type."); + PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); return NULL; } typenum = dtype->type_num; @@ -2402,7 +2367,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (typenum == NPY_OBJECT) { /* matmul is not currently implemented for object arrays */ PyErr_SetString(PyExc_TypeError, - "Object arrays are not currently supported"); + "Object arrays are not currently supported"); Py_DECREF(dtype); return NULL; } @@ -2424,7 +2389,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (PyArray_NDIM(ap1) == 0 || PyArray_NDIM(ap2) == 0) { /* Scalars are rejected */ PyErr_SetString(PyExc_ValueError, - "Scalar operands are not allowed, use '*' instead"); + "Scalar operands are not allowed, use '*' instead"); return NULL; } @@ -4530,7 +4495,7 @@ intern_strings(void) npy_ma_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); - npy_ma_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index f0b5637d2..ad1d43178 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -14,6 +14,8 @@ #include "number.h" #include "temp_elide.h" +#include "binop_override.h" + /************************************************************************* **************** Implement Number Protocol **************************** *************************************************************************/ @@ -87,88 +89,6 @@ PyArray_SetNumericOps(PyObject *dict) (PyDict_SetItemString(dict, #op, n_ops.op)==-1)) \ goto fail; -static int -has_ufunc_attr(PyObject * obj) { - /* attribute check is expensive for scalar operations, avoid if possible */ - if (PyArray_CheckExact(obj) || PyArray_CheckAnyScalarExact(obj) || - _is_basic_python_type(obj)) { - return 0; - } - else { - return PyObject_HasAttrString(obj, "__numpy_ufunc__"); - } -} - -/* - * Check whether the operation needs to be forwarded to the right-hand binary - * operation. - * - * This is the case when all of the following conditions apply: - * - * (i) the other object defines __numpy_ufunc__ - * (ii) the other object defines the right-hand operation __r*__ - * (iii) Python hasn't already called the right-hand operation - * [occurs if the other object is a strict subclass provided - * the operation is not in-place] - * - * An additional check is made in GIVE_UP_IF_HAS_RIGHT_BINOP macro below: - * - * (iv) other.__class__.__r*__ is not self.__class__.__r*__ - * - * This is needed, because CPython does not call __rmul__ if - * the tp_number slots of the two objects are the same. - * - * This always prioritizes the __r*__ routines over __numpy_ufunc__, independent - * of whether the other object is an ndarray subclass or not. - */ - -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int inplace_op) -{ - if (other == NULL || - self == NULL || - Py_TYPE(self) == Py_TYPE(other) || - PyArray_CheckExact(other) || - PyArray_CheckAnyScalar(other)) { - /* - * Quick cases - */ - return 0; - } - if ((!inplace_op && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) || - !PyArray_Check(self)) { - /* - * Bail out if Python would already have called the right-hand - * operation. - */ - return 0; - } - if (has_ufunc_attr(other) && - PyObject_HasAttrString(other, right_name)) { - return 1; - } - else { - return 0; - } -} - -/* In pure-Python, SAME_SLOTS can be replaced by - getattr(m1, op_name) is getattr(m2, op_name) */ -#define SAME_SLOTS(m1, m2, slot_name) \ - (Py_TYPE(m1)->tp_as_number != NULL && Py_TYPE(m2)->tp_as_number != NULL && \ - Py_TYPE(m1)->tp_as_number->slot_name == Py_TYPE(m2)->tp_as_number->slot_name) - -#define GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, left_name, right_name, inplace, slot_name) \ - do { \ - if (needs_right_binop_forward((PyObject *)m1, m2, right_name, inplace) && \ - (inplace || !SAME_SLOTS(m1, m2, slot_name))) { \ - Py_INCREF(Py_NotImplemented); \ - return Py_NotImplemented; \ - } \ - } while (0) - - /*NUMPY_API Get dictionary showing number functions that all arrays will use */ @@ -289,36 +209,18 @@ PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, NPY_NO_EXPORT PyObject * PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op) { + /* + * I suspect that the next few lines are buggy and cause NotImplemented to + * be returned at weird times... but if we raise an error here, then + * *everything* breaks. (Like, 'arange(10) + 1' and just + * 'repr(arange(10))' both blow up with an error here.) Not sure what's + * going on with that, but I'll leave it alone for now. - njs, 2015-06-21 + */ if (op == NULL) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_Check(m2) && !has_ufunc_attr(m2)) { - /* - * Catch priority inversion and punt, but only if it's guaranteed - * that we were called through m1 and the other guy is not an array - * at all. Note that some arrays need to pass through here even - * with priorities inverted, for example: float(17) * np.matrix(...) - * - * See also: - * - https://github.com/numpy/numpy/issues/3502 - * - https://github.com/numpy/numpy/issues/3503 - * - * NB: there's another copy of this code in - * numpy.ma.core.MaskedArray._delegate_binop - * which should possibly be updated when this is. - */ - double m1_prio = PyArray_GetPriority((PyObject *)m1, - NPY_SCALAR_PRIORITY); - double m2_prio = PyArray_GetPriority((PyObject *)m2, - NPY_SCALAR_PRIORITY); - if (m1_prio < m2_prio) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - } - return PyObject_CallFunctionObjArgs(op, m1, m2, NULL); } @@ -381,8 +283,9 @@ array_inplace_right_shift(PyArrayObject *m1, PyObject *m2); static PyObject * array_add(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__add__", "__radd__", 0, nb_add); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_add, array_add); if (try_binary_elide(m1, m2, &array_inplace_add, &res, 1)) { return res; } @@ -392,8 +295,9 @@ array_add(PyArrayObject *m1, PyObject *m2) static PyObject * array_subtract(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__sub__", "__rsub__", 0, nb_subtract); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_subtract, array_subtract); if (try_binary_elide(m1, m2, &array_inplace_subtract, &res, 0)) { return res; } @@ -403,8 +307,9 @@ array_subtract(PyArrayObject *m1, PyObject *m2) static PyObject * array_multiply(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mul__", "__rmul__", 0, nb_multiply); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, array_multiply); if (try_binary_elide(m1, m2, &array_inplace_multiply, &res, 1)) { return res; } @@ -415,8 +320,9 @@ array_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__div__", "__rdiv__", 0, nb_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_divide, array_divide); if (try_binary_elide(m1, m2, &array_inplace_divide, &res, 0)) { return res; } @@ -427,7 +333,7 @@ array_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mod__", "__rmod__", 0, nb_remainder); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_remainder, array_remainder); return PyArray_GenericBinaryFunction(m1, m2, n_ops.remainder); } @@ -443,8 +349,7 @@ array_matrix_multiply(PyArrayObject *m1, PyObject *m2) if (matmul == NULL) { return NULL; } - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__matmul__", "__rmatmul__", - 0, nb_matrix_multiply); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); return PyArray_GenericBinaryFunction(m1, m2, matmul); } @@ -458,11 +363,12 @@ array_inplace_matrix_multiply(PyArrayObject *m1, PyObject *m2) } #endif -/* Determine if object is a scalar and if so, convert the object - * to a double and place it in the out_exponent argument - * and return the "scalar kind" as a result. If the object is - * not a scalar (or if there are other error conditions) - * return NPY_NOSCALAR, and out_exponent is undefined. +/* + * Determine if object is a scalar and if so, convert the object + * to a double and place it in the out_exponent argument + * and return the "scalar kind" as a result. If the object is + * not a scalar (or if there are other error conditions) + * return NPY_NOSCALAR, and out_exponent is undefined. */ static NPY_SCALARKIND is_scalar_with_conversion(PyObject *o2, double* out_exponent) @@ -573,7 +479,8 @@ fast_scalar_power(PyArrayObject *a1, PyObject *o2, int inplace) if (inplace || can_elide_temp_unary(a1)) { return PyArray_GenericInplaceUnaryFunction(a1, fastop); - } else { + } + else { return PyArray_GenericUnaryFunction(a1, fastop); } } @@ -615,12 +522,14 @@ static PyObject * array_power(PyArrayObject *a1, PyObject *o2, PyObject *modulo) { PyObject *value; + if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__pow__", "__rpow__", 0, nb_power); + + BINOP_GIVE_UP_IF_NEEDED(a1, o2, nb_power, array_power); value = fast_scalar_power(a1, o2, 0); if (!value) { value = PyArray_GenericBinaryFunction(a1, o2, n_ops.power); @@ -659,8 +568,9 @@ array_invert(PyArrayObject *m1) static PyObject * array_left_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__lshift__", "__rlshift__", 0, nb_lshift); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_lshift, array_left_shift); if (try_binary_elide(m1, m2, &array_inplace_left_shift, &res, 0)) { return res; } @@ -670,8 +580,9 @@ array_left_shift(PyArrayObject *m1, PyObject *m2) static PyObject * array_right_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__rshift__", "__rrshift__", 0, nb_rshift); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_rshift, array_right_shift); if (try_binary_elide(m1, m2, &array_inplace_right_shift, &res, 0)) { return res; } @@ -681,8 +592,9 @@ array_right_shift(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_and(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__and__", "__rand__", 0, nb_and); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_and, array_bitwise_and); if (try_binary_elide(m1, m2, &array_inplace_bitwise_and, &res, 1)) { return res; } @@ -692,8 +604,9 @@ array_bitwise_and(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_or(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__or__", "__ror__", 0, nb_or); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_or, array_bitwise_or); if (try_binary_elide(m1, m2, &array_inplace_bitwise_or, &res, 1)) { return res; } @@ -703,8 +616,9 @@ array_bitwise_or(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__xor__", "__rxor__", 0, nb_xor); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_xor, array_bitwise_xor); if (try_binary_elide(m1, m2, &array_inplace_bitwise_xor, &res, 1)) { return res; } @@ -714,21 +628,18 @@ array_bitwise_xor(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_add(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iadd__", "__radd__", 1, nb_inplace_add); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.add); } static PyObject * array_inplace_subtract(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__isub__", "__rsub__", 1, nb_inplace_subtract); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.subtract); } static PyObject * array_inplace_multiply(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imul__", "__rmul__", 1, nb_inplace_multiply); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.multiply); } @@ -736,7 +647,6 @@ array_inplace_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__idiv__", "__rdiv__", 1, nb_inplace_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.divide); } #endif @@ -744,7 +654,6 @@ array_inplace_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imod__", "__rmod__", 1, nb_inplace_remainder); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.remainder); } @@ -753,7 +662,6 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo { /* modulo is ignored! */ PyObject *value; - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__ipow__", "__rpow__", 1, nb_inplace_power); value = fast_scalar_power(a1, o2, 1); if (!value) { value = PyArray_GenericInplaceBinaryFunction(a1, o2, n_ops.power); @@ -764,43 +672,39 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo static PyObject * array_inplace_left_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ilshift__", "__rlshift__", 1, nb_inplace_lshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.left_shift); } static PyObject * array_inplace_right_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__irshift__", "__rrshift__", 1, nb_inplace_rshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.right_shift); } static PyObject * array_inplace_bitwise_and(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iand__", "__rand__", 1, nb_inplace_and); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_and); } static PyObject * array_inplace_bitwise_or(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ior__", "__ror__", 1, nb_inplace_or); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_or); } static PyObject * array_inplace_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ixor__", "__rxor__", 1, nb_inplace_xor); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_xor); } static PyObject * array_floor_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__floordiv__", "__rfloordiv__", 0, nb_floor_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_floor_divide, array_floor_divide); if (try_binary_elide(m1, m2, &array_inplace_floor_divide, &res, 0)) { return res; } @@ -810,8 +714,9 @@ array_floor_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_true_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__truediv__", "__rtruediv__", 0, nb_true_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_true_divide, array_true_divide); if (PyArray_CheckExact(m1) && (PyArray_ISFLOAT(m1) || PyArray_ISCOMPLEX(m1)) && try_binary_elide(m1, m2, &array_inplace_true_divide, &res, 0)) { @@ -823,7 +728,6 @@ array_true_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ifloordiv__", "__rfloordiv__", 1, nb_inplace_floor_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.floor_divide); } @@ -831,7 +735,6 @@ array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_true_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__itruediv__", "__rtruediv__", 1, nb_inplace_true_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.true_divide); } @@ -864,7 +767,8 @@ static PyObject * array_divmod(PyArrayObject *op1, PyObject *op2) { PyObject *divp, *modp, *result; - GIVE_UP_IF_HAS_RIGHT_BINOP(op1, op2, "__divmod__", "__rdivmod__", 0, nb_divmod); + + BINOP_GIVE_UP_IF_NEEDED(op1, op2, nb_divmod, array_divmod); divp = array_floor_divide(op1, op2); if (divp == NULL) { diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 0c8355e31..86f681c10 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -65,8 +65,4 @@ NPY_NO_EXPORT PyObject * PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, int rtype, PyArrayObject *out); -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int is_inplace); - #endif diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 7edf3b71d..f6bd5f5a7 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -27,6 +27,8 @@ #include <stdlib.h> +#include "binop_override.h" + NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[] = { {PyObject_HEAD_INIT(&PyBoolArrType_Type) 0}, {PyObject_HEAD_INIT(&PyBoolArrType_Type) 1}, @@ -151,63 +153,14 @@ gentype_free(PyObject *v) static PyObject * gentype_power(PyObject *m1, PyObject *m2, PyObject *modulo) { - PyObject *arr, *ret, *arg2; - char *msg="unsupported operand type(s) for ** or pow()"; - if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_IsScalar(m1, Generic)) { - if (PyArray_Check(m1)) { - ret = Py_TYPE(m1)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m2, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m2, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(m1, arr, Py_None); - Py_DECREF(arr); - } - return ret; - } - if (!PyArray_IsScalar(m2, Generic)) { - if (PyArray_Check(m2)) { - ret = Py_TYPE(m2)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m1, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m1, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, m2, Py_None); - Py_DECREF(arr); - } - return ret; - } - arr = arg2 = NULL; - arr = PyArray_FromScalar(m1, NULL); - arg2 = PyArray_FromScalar(m2, NULL); - if (arr == NULL || arg2 == NULL) { - Py_XDECREF(arr); - Py_XDECREF(arg2); - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, arg2, Py_None); - Py_DECREF(arr); - Py_DECREF(arg2); - return ret; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_power, gentype_power); + return PyArray_Type.tp_as_number->nb_power(m1, m2, Py_None); } static PyObject * @@ -249,6 +202,7 @@ gentype_generic_method(PyObject *self, PyObject *args, PyObject *kwds, static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } @@ -262,6 +216,7 @@ gentype_@name@(PyObject *m1, PyObject *m2) static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } /**end repeat**/ @@ -306,8 +261,8 @@ gentype_multiply(PyObject *m1, PyObject *m2) } return PySequence_Repeat(m2, repeat); } - /* All normal cases are handled by PyArray's multiply */ + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, gentype_multiply); return PyArray_Type.tp_as_number->nb_multiply(m1, m2); } diff --git a/numpy/core/src/private/binop_override.h b/numpy/core/src/private/binop_override.h new file mode 100644 index 000000000..8b4458777 --- /dev/null +++ b/numpy/core/src/private/binop_override.h @@ -0,0 +1,196 @@ +#ifndef __BINOP_OVERRIDE_H +#define __BINOP_OVERRIDE_H + +#include <string.h> +#include <Python.h> +#include "numpy/arrayobject.h" + +#include "get_attr_string.h" + +/* + * Logic for deciding when binops should return NotImplemented versus when + * they should go ahead and call a ufunc (or similar). + * + * The interaction between binop methods (ndarray.__add__ and friends) and + * ufuncs (which dispatch to __array_ufunc__) is both complicated in its own + * right, and also has complicated historical constraints. + * + * In the very old days, the rules were: + * - If the other argument has a higher __array_priority__, then return + * NotImplemented + * - Otherwise, call the corresponding ufunc. + * - And the ufunc might return NotImplemented based on some complex + * criteria that I won't reproduce here. + * + * Ufuncs no longer return NotImplemented (except in a few marginal situations + * which are being phased out -- see https://github.com/numpy/numpy/pull/5864) + * + * So as of 1.9, the effective rules were: + * - If the other argument has a higher __array_priority__, and is *not* a + * subclass of ndarray, then return NotImplemented. (If it is a subclass, + * the regular Python rules have already given it a chance to run; so if we + * are running, then it means the other argument has already returned + * NotImplemented and is basically asking us to take care of things.) + * - Otherwise call the corresponding ufunc. + * + * We would like to get rid of __array_priority__, and __array_ufunc__ + * provides a large part of a replacement for it. Once __array_ufunc__ is + * widely available, the simplest dispatch rules that might possibly work + * would be: + * - Always call the corresponding ufunc. + * + * But: + * - Doing this immediately would break backwards compatibility -- there's a + * lot of code using __array_priority__ out there. + * - It's not at all clear whether __array_ufunc__ actually is sufficient for + * all use cases. (See https://github.com/numpy/numpy/issues/5844 for lots + * of discussion of this, and in particular + * https://github.com/numpy/numpy/issues/5844#issuecomment-112014014 + * for a summary of some conclusions.) Also, python 3.6 defines a standard + * where setting a special-method name to None is a signal that that method + * cannot be used. + * + * So for 1.13, we are going to try the following rules. a.__add__(b) will + * be implemented as follows: + * - If b does not define __array_ufunc__, apply the legacy rule: + * - If not isinstance(b, a.__class__), and b.__array_priority__ is higher + * than a.__array_priority__, return NotImplemented + * - If b does define __array_ufunc__ but it is None, return NotImplemented + * - Otherwise, call the corresponding ufunc. + * + * For reversed operations like b.__radd__(a), and for in-place operations + * like a.__iadd__(b), we: + * - Call the corresponding ufunc + * + * Rationale for __radd__: This is because by the time the reversed operation + * is called, there are only two possibilities: The first possibility is that + * the current class is a strict subclass of the other class. In practice, the + * only way this will happen is if b is a strict subclass of a, and a is + * ndarray or a subclass of ndarray, and neither a nor b has actually + * overridden this method. In this case, Python will never call a.__add__ + * (because it's identical to b.__radd__), so we have no-one to defer to; + * there's no reason to return NotImplemented. The second possibility is that + * b.__add__ has already been called and returned NotImplemented. Again, in + * this case there is no point in returning NotImplemented. + * + * Rationale for __iadd__: In-place operations do not take all the trouble + * above, because if __iadd__ returns NotImplemented then Python will silently + * convert the operation into an out-of-place operation, i.e. 'a += b' will + * silently become 'a = a + b'. We don't want to allow this for arrays, + * because it will create unexpected memory allocations, break views, + * etc. + * + * In the future we might change these rules further. For example, we plan to + * eventually deprecate __array_priority__ in cases where __array_ufunc__ is + * not present. + */ + +static int +binop_override_forward_binop_should_defer(PyObject *self, PyObject *other) +{ + /* + * This function assumes that self.__binop__(other) is underway and + * implements the rules described above. Python's C API is funny, and + * makes it tricky to tell whether a given slot is called for __binop__ + * ("forward") or __rbinop__ ("reversed"). You are responsible for + * determining this before calling this function; it only provides the + * logic for forward binop implementations. + */ + + /* + * NB: there's another copy of this code in + * numpy.ma.core.MaskedArray._delegate_binop + * which should possibly be updated when this is. + */ + + PyObject *attr; + double self_prio, other_prio; + int defer; + /* + * attribute check is expensive for scalar operations, avoid if possible + */ + if (other == NULL || + self == NULL || + Py_TYPE(self) == Py_TYPE(other) || + PyArray_CheckExact(other) || + PyArray_CheckAnyScalarExact(other) || + _is_basic_python_type(other)) { + return 0; + } + /* + * Classes with __array_ufunc__ are living in the future, and only need to + * check whether __array_ufunc__ equals None. + */ + attr = PyArray_GetAttrString_SuppressException(other, "__array_ufunc__"); + if (attr) { + defer = (attr == Py_None); + Py_DECREF(attr); + return defer; + } + /* + * Otherwise, we need to check for the legacy __array_priority__. But if + * other.__class__ is a subtype of self.__class__, then it's already had + * a chance to run, so no need to defer to it. + */ + if(PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { + return 0; + } + self_prio = PyArray_GetPriority((PyObject *)self, NPY_SCALAR_PRIORITY); + other_prio = PyArray_GetPriority((PyObject *)other, NPY_SCALAR_PRIORITY); + return self_prio < other_prio; +} + +/* + * A CPython slot like ->tp_as_number->nb_add gets called for *both* forward + * and reversed operations. E.g. + * a + b + * may call + * a->tp_as_number->nb_add(a, b) + * and + * b + a + * may call + * a->tp_as_number->nb_add(b, a) + * and the only way to tell which is which is for a slot implementation 'f' to + * check + * arg1->tp_as_number->nb_add == f + * arg2->tp_as_number->nb_add == f + * If both are true, then CPython will as a special case only call the + * operation once (i.e., it performs both the forward and reversed binops + * simultaneously). This function is mostly intended for figuring out + * whether we are a forward binop that might want to return NotImplemented, + * and in the both-at-once case we never want to return NotImplemented, so in + * that case BINOP_IS_FORWARD returns false. + * + * This is modeled on the checks in CPython's typeobject.c SLOT1BINFULL + * macro. + */ +#define BINOP_IS_FORWARD(m1, m2, SLOT_NAME, test_func) \ + (Py_TYPE(m2)->tp_as_number != NULL && \ + (void*)(Py_TYPE(m2)->tp_as_number->SLOT_NAME) != (void*)(test_func)) + +#define BINOP_GIVE_UP_IF_NEEDED(m1, m2, slot_expr, test_func) \ + do { \ + if (BINOP_IS_FORWARD(m1, m2, slot_expr, test_func) && \ + binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +/* + * For rich comparison operations, it's impossible to distinguish + * between a forward comparison and a reversed/reflected + * comparison. So we assume they are all forward. This only works because the + * logic in binop_override_forward_binop_should_defer is essentially + * asymmetric -- you can never have two duck-array types that each decide to + * defer to the other. + */ +#define RICHCMP_GIVE_UP_IF_NEEDED(m1, m2) \ + do { \ + if (binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +#endif diff --git a/numpy/core/src/private/get_attr_string.h b/numpy/core/src/private/get_attr_string.h new file mode 100644 index 000000000..b32be28f7 --- /dev/null +++ b/numpy/core/src/private/get_attr_string.h @@ -0,0 +1,85 @@ +#ifndef __GET_ATTR_STRING_H +#define __GET_ATTR_STRING_H + +static NPY_INLINE int +_is_basic_python_type(PyObject * obj) +{ + if (obj == Py_None || + PyBool_Check(obj) || + /* Basic number types */ +#if !defined(NPY_PY3K) + PyInt_CheckExact(obj) || + PyString_CheckExact(obj) || +#endif + PyLong_CheckExact(obj) || + PyFloat_CheckExact(obj) || + PyComplex_CheckExact(obj) || + /* Basic sequence types */ + PyList_CheckExact(obj) || + PyTuple_CheckExact(obj) || + PyDict_CheckExact(obj) || + PyAnySet_CheckExact(obj) || + PyUnicode_CheckExact(obj) || + PyBytes_CheckExact(obj) || + PySlice_Check(obj)) { + + return 1; + } + + return 0; +} + +/* + * PyArray_GetAttrString_SuppressException: + * + * Stripped down version of PyObject_GetAttrString, + * avoids lookups for None, tuple, and List objects, + * and doesn't create a PyErr since this code ignores it. + * + * This can be much faster then PyObject_GetAttrString where + * exceptions are not used by caller. + * + * 'obj' is the object to search for attribute. + * + * 'name' is the attribute to search for. + * + * Returns attribute value on success, 0 on failure. + */ +static PyObject * +PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) +{ + PyTypeObject *tp = Py_TYPE(obj); + PyObject *res = (PyObject *)NULL; + + /* We do not need to check for special attributes on trivial types */ + if (_is_basic_python_type(obj)) { + return NULL; + } + + /* Attribute referenced by (char *)name */ + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(obj, name); + if (res == NULL) { + PyErr_Clear(); + } + } + /* Attribute referenced by (PyObject *)name */ + else if (tp->tp_getattro != NULL) { +#if defined(NPY_PY3K) + PyObject *w = PyUnicode_InternFromString(name); +#else + PyObject *w = PyString_InternFromString(name); +#endif + if (w == NULL) { + return (PyObject *)NULL; + } + res = (*tp->tp_getattro)(obj, w); + Py_DECREF(w); + if (res == NULL) { + PyErr_Clear(); + } + } + return res; +} + +#endif diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c new file mode 100644 index 000000000..b5cd46b89 --- /dev/null +++ b/numpy/core/src/private/ufunc_override.c @@ -0,0 +1,130 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + +#include "npy_pycompat.h" +#include "get_attr_string.h" +#include "npy_import.h" + +#include "ufunc_override.h" + +/* + * Check whether an object has __array_ufunc__ defined on its class and it + * is not the default, i.e., the object is not an ndarray, and its + * __array_ufunc__ is not the same as that of ndarray. + * + * Note that since this module is used with both multiarray and umath, we do + * not have access to PyArray_Type and therewith neither to PyArray_CheckExact + * nor to the default __array_ufunc__ method, so instead we import locally. + * TODO: Can this really not be done more smartly? + */ +static int +has_non_default_array_ufunc(PyObject *obj) +{ + static PyObject *ndarray = NULL; + static PyObject *ndarray_array_ufunc = NULL; + PyObject *cls_array_ufunc; + int non_default; + + /* on first entry, import and cache ndarray and its __array_ufunc__ */ + if (ndarray == NULL) { + npy_cache_import("numpy.core.multiarray", "ndarray", &ndarray); + ndarray_array_ufunc = PyObject_GetAttrString(ndarray, + "__array_ufunc__"); + } + + /* Fast return for ndarray */ + if ((PyObject *)Py_TYPE(obj) == ndarray) { + return 0; + } + /* does the class define __array_ufunc__? */ + cls_array_ufunc = PyArray_GetAttrString_SuppressException( + (PyObject *)Py_TYPE(obj), "__array_ufunc__"); + if (cls_array_ufunc == NULL) { + return 0; + } + /* is it different from ndarray.__array_ufunc__? */ + non_default = (cls_array_ufunc != ndarray_array_ufunc); + Py_DECREF(cls_array_ufunc); + return non_default; +} + +/* + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Return the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL) + * using borrowed references. + * + * returns -1 on failure. + */ +NPY_NO_EXPORT int +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override) +{ + int i; + + int nargs; + int nout_kwd = 0; + int out_kwd_is_tuple = 0; + int noa = 0; /* Number of overriding args.*/ + + PyObject *obj; + PyObject *out_kwd_obj = NULL; + /* + * Check inputs + */ + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, + "Internal Numpy error: call to PyUFunc_HasOverride " + "with non-tuple"); + goto fail; + } + nargs = PyTuple_GET_SIZE(args); + if (nargs > NPY_MAXARGS) { + PyErr_SetString(PyExc_TypeError, + "Internal Numpy error: too many arguments in call " + "to PyUFunc_HasOverride"); + goto fail; + } + /* be sure to include possible 'out' keyword argument. */ + if (kwds && PyDict_CheckExact(kwds)) { + out_kwd_obj = PyDict_GetItemString(kwds, "out"); + if (out_kwd_obj != NULL) { + out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); + if (out_kwd_is_tuple) { + nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); + } + else { + nout_kwd = 1; + } + } + } + + for (i = 0; i < nargs + nout_kwd; ++i) { + if (i < nargs) { + obj = PyTuple_GET_ITEM(args, i); + } + else { + if (out_kwd_is_tuple) { + obj = PyTuple_GET_ITEM(out_kwd_obj, i - nargs); + } + else { + obj = out_kwd_obj; + } + } + /* + * Now see if the object provides an __array_ufunc__. However, we should + * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as + * any ndarray subclass instances that did not override __array_ufunc__. + */ + if (has_non_default_array_ufunc(obj)) { + if (with_override != NULL) { + with_override[noa] = obj; + } + ++noa; + } + } + return noa; + +fail: + return -1; +} diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 59a90c770..15a932174 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -1,420 +1,15 @@ #ifndef __UFUNC_OVERRIDE_H #define __UFUNC_OVERRIDE_H -#include <npy_config.h> -#include "numpy/arrayobject.h" -#include "common.h" -#include <string.h> -#include "numpy/ufuncobject.h" -static void -normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds, - int nin) -{ - /* ufunc.__call__(*args, **kwds) */ - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); - - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - if (obj != NULL) { - Py_INCREF(obj); - PyDict_SetItemString(*normal_kwds, "signature", obj); - PyDict_DelItemString(*normal_kwds, "sig"); - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - - /* If we have more args than nin, they must be the output variables.*/ - if (nargs > nin) { - if ((nargs - nin) == 1) { - obj = PyTuple_GET_ITEM(args, nargs - 1); - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - obj = PyTuple_GetSlice(args, nin, nargs); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } - } -} - -static void -normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else if (i == 3) { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - /* keepdims */ - PyDict_SetItemString(*normal_kwds, "keepdims", obj); - } - } - return; -} - -static void -normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.accumulate(a[, axis, dtype, out]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} - -static void -normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ - int i; - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - /* a and indicies */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - } - else if (i == 1) { - /* Handled above, when i == 0. */ - continue; - } - else if (i == 2) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 3) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} - -static void -normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.outer(A, B) - * This has no kwds so we don't need to do any kwd stuff. - */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - return; -} - -static void -normalize_at_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.at(a, indices[, b]) */ - int nargs = PyTuple_GET_SIZE(args); - - *normal_args = PyTuple_GetSlice(args, 0, nargs); - return; -} +#include "npy_config.h" /* - * Check a set of args for the `__numpy_ufunc__` method. If more than one of - * the input arguments implements `__numpy_ufunc__`, they are tried in the - * order: subclasses before superclasses, otherwise left to right. The first - * routine returning something other than `NotImplemented` determines the - * result. If all of the `__numpy_ufunc__` operations returns `NotImplemented`, - * a `TypeError` is raised. - * - * Returns 0 on success and 1 on exception. On success, *result contains the - * result of the operation, if any. If *result is NULL, there is no override. + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Returns the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL). + * returns -1 on failure. */ -static int -PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result, - int nin) -{ - int i; - int override_pos; /* Position of override in args.*/ - int j; - - int nargs; - int nout_kwd = 0; - int out_kwd_is_tuple = 0; - int noa = 0; /* Number of overriding args.*/ - - PyObject *obj; - PyObject *out_kwd_obj = NULL; - PyObject *other_obj; - - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ - PyObject *normal_kwds = NULL; - - PyObject *with_override[NPY_MAXARGS]; - - /* Pos of each override in args */ - int with_override_pos[NPY_MAXARGS]; - - /* 2016-01-29: Disable for now in master -- can re-enable once details are - * sorted out. All commented bits are tagged NUMPY_UFUNC_DISABLED. -njs - */ - result = NULL; - return 0; - - /* - * Check inputs - */ - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: call to PyUFunc_CheckOverride " - "with non-tuple"); - goto fail; - } - nargs = PyTuple_GET_SIZE(args); - if (nargs > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: too many arguments in call " - "to PyUFunc_CheckOverride"); - goto fail; - } - - /* be sure to include possible 'out' keyword argument. */ - if ((kwds)&& (PyDict_CheckExact(kwds))) { - out_kwd_obj = PyDict_GetItemString(kwds, "out"); - if (out_kwd_obj != NULL) { - out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); - if (out_kwd_is_tuple) { - nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); - } - else { - nout_kwd = 1; - } - } - } - - for (i = 0; i < nargs + nout_kwd; ++i) { - if (i < nargs) { - obj = PyTuple_GET_ITEM(args, i); - } - else { - if (out_kwd_is_tuple) { - obj = PyTuple_GET_ITEM(out_kwd_obj, i-nargs); - } - else { - obj = out_kwd_obj; - } - } - /* - * TODO: could use PyArray_GetAttrString_SuppressException if it - * weren't private to multiarray.so - */ - if (PyArray_CheckExact(obj) || PyArray_IsScalar(obj, Generic) || - _is_basic_python_type(obj)) { - continue; - } - if (PyObject_HasAttrString(obj, "__numpy_ufunc__")) { - with_override[noa] = obj; - with_override_pos[noa] = i; - ++noa; - } - } - - /* No overrides, bail out.*/ - if (noa == 0) { - *result = NULL; - return 0; - } - - method_name = PyUString_FromString(method); - if (method_name == NULL) { - goto fail; - } - - /* - * Normalize ufunc arguments. - */ - - /* Build new kwds */ - if (kwds && PyDict_CheckExact(kwds)) { - normal_kwds = PyDict_Copy(kwds); - } - else { - normal_kwds = PyDict_New(); - } - if (normal_kwds == NULL) { - goto fail; - } - - /* decide what to do based on the method. */ - /* ufunc.__call__ */ - if (strcmp(method, "__call__") == 0) { - normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); - } - - /* ufunc.reduce */ - else if (strcmp(method, "reduce") == 0) { - normalize_reduce_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.accumulate */ - else if (strcmp(method, "accumulate") == 0) { - normalize_accumulate_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.reduceat */ - else if (strcmp(method, "reduceat") == 0) { - normalize_reduceat_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.outer */ - else if (strcmp(method, "outer") == 0) { - normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.at */ - else if (strcmp(method, "at") == 0) { - normalize_at_args(ufunc, args, &normal_args, &normal_kwds); - } - - if (normal_args == NULL) { - goto fail; - } - - /* - * Call __numpy_ufunc__ functions in correct order - */ - while (1) { - PyObject *numpy_ufunc; - PyObject *override_args; - PyObject *override_obj; - - override_obj = NULL; - *result = NULL; - - /* Choose an overriding argument */ - for (i = 0; i < noa; i++) { - obj = with_override[i]; - if (obj == NULL) { - continue; - } - - /* Get the first instance of an overriding arg.*/ - override_pos = with_override_pos[i]; - override_obj = obj; - - /* Check for sub-types to the right of obj. */ - for (j = i + 1; j < noa; j++) { - other_obj = with_override[j]; - if (PyObject_Type(other_obj) != PyObject_Type(obj) && - PyObject_IsInstance(other_obj, - PyObject_Type(override_obj))) { - override_obj = NULL; - break; - } - } - - /* override_obj had no subtypes to the right. */ - if (override_obj) { - with_override[i] = NULL; /* We won't call this one again */ - break; - } - } - - /* Check if there is a method left to call */ - if (!override_obj) { - /* No acceptable override found. */ - PyErr_SetString(PyExc_TypeError, - "__numpy_ufunc__ not implemented for this type."); - goto fail; - } - - /* Call the override */ - numpy_ufunc = PyObject_GetAttrString(override_obj, - "__numpy_ufunc__"); - if (numpy_ufunc == NULL) { - goto fail; - } - - override_args = Py_BuildValue("OOiO", ufunc, method_name, - override_pos, normal_args); - if (override_args == NULL) { - Py_DECREF(numpy_ufunc); - goto fail; - } - - *result = PyObject_Call(numpy_ufunc, override_args, normal_kwds); - - Py_DECREF(numpy_ufunc); - Py_DECREF(override_args); - - if (*result == NULL) { - /* Exception occurred */ - goto fail; - } - else if (*result == Py_NotImplemented) { - /* Try the next one */ - Py_DECREF(*result); - continue; - } - else { - /* Good result. */ - break; - } - } - - /* Override found, return it. */ - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 0; - -fail: - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 1; -} +NPY_NO_EXPORT int +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override); #endif diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c new file mode 100644 index 000000000..1faf2568b --- /dev/null +++ b/numpy/core/src/umath/override.c @@ -0,0 +1,590 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + +#include "npy_pycompat.h" +#include "numpy/ufuncobject.h" +#include "npy_import.h" + +#include "ufunc_override.h" +#include "override.h" + +/* + * The following functions normalize ufunc arguments. The work done is similar + * to what is done inside ufunc_object by get_ufunc_arguments for __call__ and + * generalized ufuncs, and by PyUFunc_GenericReduction for the other methods. + * It would be good to unify (see gh-8892). + */ + +/* + * ufunc() and ufunc.outer() accept 'sig' or 'signature'; + * normalize to 'signature' + */ +static int +normalize_signature_keyword(PyObject *normal_kwds) +{ + PyObject* obj = PyDict_GetItemString(normal_kwds, "sig"); + if (obj != NULL) { + if (PyDict_GetItemString(normal_kwds, "signature")) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + return -1; + } + Py_INCREF(obj); + PyDict_SetItemString(normal_kwds, "signature", obj); + PyDict_DelItemString(normal_kwds, "sig"); + } + return 0; +} + +static int +normalize___call___args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.__call__(*args, **kwds) + */ + int i; + int not_all_none; + int nin = ufunc->nin; + int nout = ufunc->nout; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "ufunc() missing %d of %d required positional argument(s)", + nin - nargs, nin); + return -1; + } + if (nargs > nin+nout) { + PyErr_Format(PyExc_TypeError, + "ufunc() takes from %d to %d arguments but %d were given", + nin, nin+nout, nargs); + return -1; + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* If we have more args than nin, they must be the output variables.*/ + if (nargs > nin) { + if(PyDict_GetItemString(*normal_kwds, "out")) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('out') and position (%d)", + nin); + return -1; + } + for (i = nin; i < nargs; i++) { + not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); + if (not_all_none) { + break; + } + } + if (not_all_none) { + if (nargs - nin == nout) { + obj = PyTuple_GetSlice(args, nin, nargs); + } + else { + PyObject *item; + + obj = PyTuple_New(nout); + if (obj == NULL) { + return -1; + } + for (i = 0; i < nout; i++) { + if (i + nin < nargs) { + item = PyTuple_GET_ITEM(args, nin+i); + } + else { + item = Py_None; + } + Py_INCREF(item); + PyTuple_SET_ITEM(obj, i, item); + } + } + PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); + } + } + /* finally, ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); +} + +static int +normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduce(a[, axis, dtype, out, keepdims]) + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + if (nargs < 1 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduce() takes from 1 to 5 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.accumulate(a[, axis, dtype, out]) + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + if (nargs < 1 || nargs > 4) { + PyErr_Format(PyExc_TypeError, + "ufunc.accumulate() takes from 1 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduceat(a, indicies[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ + int i; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; + + if (nargs < 2 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduceat() takes from 2 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } + /* a and indicies */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + if (*normal_args == NULL) { + return -1; + } + + for (i = 2; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 4) { + obj = PyTuple_GetSlice(args, 4, 5); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 4) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.outer(*args, **kwds) + * all positional arguments should be inputs. + * for the keywords, we only need to check 'sig' vs 'signature'. + */ + int nin = ufunc->nin; + int nargs = PyTuple_GET_SIZE(args); + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() missing %d of %d required positional " + "argument(s)", nin - nargs, nin); + return -1; + } + if (nargs > nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() takes %d arguments but %d were given", + nin, nargs); + return -1; + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); +} + +static int +normalize_at_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.at(a, indices[, b]) */ + int nargs = PyTuple_GET_SIZE(args); + + if (nargs < 2 || nargs > 3) { + PyErr_Format(PyExc_TypeError, + "ufunc.at() takes from 2 to 3 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, nargs); + return (*normal_args == NULL); +} + +/* + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. + * + * Returns 0 on success and 1 on exception. On success, *result contains the + * result of the operation, if any. If *result is NULL, there is no override. + */ +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result) +{ + int i; + int j; + int status; + + int noa; + PyObject *with_override[NPY_MAXARGS]; + + PyObject *obj; + PyObject *other_obj; + PyObject *out; + + PyObject *method_name = NULL; + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + + PyObject *override_args = NULL; + Py_ssize_t len; + + /* + * Check inputs for overrides + */ + noa = PyUFunc_WithOverride(args, kwds, with_override); + /* No overrides, bail out.*/ + if (noa == 0) { + *result = NULL; + return 0; + } + + /* + * Normalize ufunc arguments. + */ + + /* Build new kwds */ + if (kwds && PyDict_CheckExact(kwds)) { + + /* ensure out is always a tuple */ + normal_kwds = PyDict_Copy(kwds); + out = PyDict_GetItemString(normal_kwds, "out"); + if (out != NULL) { + int nout = ufunc->nout; + + if (PyTuple_Check(out)) { + int all_none = 1; + + if (PyTuple_GET_SIZE(out) != nout) { + PyErr_Format(PyExc_TypeError, + "The 'out' tuple must have exactly " + "%d entries: one per ufunc output", nout); + goto fail; + } + for (i = 0; i < PyTuple_GET_SIZE(out); i++) { + all_none = (PyTuple_GET_ITEM(out, i) == Py_None); + if (!all_none) { + break; + } + } + if (all_none) { + PyDict_DelItemString(normal_kwds, "out"); + } + } + else { + /* not a tuple */ + if (nout > 1 && DEPRECATE("passing a single argument to the " + "'out' keyword argument of a " + "ufunc with\n" + "more than one output will " + "result in an error in the " + "future") < 0) { + /* + * If the deprecation is removed, also remove the loop + * below setting tuple items to None (but keep this future + * error message.) + */ + PyErr_SetString(PyExc_TypeError, + "'out' must be a tuple of arguments"); + goto fail; + } + if (out != Py_None) { + /* not already a tuple and not None */ + PyObject *out_tuple = PyTuple_New(nout); + + if (out_tuple == NULL) { + goto fail; + } + for (i = 1; i < nout; i++) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(out_tuple, i, Py_None); + } + /* out was borrowed ref; make it permanent */ + Py_INCREF(out); + /* steals reference */ + PyTuple_SET_ITEM(out_tuple, 0, out); + PyDict_SetItemString(normal_kwds, "out", out_tuple); + Py_DECREF(out_tuple); + } + else { + /* out=None; remove it */ + PyDict_DelItemString(normal_kwds, "out"); + } + } + } + } + else { + normal_kwds = PyDict_New(); + } + if (normal_kwds == NULL) { + goto fail; + } + + /* decide what to do based on the method. */ + + /* ufunc.__call__ */ + if (strcmp(method, "__call__") == 0) { + status = normalize___call___args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduce */ + else if (strcmp(method, "reduce") == 0) { + status = normalize_reduce_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.accumulate */ + else if (strcmp(method, "accumulate") == 0) { + status = normalize_accumulate_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduceat */ + else if (strcmp(method, "reduceat") == 0) { + status = normalize_reduceat_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.outer */ + else if (strcmp(method, "outer") == 0) { + status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + } + /* ufunc.at */ + else if (strcmp(method, "at") == 0) { + status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + } + /* unknown method */ + else { + PyErr_Format(PyExc_TypeError, + "Internal Numpy error: unknown ufunc method '%s' in call " + "to PyUFunc_CheckOverride", method); + status = -1; + } + if (status != 0) { + Py_XDECREF(normal_args); + goto fail; + } + + len = PyTuple_GET_SIZE(normal_args); + override_args = PyTuple_New(len + 2); + if (override_args == NULL) { + goto fail; + } + + Py_INCREF(ufunc); + /* PyTuple_SET_ITEM steals reference */ + PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); + method_name = PyUString_FromString(method); + if (method_name == NULL) { + goto fail; + } + Py_INCREF(method_name); + PyTuple_SET_ITEM(override_args, 1, method_name); + for (i = 0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(normal_args, i); + + Py_INCREF(item); + PyTuple_SET_ITEM(override_args, i + 2, item); + } + Py_DECREF(normal_args); + + /* Call __array_ufunc__ functions in correct order */ + while (1) { + PyObject *array_ufunc; + PyObject *override_obj; + + override_obj = NULL; + *result = NULL; + + /* Choose an overriding argument */ + for (i = 0; i < noa; i++) { + obj = with_override[i]; + if (obj == NULL) { + continue; + } + + /* Get the first instance of an overriding arg.*/ + override_obj = obj; + + /* Check for sub-types to the right of obj. */ + for (j = i + 1; j < noa; j++) { + other_obj = with_override[j]; + if (other_obj != NULL && + PyObject_Type(other_obj) != PyObject_Type(obj) && + PyObject_IsInstance(other_obj, + PyObject_Type(override_obj))) { + override_obj = NULL; + break; + } + } + + /* override_obj had no subtypes to the right. */ + if (override_obj) { + /* We won't call this one again */ + with_override[i] = NULL; + break; + } + } + + /* Check if there is a method left to call */ + if (!override_obj) { + /* No acceptable override found. */ + static PyObject *errmsg_formatter = NULL; + PyObject *errmsg; + + npy_cache_import("numpy.core._internal", + "array_ufunc_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + errmsg = PyObject_Call(errmsg_formatter, override_args, + normal_kwds); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } + goto fail; + } + + /* Access the override */ + array_ufunc = PyObject_GetAttrString(override_obj, + "__array_ufunc__"); + if (array_ufunc == NULL) { + goto fail; + } + + /* If None, try next one (i.e., as if it returned NotImplemented) */ + if (array_ufunc == Py_None) { + Py_DECREF(array_ufunc); + continue; + } + + *result = PyObject_Call(array_ufunc, override_args, normal_kwds); + Py_DECREF(array_ufunc); + + if (*result == NULL) { + /* Exception occurred */ + goto fail; + } + else if (*result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(*result); + continue; + } + else { + /* Good result. */ + break; + } + } + + /* Override found, return it. */ + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_DECREF(override_args); + return 0; + +fail: + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_XDECREF(override_args); + return 1; +} diff --git a/numpy/core/src/umath/override.h b/numpy/core/src/umath/override.h new file mode 100644 index 000000000..68f3c6ef0 --- /dev/null +++ b/numpy/core/src/umath/override.h @@ -0,0 +1,11 @@ +#ifndef _NPY_UMATH_OVERRIDE_H +#define _NPY_UMATH_OVERRIDE_H + +#include "npy_config.h" +#include "numpy/ufuncobject.h" + +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result); +#endif diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index 723ee998a..eb75b7375 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -23,6 +23,8 @@ #include "numpy/halffloat.h" #include "templ_common.h" +#include "binop_override.h" + /* Basic operations: * * BINARY: @@ -827,6 +829,8 @@ static PyObject * int first; #endif + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_@oper@, @name@_@oper@); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -964,6 +968,8 @@ static PyObject * int first; @type@ out = {@zero@, @zero@}; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1041,6 +1047,8 @@ static PyObject * PyObject *ret; @type@ arg1, arg2, out; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1102,6 +1110,9 @@ static PyObject * int first; @type@ out = @zero@; + + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1506,6 +1517,8 @@ static PyObject* npy_@name@ arg1, arg2; int out=0; + RICHCMP_GIVE_UP_IF_NEEDED(self, other); + switch(_@name@_convert2_to_ctypes(self, &arg1, other, &arg2)) { case 0: break; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 22a73e6ba..137a93781 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -44,7 +44,7 @@ #include "mem_overlap.h" #include "ufunc_object.h" -#include "ufunc_override.h" +#include "override.h" /********** PRINTF DEBUG TRACING **************/ #define NPY_UF_DBG_TRACING 0 @@ -4370,8 +4370,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) mps[i] = NULL; } - errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override, - ufunc->nin); + errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override); if (errval) { return NULL; } @@ -5068,6 +5067,14 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *new_args, *tmp; PyObject *shape1, *shape2, *newshape; + errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); + if (errval) { + return NULL; + } + else if (override) { + return override; + } + if (ufunc->core_enabled) { PyErr_Format(PyExc_TypeError, "method outer is not allowed in ufunc with non-trivial"\ @@ -5087,15 +5094,6 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return NULL; } - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override, 0); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - tmp = PySequence_GetItem(args, 0); if (tmp == NULL) { return NULL; @@ -5165,8 +5163,7 @@ ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override); if (errval) { return NULL; } @@ -5182,8 +5179,7 @@ ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override); if (errval) { return NULL; } @@ -5199,8 +5195,7 @@ ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override); if (errval) { return NULL; } @@ -5264,8 +5259,7 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) char * err_msg = NULL; NPY_BEGIN_THREADS_DEF; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override); if (errval) { return NULL; } diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 2419c31f8..1a6cee030 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -266,7 +266,7 @@ intern_strings(void) npy_um_str_array_prepare = PyUString_InternFromString("__array_prepare__"); npy_um_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_um_str_array_finalize = PyUString_InternFromString("__array_finalize__"); - npy_um_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_um_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_um_str_pyvals_name = PyUString_InternFromString(UFUNC_PYVALS_NAME); return npy_um_str_out && npy_um_str_subok && npy_um_str_array_prepare && diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 82b8fec14..6d9a8fdc3 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2406,27 +2406,6 @@ class TestMethods(TestCase): a.dot(b=b, out=c) assert_equal(c, np.dot(a, b)) - def test_dot_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return "A" - - class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return NotImplemented - - a = A() - b = B() - c = np.array([[1]]) - - assert_equal(np.dot(a, b), "A") - assert_equal(c.dot(a), "A") - assert_raises(TypeError, np.dot, b, c) - assert_raises(TypeError, c.dot, b) - def test_dot_type_mismatch(self): c = 1. A = np.array((1,1), dtype='i,i') @@ -2886,241 +2865,192 @@ class TestBinop(object): a = np.bool_() assert_(type(~(a & a)) is np.bool_) - def test_ufunc_override_rop_precedence(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - # Check that __rmul__ and other right-hand operations have - # precedence over __numpy_ufunc__ - + # ndarray.__rop__ always calls ufunc + # ndarray.__iop__ always calls ufunc + # ndarray.__op__, __rop__: + # - defer if other has __array_ufunc__ and it is None + # or other is not a subclass and has higher array priority + # - else, call ufunc + def test_ufunc_binop_interaction(self): + # Python method name (without underscores) + # -> (numpy ufunc, has_in_place_version, preferred_dtype) ops = { - '__add__': ('__radd__', np.add, True), - '__sub__': ('__rsub__', np.subtract, True), - '__mul__': ('__rmul__', np.multiply, True), - '__truediv__': ('__rtruediv__', np.true_divide, True), - '__floordiv__': ('__rfloordiv__', np.floor_divide, True), - '__mod__': ('__rmod__', np.remainder, True), - '__divmod__': ('__rdivmod__', None, False), - '__pow__': ('__rpow__', np.power, True), - '__lshift__': ('__rlshift__', np.left_shift, True), - '__rshift__': ('__rrshift__', np.right_shift, True), - '__and__': ('__rand__', np.bitwise_and, True), - '__xor__': ('__rxor__', np.bitwise_xor, True), - '__or__': ('__ror__', np.bitwise_or, True), - '__ge__': ('__le__', np.less_equal, False), - '__gt__': ('__lt__', np.less, False), - '__le__': ('__ge__', np.greater_equal, False), - '__lt__': ('__gt__', np.greater, False), - '__eq__': ('__eq__', np.equal, False), - '__ne__': ('__ne__', np.not_equal, False), + 'add': (np.add, True, float), + 'sub': (np.subtract, True, float), + 'mul': (np.multiply, True, float), + 'truediv': (np.true_divide, True, float), + 'floordiv': (np.floor_divide, True, float), + 'mod': (np.remainder, True, float), + 'divmod': (None, False, float), + 'pow': (np.power, True, int), + 'lshift': (np.left_shift, True, int), + 'rshift': (np.right_shift, True, int), + 'and': (np.bitwise_and, True, int), + 'xor': (np.bitwise_xor, True, int), + 'or': (np.bitwise_or, True, int), + # 'ge': (np.less_equal, False), + # 'gt': (np.less, False), + # 'le': (np.greater_equal, False), + # 'lt': (np.greater, False), + # 'eq': (np.equal, False), + # 'ne': (np.not_equal, False), } - class OtherNdarraySubclass(np.ndarray): + class Coerced(Exception): pass - class OtherNdarraySubclassWithOverride(np.ndarray): - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def check(op_name, ndsubclass): - rop_name, np_op, has_iop = ops[op_name] - - if has_iop: - iop_name = '__i' + op_name[2:] - iop = getattr(operator, iop_name) - - if op_name == "__divmod__": - op = divmod - else: - op = getattr(operator, op_name) - - # Dummy class - def __init__(self, *a, **kw): - pass - - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def __op__(self, *other): - return "op" - - def __rop__(self, *other): - return "rop" - - if ndsubclass: - bases = (np.ndarray,) + def array_impl(self): + raise Coerced + + def op_impl(self, other): + return "forward" + + def rop_impl(self, other): + return "reverse" + + def iop_impl(self, other): + return "in-place" + + def array_ufunc_impl(self, ufunc, method, *args, **kwargs): + return ("__array_ufunc__", ufunc, method, args, kwargs) + + # Create an object with the given base, in the given module, with a + # bunch of placeholder __op__ methods, and optionally a + # __array_ufunc__ and __array_priority__. + def make_obj(base, array_priority=False, array_ufunc=False, + alleged_module="__main__"): + class_namespace = {"__array__": array_impl} + if array_priority is not False: + class_namespace["__array_priority__"] = array_priority + for op in ops: + class_namespace["__{0}__".format(op)] = op_impl + class_namespace["__r{0}__".format(op)] = rop_impl + class_namespace["__i{0}__".format(op)] = iop_impl + if array_ufunc is not False: + class_namespace["__array_ufunc__"] = array_ufunc + eval_namespace = {"base": base, + "class_namespace": class_namespace, + "__name__": alleged_module, + } + MyType = eval("type('MyType', (base,), class_namespace)", + eval_namespace) + if issubclass(MyType, np.ndarray): + # Use this range to avoid special case weirdnesses around + # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc. + return np.arange(3, 5).view(MyType) else: - bases = (object,) - - dct = {'__init__': __init__, - '__numpy_ufunc__': __numpy_ufunc__, - op_name: __op__} - if op_name != rop_name: - dct[rop_name] = __rop__ - - cls = type("Rop" + rop_name, bases, dct) - - # Check behavior against both bare ndarray objects and a - # ndarray subclasses with and without their own override - obj = cls((1,), buffer=np.ones(1,)) - - arr_objs = [np.array([1]), - np.array([2]).view(OtherNdarraySubclass), - np.array([3]).view(OtherNdarraySubclassWithOverride), - ] - - for arr in arr_objs: - err_msg = "%r %r" % (op_name, arr,) - - # Check that ndarray op gives up if it sees a non-subclass - if not isinstance(obj, arr.__class__): - assert_equal(getattr(arr, op_name)(obj), - NotImplemented, err_msg=err_msg) - - # Check that the Python binops have priority - assert_equal(op(obj, arr), "op", err_msg=err_msg) - if op_name == rop_name: - assert_equal(op(arr, obj), "op", err_msg=err_msg) - else: - assert_equal(op(arr, obj), "rop", err_msg=err_msg) - - # Check that Python binops have priority also for in-place ops - if has_iop: - assert_equal(getattr(arr, iop_name)(obj), - NotImplemented, err_msg=err_msg) - if op_name != "__pow__": - # inplace pow requires the other object to be - # integer-like? - assert_equal(iop(arr, obj), "rop", err_msg=err_msg) - - # Check that ufunc call __numpy_ufunc__ normally - if np_op is not None: - assert_raises(AssertionError, np_op, arr, obj, - err_msg=err_msg) - assert_raises(AssertionError, np_op, obj, arr, - err_msg=err_msg) - - # Check all binary operations - for op_name in sorted(ops.keys()): - yield check, op_name, True - yield check, op_name, False - - def test_ufunc_override_rop_simple(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - # Check parts of the binary op overriding behavior in an - # explicit test case that is easier to understand. - class SomeClass(object): - def __numpy_ufunc__(self, *a, **kw): - return "ufunc" - - def __mul__(self, other): - return 123 - - def __rmul__(self, other): - return 321 - - def __rsub__(self, other): - return "no subs for me" - - def __gt__(self, other): - return "yep" - - def __lt__(self, other): - return "nope" - - class SomeClass2(SomeClass, np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): - if ufunc is np.multiply or ufunc is np.bitwise_and: - return "ufunc" - else: - inputs = list(inputs) - if i < len(inputs): - inputs[i] = np.asarray(self) - func = getattr(ufunc, method) - if ('out' in kw) and (kw['out'] is not None): - kw['out'] = np.asarray(kw['out']) - r = func(*inputs, **kw) - x = self.__class__(r.shape, dtype=r.dtype) - x[...] = r - return x - - class SomeClass3(SomeClass2): - def __rsub__(self, other): - return "sub for me" - - arr = np.array([0]) - obj = SomeClass() - obj2 = SomeClass2((1,), dtype=np.int_) - obj2[0] = 9 - obj3 = SomeClass3((1,), dtype=np.int_) - obj3[0] = 4 - - # obj is first, so should get to define outcome. - assert_equal(obj * arr, 123) - # obj is second, but has __numpy_ufunc__ and defines __rmul__. - assert_equal(arr * obj, 321) - # obj is second, but has __numpy_ufunc__ and defines __rsub__. - assert_equal(arr - obj, "no subs for me") - # obj is second, but has __numpy_ufunc__ and defines __lt__. - assert_equal(arr > obj, "nope") - # obj is second, but has __numpy_ufunc__ and defines __gt__. - assert_equal(arr < obj, "yep") - # Called as a ufunc, obj.__numpy_ufunc__ is used. - assert_equal(np.multiply(arr, obj), "ufunc") - # obj is second, but has __numpy_ufunc__ and defines __rmul__. - arr *= obj - assert_equal(arr, 321) - - # obj2 is an ndarray subclass, so CPython takes care of the same rules. - assert_equal(obj2 * arr, 123) - assert_equal(arr * obj2, 321) - assert_equal(arr - obj2, "no subs for me") - assert_equal(arr > obj2, "nope") - assert_equal(arr < obj2, "yep") - # Called as a ufunc, obj2.__numpy_ufunc__ is called. - assert_equal(np.multiply(arr, obj2), "ufunc") - # Also when the method is not overridden. - assert_equal(arr & obj2, "ufunc") - arr *= obj2 - assert_equal(arr, 321) - - obj2 += 33 - assert_equal(obj2[0], 42) - assert_equal(obj2.sum(), 42) - assert_(isinstance(obj2, SomeClass2)) - - # Obj3 is subclass that defines __rsub__. CPython calls it. - assert_equal(arr - obj3, "sub for me") - assert_equal(obj2 - obj3, "sub for me") - # obj3 is a subclass that defines __rmul__. CPython calls it. - assert_equal(arr * obj3, 321) - # But not here, since obj3.__rmul__ is obj2.__rmul__. - assert_equal(obj2 * obj3, 123) - # And of course, here obj3.__mul__ should be called. - assert_equal(obj3 * obj2, 123) - # obj3 defines __numpy_ufunc__ but obj3.__radd__ is obj2.__radd__. - # (and both are just ndarray.__radd__); see #4815. - res = obj2 + obj3 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass2)) - # Since obj3 is a subclass, it should have precedence, like CPython - # would give, even though obj2 has __numpy_ufunc__ and __radd__. - # See gh-4815 and gh-5747. - res = obj3 + obj2 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass3)) + return MyType() + + def check(obj, binop_override_expected, ufunc_override_expected, + check_scalar=True): + for op, (ufunc, has_inplace, dtype) in ops.items(): + check_objs = [np.arange(3, 5, dtype=dtype)] + if check_scalar: + check_objs.append(check_objs[0][0]) + for arr in check_objs: + arr_method = getattr(arr, "__{0}__".format(op)) + + def norm(result): + if op == "divmod": + assert_(isinstance(result, tuple)) + return result[0] + else: + return result + + if binop_override_expected: + assert_equal(arr_method(obj), NotImplemented) + elif ufunc_override_expected: + assert_equal(norm(arr_method(obj))[0], + "__array_ufunc__") + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + res = norm(arr_method(obj)) + assert_(res.__class__ is obj.__class__) + else: + assert_raises((TypeError, Coerced), + arr_method, obj) + + arr_rmethod = getattr(arr, "__r{0}__".format(op)) + if ufunc_override_expected: + res = norm(arr_rmethod(obj)) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + res = norm(arr_rmethod(obj)) + assert_(res.__class__ is obj.__class__) + else: + # __array_ufunc__ = "asdf" creates a TypeError + assert_raises((TypeError, Coerced), + arr_rmethod, obj) + + # array scalars don't have in-place operators + if has_inplace and isinstance(arr, np.ndarray): + arr_imethod = getattr(arr, "__i{0}__".format(op)) + if ufunc_override_expected: + res = arr_imethod(obj) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + assert_(type(res[-1]["out"]) is tuple) + assert_(res[-1]["out"][0] is arr) + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + assert_(arr_imethod(obj) is arr) + else: + assert_raises((TypeError, Coerced), + arr_imethod, obj) + + op_fn = getattr(operator, op, None) + if op_fn is None: + op_fn = getattr(operator, op + "_", None) + if op_fn is None: + op_fn = getattr(builtins, op) + assert_equal(op_fn(obj, arr), "forward") + if not isinstance(obj, np.ndarray): + if binop_override_expected: + assert_equal(op_fn(arr, obj), "reverse") + elif ufunc_override_expected: + assert_equal(norm(op_fn(arr, obj))[0], + "__array_ufunc__") + if ufunc_override_expected and ufunc is not None: + assert_equal(norm(ufunc(obj, arr))[0], + "__array_ufunc__") + + # No array priority, no numpy ufunc -> nothing called + check(make_obj(object), False, False) + # Negative array priority, no numpy ufunc -> nothing called + # (has to be very negative, because scalar priority is -1000000.0) + check(make_obj(object, array_priority=-2**30), False, False) + # Positive array priority, no numpy ufunc -> binops only + check(make_obj(object, array_priority=1), True, False) + # ndarray ignores array priority for ndarray subclasses + check(make_obj(np.ndarray, array_priority=1), False, False, + check_scalar=False) + # Positive array priority and numpy ufunc -> numpy ufunc only + check(make_obj(object, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + check(make_obj(np.ndarray, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + # array_ufunc set to None -> defer binops only + check(make_obj(object, array_ufunc=None), True, False) + check(make_obj(np.ndarray, array_ufunc=None), True, False, + check_scalar=False) def test_ufunc_override_normalize_signature(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # gh-5674 class SomeClass(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): return kw a = SomeClass() @@ -3133,58 +3063,63 @@ class TestBinop(object): assert_('sig' not in kw and 'signature' in kw) assert_equal(kw['signature'], 'ii->i') - def test_numpy_ufunc_index(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - + def test_array_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) + # This also checks implicitly that 'out' is always a tuple. class CheckIndex(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): - return i + def __array_ufunc__(self, ufunc, method, *inputs, **kw): + for i, a in enumerate(inputs): + if a is self: + return i + # calls below mean we must be in an output. + for j, a in enumerate(kw['out']): + if a is self: + return (j,) a = CheckIndex() dummy = np.arange(2.) # 1 input, 1 output assert_equal(np.sin(a), 0) - assert_equal(np.sin(dummy, a), 1) - assert_equal(np.sin(dummy, out=a), 1) - assert_equal(np.sin(dummy, out=(a,)), 1) + assert_equal(np.sin(dummy, a), (0,)) + assert_equal(np.sin(dummy, out=a), (0,)) + assert_equal(np.sin(dummy, out=(a,)), (0,)) assert_equal(np.sin(a, a), 0) assert_equal(np.sin(a, out=a), 0) assert_equal(np.sin(a, out=(a,)), 0) # 1 input, 2 outputs - assert_equal(np.modf(dummy, a), 1) - assert_equal(np.modf(dummy, None, a), 2) - assert_equal(np.modf(dummy, dummy, a), 2) - assert_equal(np.modf(dummy, out=a), 1) - assert_equal(np.modf(dummy, out=(a,)), 1) - assert_equal(np.modf(dummy, out=(a, None)), 1) - assert_equal(np.modf(dummy, out=(a, dummy)), 1) - assert_equal(np.modf(dummy, out=(None, a)), 2) - assert_equal(np.modf(dummy, out=(dummy, a)), 2) + assert_equal(np.modf(dummy, a), (0,)) + assert_equal(np.modf(dummy, None, a), (1,)) + assert_equal(np.modf(dummy, dummy, a), (1,)) + assert_equal(np.modf(dummy, out=(a, None)), (0,)) + assert_equal(np.modf(dummy, out=(a, dummy)), (0,)) + assert_equal(np.modf(dummy, out=(None, a)), (1,)) + assert_equal(np.modf(dummy, out=(dummy, a)), (1,)) assert_equal(np.modf(a, out=(dummy, a)), 0) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_equal(np.modf(dummy, out=a), (0,)) + assert_(w[0].category is DeprecationWarning) + assert_raises(TypeError, np.modf, dummy, out=(a,)) + # 2 inputs, 1 output assert_equal(np.add(a, dummy), 0) assert_equal(np.add(dummy, a), 1) - assert_equal(np.add(dummy, dummy, a), 2) + assert_equal(np.add(dummy, dummy, a), (0,)) assert_equal(np.add(dummy, a, a), 1) - assert_equal(np.add(dummy, dummy, out=a), 2) - assert_equal(np.add(dummy, dummy, out=(a,)), 2) + assert_equal(np.add(dummy, dummy, out=a), (0,)) + assert_equal(np.add(dummy, dummy, out=(a,)), (0,)) assert_equal(np.add(a, dummy, out=a), 0) def test_out_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # regression test for github bug 4753 class OutClass(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): if 'out' in kw: tmp_kw = kw.copy() tmp_kw.pop('out') func = getattr(ufunc, method) - kw['out'][...] = func(*inputs, **tmp_kw) + kw['out'][0][...] = func(*inputs, **tmp_kw) A = np.array([0]).view(OutClass) B = np.array([5]) @@ -5319,31 +5254,6 @@ class MatmulCommon(): res = self.matmul(m12, m21) assert_equal(res, tgt12_21) - def test_numpy_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - class A(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return "A" - - class B(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return NotImplemented - - a = A([1, 2]) - b = B([1, 2]) - c = np.ones(2) - assert_equal(self.matmul(a, b), "A") - assert_equal(self.matmul(b, a), "A") - assert_raises(TypeError, self.matmul, b, c) - class TestMatmul(MatmulCommon, TestCase): matmul = np.matmul diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 26e15539e..1d0518f88 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1225,7 +1225,7 @@ class TestUfunc(TestCase): # https://github.com/numpy/numpy/issues/4855 class MyA(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return getattr(ufunc, method)(*(input.view(np.ndarray) for input in inputs), **kwargs) a = np.arange(12.).reshape(4,3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index d3b379a52..41108ab5f 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -3,15 +3,18 @@ from __future__ import division, absolute_import, print_function import sys import platform import warnings +import fnmatch import itertools from numpy.testing.utils import _gen_alignment_data import numpy.core.umath as ncu +from numpy.core import umath_tests as ncu_tests import numpy as np from numpy.testing import ( TestCase, run_module_suite, assert_, assert_equal, assert_raises, - assert_array_equal, assert_almost_equal, assert_array_almost_equal, - dec, assert_allclose, assert_no_warnings, suppress_warnings + assert_raises_regex, assert_array_equal, assert_almost_equal, + assert_array_almost_equal, dec, assert_allclose, assert_no_warnings, + suppress_warnings ) @@ -1568,51 +1571,30 @@ class TestSpecialMethods(TestCase): assert_equal(ncu.maximum(a, B()), 0) assert_equal(ncu.maximum(a, C()), 0) - def test_ufunc_override_disabled(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - # This test should be removed when __numpy_ufunc__ is re-enabled. - - class MyArray(object): - def __numpy_ufunc__(self, *args, **kwargs): - self._numpy_ufunc_called = True - - my_array = MyArray() - real_array = np.ones(10) - assert_raises(TypeError, lambda: real_array + my_array) - assert_raises(TypeError, np.add, real_array, my_array) - assert not hasattr(my_array, "_numpy_ufunc_called") - - def test_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): - return self, func, method, pos, inputs, kwargs + def __array_ufunc__(self, func, method, *inputs, **kwargs): + return self, func, method, inputs, kwargs a = A() b = np.matrix([1]) res0 = np.multiply(a, b) - res1 = np.dot(a, b) + res1 = np.multiply(b, b, out=a) # self assert_equal(res0[0], a) assert_equal(res1[0], a) assert_equal(res0[1], np.multiply) - assert_equal(res1[1], np.dot) + assert_equal(res1[1], np.multiply) assert_equal(res0[2], '__call__') assert_equal(res1[2], '__call__') - assert_equal(res0[3], 0) - assert_equal(res1[3], 0) - assert_equal(res0[4], (a, b)) - assert_equal(res1[4], (a, b)) - assert_equal(res0[5], {}) - assert_equal(res1[5], {}) + assert_equal(res0[3], (a, b)) + assert_equal(res1[3], (b, b)) + assert_equal(res0[4], {}) + assert_equal(res1[4], {'out': (a,)}) def test_ufunc_override_mro(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return # Some multi arg functions for testing. def tres_mul(a, b, c): @@ -1626,23 +1608,23 @@ class TestSpecialMethods(TestCase): four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "A" class ASub(A): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "ASub" class B(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "B" class C(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented - class CSub(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + class CSub(C): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented a = A() @@ -1704,12 +1686,10 @@ class TestSpecialMethods(TestCase): assert_raises(TypeError, four_mul_ufunc, 1, c, c_sub, c) def test_ufunc_override_methods(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return self, ufunc, method, pos, inputs, kwargs + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self, ufunc, method, inputs, kwargs # __call__ a = A() @@ -1717,21 +1697,24 @@ class TestSpecialMethods(TestCase): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], '__call__') - assert_equal(res[3], 1) - assert_equal(res[4], (1, a)) - assert_equal(res[5], {'foo': 'bar', 'answer': 42}) + assert_equal(res[3], (1, a)) + assert_equal(res[4], {'foo': 'bar', 'answer': 42}) + + # __call__, wrong args + assert_raises(TypeError, np.multiply, a) + assert_raises(TypeError, np.multiply, a, a, a, a) + assert_raises(TypeError, np.multiply, a, a, sig='a', signature='a') # reduce, positional args res = np.multiply.reduce(a, 'axis0', 'dtype0', 'out0', 'keep0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) # reduce, kwargs res = np.multiply.reduce(a, axis='axis0', dtype='dtype0', out='out0', @@ -1739,23 +1722,32 @@ class TestSpecialMethods(TestCase): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) + + # reduce, output equal to None removed. + res = np.multiply.reduce(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.reduce(a, out=(None,)) + assert_equal(res[4], {}) + + # reduce, wrong args + assert_raises(TypeError, np.multiply.reduce, a, out=()) + assert_raises(TypeError, np.multiply.reduce, a, out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, 'axis0', axis='axis0') # accumulate, pos args res = np.multiply.accumulate(a, 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) # accumulate, kwargs res = np.multiply.accumulate(a, axis='axis0', dtype='dtype0', @@ -1763,22 +1755,33 @@ class TestSpecialMethods(TestCase): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) + + # accumulate, output equal to None removed. + res = np.multiply.accumulate(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.accumulate(a, out=(None,)) + assert_equal(res[4], {}) + + # accumulate, wrong args + assert_raises(TypeError, np.multiply.accumulate, a, out=()) + assert_raises(TypeError, np.multiply.accumulate, a, + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.accumulate, a, + 'axis0', axis='axis0') # reduceat, pos args res = np.multiply.reduceat(a, [4, 2], 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) # reduceat, kwargs res = np.multiply.reduceat(a, [4, 2], axis='axis0', dtype='dtype0', @@ -1786,39 +1789,55 @@ class TestSpecialMethods(TestCase): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) + + # reduceat, output equal to None removed. + res = np.multiply.reduceat(a, [4, 2], out=None) + assert_equal(res[4], {}) + res = np.multiply.reduceat(a, [4, 2], out=(None,)) + assert_equal(res[4], {}) + + # reduceat, wrong args + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], out=()) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + 'axis0', axis='axis0') # outer res = np.multiply.outer(a, 42) assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'outer') - assert_equal(res[3], 0) - assert_equal(res[4], (a, 42)) - assert_equal(res[5], {}) + assert_equal(res[3], (a, 42)) + assert_equal(res[4], {}) + + # outer, wrong args + assert_raises(TypeError, np.multiply.outer, a) + assert_raises(TypeError, np.multiply.outer, a, a, a, a) # at res = np.multiply.at(a, [4, 2], 'b0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'at') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2], 'b0')) + assert_equal(res[3], (a, [4, 2], 'b0')) + + # at, wrong args + assert_raises(TypeError, np.multiply.at, a) + assert_raises(TypeError, np.multiply.at, a, a, a, a) def test_ufunc_override_out(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs a = A() @@ -1830,12 +1849,12 @@ class TestSpecialMethods(TestCase): res4 = np.multiply(a, 4, 'out_arg') res5 = np.multiply(a, 5, out='out_arg') - assert_equal(res0['out'], 'out_arg') - assert_equal(res1['out'], 'out_arg') - assert_equal(res2['out'], 'out_arg') - assert_equal(res3['out'], 'out_arg') - assert_equal(res4['out'], 'out_arg') - assert_equal(res5['out'], 'out_arg') + assert_equal(res0['out'][0], 'out_arg') + assert_equal(res1['out'][0], 'out_arg') + assert_equal(res2['out'][0], 'out_arg') + assert_equal(res3['out'][0], 'out_arg') + assert_equal(res4['out'][0], 'out_arg') + assert_equal(res5['out'][0], 'out_arg') # ufuncs with multiple output modf and frexp. res6 = np.modf(a, 'out0', 'out1') @@ -1845,17 +1864,192 @@ class TestSpecialMethods(TestCase): assert_equal(res7['out'][0], 'out0') assert_equal(res7['out'][1], 'out1') + # While we're at it, check that default output is never passed on. + assert_(np.sin(a, None) == {}) + assert_(np.sin(a, out=None) == {}) + assert_(np.sin(a, out=(None,)) == {}) + assert_(np.modf(a, None) == {}) + assert_(np.modf(a, None, None) == {}) + assert_(np.modf(a, out=(None, None)) == {}) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_(np.modf(a, out=None) == {}) + assert_(w[0].category is DeprecationWarning) + + # don't give positional and output argument, or too many arguments. + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, np.multiply, a, b, 'one', out='two') + assert_raises(TypeError, np.multiply, a, b, 'one', 'two') + assert_raises(TypeError, np.multiply, a, b, out=('one', 'two')) + assert_raises(TypeError, np.multiply, a, out=()) + assert_raises(TypeError, np.modf, a, 'one', out=('two', 'three')) + assert_raises(TypeError, np.modf, a, 'one', 'two', 'three') + assert_raises(TypeError, np.modf, a, out=('one', 'two', 'three')) + assert_raises(TypeError, np.modf, a, out=('one',)) + def test_ufunc_override_exception(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, *a, **kwargs): + def __array_ufunc__(self, *a, **kwargs): raise ValueError("oops") a = A() - for func in [np.divide, np.dot]: - assert_raises(ValueError, func, a, a) + assert_raises(ValueError, np.negative, 1, out=a) + assert_raises(ValueError, np.negative, a) + assert_raises(ValueError, np.divide, 1., a) + + def test_ufunc_override_not_implemented(self): + + class A(object): + __array_ufunc__ = None + + msg = ("operand type(s) do not implement __array_ufunc__(" + "<ufunc 'negative'>, '__call__', <*>): 'A'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.negative(A()) + + msg = ("operand type(s) do not implement __array_ufunc__(" + "<ufunc 'add'>, '__call__', <*>, <object *>, out=(1,)): " + "'A', 'object', 'int'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.add(A(), object(), out=1) + + def test_gufunc_override(self): + # gufunc are just ufunc instances, but follow a different path, + # so check __array_ufunc__ overrides them properly. + class A(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self, ufunc, method, inputs, kwargs + + inner1d = ncu_tests.inner1d + a = A() + res = inner1d(a, a) + assert_equal(res[0], a) + assert_equal(res[1], inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (a, a)) + assert_equal(res[4], {}) + + res = inner1d(1, 1, out=a) + assert_equal(res[0], a) + assert_equal(res[1], inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (1, 1)) + assert_equal(res[4], {'out': (a,)}) + + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, inner1d, a, out='two') + assert_raises(TypeError, inner1d, a, a, 'one', out='two') + assert_raises(TypeError, inner1d, a, a, 'one', 'two') + assert_raises(TypeError, inner1d, a, a, out=('one', 'two')) + assert_raises(TypeError, inner1d, a, a, out=()) + + def test_ufunc_override_with_super(self): + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', None) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout + + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if results is NotImplemented: + return NotImplemented + + if method == 'at': + return + + if ufunc.nout == 1: + results = (results,) + + results = tuple((np.asarray(result).view(A) + if output is None else output) + for result, output in zip(results, outputs)) + if results and isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + + class B(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + if any(isinstance(input_, A) for input_ in inputs): + return "A!" + else: + return NotImplemented + + d = np.arange(5.) + # 1 input, 1 output + a = np.arange(5.).view(A) + b = np.sin(a) + check = np.sin(d) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0]}) + b = np.sin(d, out=(a,)) + assert_(np.all(check == b)) + assert_equal(b.info, {'outputs': [0]}) + assert_(b is a) + a = np.arange(5.).view(A) + b = np.sin(a, out=a) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0], 'outputs': [0]}) + + # 1 input, 2 outputs + a = np.arange(5.).view(A) + b1, b2 = np.modf(a) + assert_equal(b1.info, {'inputs': [0]}) + b1, b2 = np.modf(d, out=(None, a)) + assert_(b2 is a) + assert_equal(b1.info, {'outputs': [1]}) + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c1, c2 = np.modf(a, out=(a, b)) + assert_(c1 is a) + assert_(c2 is b) + assert_equal(c1.info, {'inputs': [0], 'outputs': [0, 1]}) + + # 2 input, 1 output + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c = np.add(a, b, out=a) + assert_(c is a) + assert_equal(c.info, {'inputs': [0, 1], 'outputs': [0]}) + # some tests with a non-ndarray subclass + a = np.arange(5.) + b = B() + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_raises(TypeError, np.add, a, b) + a = a.view(A) + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) == "A!") + assert_(np.add(a, b) == "A!") + class TestChoose(TestCase): def test_mixed(self): diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index b6c742a2b..36d8ff97d 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -1,5 +1,4 @@ -""" -============================= +"""============================= Subclassing ndarray in python ============================= @@ -220,8 +219,9 @@ where our object creation housekeeping usually goes. * For the explicit constructor call, our subclass will need to create a new ndarray instance of its own class. In practice this means that we, the authors of the code, will need to make a call to - ``ndarray.__new__(MySubClass,...)``, or do view casting of an existing - array (see below) + ``ndarray.__new__(MySubClass,...)``, a class-hierarchy prepared call to + ``super(MySubClass, cls).__new__(cls, ...)``, or do view casting of an + existing array (see below) * For view casting and new-from-template, the equivalent of ``ndarray.__new__(MySubClass,...`` is called, at the C level. @@ -237,7 +237,7 @@ The following code allows us to look at the call sequences and arguments: class C(np.ndarray): def __new__(cls, *args, **kwargs): print('In __new__ with class %s' % cls) - return np.ndarray.__new__(cls, *args, **kwargs) + return super(C, cls).__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): # in practice you probably will not need or want an __init__ @@ -275,7 +275,8 @@ The signature of ``__array_finalize__`` is:: def __array_finalize__(self, obj): -``ndarray.__new__`` passes ``__array_finalize__`` the new object, of our +One sees that the ``super`` call, which goes to +``ndarray.__new__``, passes ``__array_finalize__`` the new object, of our own class (``self``) as well as the object from which the view has been taken (``obj``). As you can see from the output above, the ``self`` is always a newly created instance of our subclass, and the type of ``obj`` @@ -303,13 +304,14 @@ Simple example - adding an extra attribute to ndarray class InfoArray(np.ndarray): def __new__(subtype, shape, dtype=float, buffer=None, offset=0, - strides=None, order=None, info=None): + strides=None, order=None, info=None): # Create the ndarray instance of our type, given the usual # ndarray input arguments. This will call the standard # ndarray constructor, but return an object of our type. # It also triggers a call to InfoArray.__array_finalize__ - obj = np.ndarray.__new__(subtype, shape, dtype, buffer, offset, strides, - order) + obj = super(InfoArray, subtype).__new__(subtype, shape, dtype, + buffer, offset, strides, + order) # set the new 'info' attribute to the value passed obj.info = info # Finally, we must return the newly created object: @@ -412,15 +414,162 @@ So: >>> v.info 'information' -.. _array-wrap: +.. _array-ufunc: + +``__array_ufunc__`` for ufuncs +------------------------------ + + .. versionadded:: 1.13 + +A subclass can override what happens when executing numpy ufuncs on it by +overriding the default ``ndarray.__array_ufunc__`` method. This method is +executed *instead* of the ufunc and should return either the result of the +operation, or :obj:`NotImplemented` if the operation requested is not +implemented. + +The signature of ``__array_ufunc__`` is:: + + def __array_ufunc__(ufunc, method, *inputs, **kwargs): -``__array_wrap__`` for ufuncs -------------------------------------------------------- + - *ufunc* is the ufunc object that was called. + - *method* is a string indicating how the Ufunc was called, either + ``"__call__"`` to indicate it was called directly, or one of its + :ref:`methods<ufuncs.methods>`: ``"reduce"``, ``"accumulate"``, + ``"reduceat"``, ``"outer"``, or ``"at"``. + - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *kwargs* contains any optional or keyword arguments passed to the + function. This includes any ``out`` arguments, which are always + contained in a tuple. + +A typical implementation would convert any inputs or ouputs that are +instances of one's own class, pass everything on to a superclass using +``super()``, and finally return the results after possible +back-conversion. An example, taken from the test case +``test_ufunc_override_with_super`` in ``core/tests/test_umath.py``, is the +following. + +.. testcode:: + + input numpy as np + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', None) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout + + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if results is NotImplemented: + return NotImplemented + + if method == 'at': + return + + if ufunc.nout == 1: + results = (results,) + + results = tuple((np.asarray(result).view(A) + if output is None else output) + for result, output in zip(results, outputs)) + if results and isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + +So, this class does not actually do anything interesting: it just +converts any instances of its own to regular ndarray (otherwise, we'd +get infinite recursion!), and adds an ``info`` dictionary that tells +which inputs and outputs it converted. Hence, e.g., + +>>> a = np.arange(5.).view(A) +>>> b = np.sin(a) +>>> b.info +{'inputs': [0]} +>>> b = np.sin(np.arange(5.), out=(a,)) +>>> b.info +{'outputs': [0]} +>>> a = np.arange(5.).view(A) +>>> b = np.ones(1).view(A) +>>> c = a + b +>>> c.info +{'inputs': [0, 1]} +>>> a += b +>>> a.info +{'inputs': [0, 1], 'outputs': [0]} + +Note that another approach would be to to use ``getattr(ufunc, +methods)(*inputs, **kwargs)`` instead of the ``super`` call. For this example, +the result would be identical, but there is a difference if another operand +also defines ``__array_ufunc__``. E.g., lets assume that we evalulate +``np.add(a, b)``, where ``b`` is an instance of another class ``B`` that has +an override. If you use ``super`` as in the example, +``ndarray.__array_ufunc__`` will notice that ``b`` has an override, which +means it cannot evaluate the result itself. Thus, it will return +`NotImplemented` and so will our class ``A``. Then, control will be passed +over to ``b``, which either knows how to deal with us and produces a result, +or does not and returns `NotImplemented`, raising a ``TypeError``. + +If instead, we replace our ``super`` call with ``getattr(ufunc, method)``, we +effectively do ``np.add(a.view(np.ndarray), b)``. Again, ``B.__array_ufunc__`` +will be called, but now it sees an ``ndarray`` as the other argument. Likely, +it will know how to handle this, and return a new instance of the ``B`` class +to us. Our example class is not set up to handle this, but it might well be +the best approach if, e.g., one were to re-implement ``MaskedArray`` using + ``__array_ufunc__``. + +As a final note: if the ``super`` route is suited to a given class, an +advantage of using it is that it helps in constructing class hierarchies. +E.g., suppose that our other class ``B`` also used the ``super`` in its +``__array_ufunc__`` implementation, and we created a class ``C`` that depended +on both, i.e., ``class C(A, B)`` (with, for simplicity, not another +``__array_ufunc__`` override). Then any ufunc on an instance of ``C`` would +pass on to ``A.__array_ufunc__``, the ``super`` call in ``A`` would go to +``B.__array_ufunc__``, and the ``super`` call in ``B`` would go to +``ndarray.__array_ufunc__``, thus allowing ``A`` and ``B`` to collaborate. + +.. _array-wrap: -``__array_wrap__`` gets called at the end of numpy ufuncs and other numpy -functions, to allow a subclass to set the type of the return value -and update attributes and metadata. Let's show how this works with an example. -First we make the same subclass as above, but with a different name and +``__array_wrap__`` for ufuncs and other functions +------------------------------------------------- + +Prior to numpy 1.13, the behaviour of ufuncs could only be tuned using +``__array_wrap__`` and ``__array_prepare__``. These two allowed one to +change the output type of a ufunc, but, in constrast to +``__array_ufunc__``, did not allow one to make any changes to the inputs. +It is hoped to eventually deprecate these, but ``__array_wrap__`` is also +used by other numpy functions and methods, such as ``squeeze``, so at the +present time is still needed for full functionality. + +Conceptually, ``__array_wrap__`` "wraps up the action" in the sense of +allowing a subclass to set the type of the return value and update +attributes and metadata. Let's show how this works with an example. First +we return to the simpler example subclass, but with a different name and some print statements: .. testcode:: @@ -446,7 +595,7 @@ some print statements: print(' self is %s' % repr(self)) print(' arr is %s' % repr(out_arr)) # then just call the parent - return np.ndarray.__array_wrap__(self, out_arr, context) + return super(MySubClass, self).__array_wrap__(self, out_arr, context) We run a ufunc on an instance of our new array: @@ -467,13 +616,12 @@ MySubClass([1, 3, 5, 7, 9]) >>> ret.info 'spam' -Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method of the -input with the highest ``__array_priority__`` value, in this case -``MySubClass.__array_wrap__``, with arguments ``self`` as ``obj``, and -``out_arr`` as the (ndarray) result of the addition. In turn, the -default ``__array_wrap__`` (``ndarray.__array_wrap__``) has cast the -result to class ``MySubClass``, and called ``__array_finalize__`` - -hence the copying of the ``info`` attribute. This has all happened at the C level. +Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method +with arguments ``self`` as ``obj``, and ``out_arr`` as the (ndarray) result +of the addition. In turn, the default ``__array_wrap__`` +(``ndarray.__array_wrap__``) has cast the result to class ``MySubClass``, +and called ``__array_finalize__`` - hence the copying of the ``info`` +attribute. This has all happened at the C level. But, we could do anything we wanted: @@ -494,11 +642,12 @@ But, we could do anything we wanted: So, by defining a specific ``__array_wrap__`` method for our subclass, we can tweak the output from ufuncs. The ``__array_wrap__`` method requires ``self``, then an argument - which is the result of the ufunc - -and an optional parameter *context*. This parameter is returned by some -ufuncs as a 3-element tuple: (name of the ufunc, argument of the ufunc, -domain of the ufunc). ``__array_wrap__`` should return an instance of -its containing class. See the masked array subclass for an -implementation. +and an optional parameter *context*. This parameter is returned by +ufuncs as a 3-element tuple: (name of the ufunc, arguments of the ufunc, +domain of the ufunc), but is not set by other numpy functions. Though, +as seen above, it is possible to do otherwise, ``__array_wrap__`` should +return an instance of its containing class. See the masked array +subclass for an implementation. In addition to ``__array_wrap__``, which is called on the way out of the ufunc, there is also an ``__array_prepare__`` method which is called on diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index 1d65db55e..4cdb76b20 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -8,6 +8,7 @@ from numpy.version import version as __version__ from .type_check import * from .index_tricks import * from .function_base import * +from .mixins import * from .nanfunctions import * from .shape_base import * from .stride_tricks import * @@ -29,6 +30,7 @@ __all__ = ['emath', 'math'] __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ +__all__ += mixins.__all__ __all__ += shape_base.__all__ __all__ += stride_tricks.__all__ __all__ += twodim_base.__all__ diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py new file mode 100644 index 000000000..21e4b346f --- /dev/null +++ b/numpy/lib/mixins.py @@ -0,0 +1,171 @@ +"""Mixin classes for custom array types that don't inherit from ndarray.""" +from __future__ import division, absolute_import, print_function + +import sys + +from numpy.core import umath as um + +# Nothing should be exposed in the top-level NumPy module. +__all__ = [] + + +def _disables_array_ufunc(obj): + """True when __array_ufunc__ is set to None.""" + try: + return obj.__array_ufunc__ is None + except AttributeError: + return False + + +def _binary_method(ufunc): + """Implement a forward binary method with a ufunc, e.g., __add__.""" + def func(self, other): + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(self, other) + return func + + +def _reflected_binary_method(ufunc): + """Implement a reflected binary method with a ufunc, e.g., __radd__.""" + def func(self, other): + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(other, self) + return func + + +def _inplace_binary_method(ufunc): + """Implement an in-place binary method with a ufunc, e.g., __iadd__.""" + def func(self, other): + return ufunc(self, other, out=(self,)) + return func + + +def _numeric_methods(ufunc): + """Implement forward, reflected and inplace binary methods with a ufunc.""" + return (_binary_method(ufunc), + _reflected_binary_method(ufunc), + _inplace_binary_method(ufunc)) + + +def _unary_method(ufunc): + """Implement a unary special method with a ufunc.""" + def func(self): + return ufunc(self) + return func + + +class NDArrayOperatorsMixin(object): + """Mixin defining all operator special methods using __array_ufunc__. + + This class implements the special methods for almost all of Python's + builtin operators defined in the `operator` module, including comparisons + (``==``, ``>``, etc.) and arithmetic (``+``, ``*``, ``-``, etc.), by + deferring to the ``__array_ufunc__`` method, which subclasses must + implement. + + This class does not yet implement the special operators corresponding + to ``divmod``, unary ``+`` or ``matmul`` (``@``), because these operation + do not yet have corresponding NumPy ufuncs. + + It is useful for writing classes that do not inherit from `numpy.ndarray`, + but that should support arithmetic and numpy universal functions like + arrays as described in :ref:`A Mechanism for Overriding Ufuncs + <neps.ufunc-overrides>`. + + As an trivial example, consider this implementation of an ``ArrayLike`` + class that simply wraps a NumPy array and ensures that the result of any + arithmetic operation is also an ``ArrayLike`` object:: + + class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(x, ArrayLike) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + In interactions between ``ArrayLike`` objects and numbers or numpy arrays, + the result is always another ``ArrayLike``: + + >>> x = ArrayLike([1, 2, 3]) + >>> x - 1 + ArrayLike(array([0, 1, 2])) + >>> 1 - x + ArrayLike(array([ 0, -1, -2])) + >>> np.arange(3) - x + ArrayLike(array([-1, -1, -1])) + >>> x - np.arange(3) + ArrayLike(array([1, 1, 1])) + + Note that unlike ``numpy.ndarray``, ``ArrayLike`` does not allow operations + with arbitrary, unrecognized types. This ensures that interactions with + ArrayLike preserve a well-defined casting hierarchy. + """ + # Like np.ndarray, this mixin class implements "Option 1" from the ufunc + # overrides NEP. + + # comparisons don't have reflected and in-place versions + __lt__ = _binary_method(um.less) + __le__ = _binary_method(um.less_equal) + __eq__ = _binary_method(um.equal) + __ne__ = _binary_method(um.not_equal) + __gt__ = _binary_method(um.greater) + __ge__ = _binary_method(um.greater_equal) + + # numeric methods + __add__, __radd__, __iadd__ = _numeric_methods(um.add) + __sub__, __rsub__, __isub__ = _numeric_methods(um.subtract) + __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply) + if sys.version_info.major < 3: + # Python 3 uses only __truediv__ and __floordiv__ + __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide) + __truediv__, __rtruediv__, __itruediv__ = _numeric_methods(um.true_divide) + __floordiv__, __rfloordiv__, __ifloordiv__ = _numeric_methods( + um.floor_divide) + __mod__, __rmod__, __imod__ = _numeric_methods(um.mod) + # TODO: handle the optional third argument for __pow__? + __pow__, __rpow__, __ipow__ = _numeric_methods(um.power) + __lshift__, __rlshift__, __ilshift__ = _numeric_methods(um.left_shift) + __rshift__, __rrshift__, __irshift__ = _numeric_methods(um.right_shift) + __and__, __rand__, __iand__ = _numeric_methods(um.bitwise_and) + __xor__, __rxor__, __ixor__ = _numeric_methods(um.bitwise_xor) + __or__, __ror__, __ior__ = _numeric_methods(um.bitwise_or) + + # unary methods + __neg__ = _unary_method(um.negative) + __abs__ = _unary_method(um.absolute) + __invert__ = _unary_method(um.invert) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py new file mode 100644 index 000000000..57c4a4cd8 --- /dev/null +++ b/numpy/lib/tests/test_mixins.py @@ -0,0 +1,200 @@ +from __future__ import division, absolute_import, print_function + +import numbers +import operator +import sys + +import numpy as np +from numpy.testing import ( + TestCase, run_module_suite, assert_, assert_equal, assert_raises) + + +PY2 = sys.version_info.major < 3 + + +# NOTE: This class should be kept as an exact copy of the example from the +# docstring for NDArrayOperatorsMixin. + +class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(x, ArrayLike) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + +def _assert_equal_type_and_value(result, expected, err_msg=None): + assert_equal(type(result), type(expected), err_msg=err_msg) + assert_equal(result.value, expected.value, err_msg=err_msg) + assert_equal(getattr(result.value, 'dtype', None), + getattr(expected.value, 'dtype', None), err_msg=err_msg) + + +class TestNDArrayOperatorsMixin(TestCase): + + def test_array_like_add(self): + + def check(result): + _assert_equal_type_and_value(result, ArrayLike(0)) + + check(ArrayLike(0) + 0) + check(0 + ArrayLike(0)) + + check(ArrayLike(0) + np.array(0)) + check(np.array(0) + ArrayLike(0)) + + check(ArrayLike(np.array(0)) + 0) + check(0 + ArrayLike(np.array(0))) + + check(ArrayLike(np.array(0)) + np.array(0)) + check(np.array(0) + ArrayLike(np.array(0))) + + def test_inplace(self): + array_like = ArrayLike(np.array([0])) + array_like += 1 + _assert_equal_type_and_value(array_like, ArrayLike(np.array([1]))) + + array = np.array([0]) + array += ArrayLike(1) + _assert_equal_type_and_value(array, ArrayLike(np.array([1]))) + + def test_opt_out(self): + + class OptOut(object): + """Object that opts out of __array_ufunc__.""" + __array_ufunc__ = None + + def __add__(self, other): + return self + + def __radd__(self, other): + return self + + array_like = ArrayLike(1) + opt_out = OptOut() + + # supported operations + assert_(array_like + opt_out is opt_out) + assert_(opt_out + array_like is opt_out) + + # not supported + with assert_raises(TypeError): + # don't use the Python default, array_like = array_like + opt_out + array_like += opt_out + with assert_raises(TypeError): + array_like - opt_out + with assert_raises(TypeError): + opt_out - array_like + + def test_subclass(self): + + class SubArrayLike(ArrayLike): + """Should take precedence over ArrayLike.""" + + x = ArrayLike(0) + y = SubArrayLike(1) + _assert_equal_type_and_value(x + y, y) + _assert_equal_type_and_value(y + x, y) + + def test_object(self): + x = ArrayLike(0) + obj = object() + with assert_raises(TypeError): + x + obj + with assert_raises(TypeError): + obj + x + with assert_raises(TypeError): + x += obj + + def test_unary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + for op in [operator.neg, + # pos is not yet implemented + abs, + operator.invert]: + _assert_equal_type_and_value(op(array_like), ArrayLike(op(array))) + + def test_binary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + operators = [ + operator.lt, + operator.le, + operator.eq, + operator.ne, + operator.gt, + operator.ge, + operator.add, + operator.sub, + operator.mul, + operator.truediv, + operator.floordiv, + # TODO: test div on Python 2, only + operator.mod, + # divmod is not yet implemented + pow, + operator.lshift, + operator.rshift, + operator.and_, + operator.xor, + operator.or_, + ] + for op in operators: + expected = ArrayLike(op(array, 1)) + actual = op(array_like, 1) + err_msg = 'failed for operator {}'.format(op) + _assert_equal_type_and_value(expected, actual, err_msg=err_msg) + + def test_ufunc_at(self): + array = ArrayLike(np.array([1, 2, 3, 4])) + assert_(np.negative.at(array, np.array([0, 1])) is None) + _assert_equal_type_and_value(array, ArrayLike([-1, -2, 3, 4])) + + def test_ufunc_two_outputs(self): + def check(result): + assert_(type(result) is tuple) + assert_equal(len(result), 2) + mantissa, exponent = np.frexp(2 ** -3) + _assert_equal_type_and_value(result[0], ArrayLike(mantissa)) + _assert_equal_type_and_value(result[1], ArrayLike(exponent)) + + check(np.frexp(ArrayLike(2 ** -3))) + check(np.frexp(ArrayLike(np.array(2 ** -3)))) + + +if __name__ == "__main__": + run_module_suite() diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 554fd6dc5..cb0bfdde2 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -3933,13 +3933,17 @@ class MaskedArray(ndarray): def _delegate_binop(self, other): # This emulates the logic in - # multiarray/number.c:PyArray_GenericBinaryFunction - if (not isinstance(other, np.ndarray) - and not hasattr(other, "__numpy_ufunc__")): + # private/binop_override.h:forward_binop_should_defer + if isinstance(other, type(self)): + return False + array_ufunc = getattr(other, "__array_ufunc__", False) + if array_ufunc is False: other_priority = getattr(other, "__array_priority__", -1000000) - if self.__array_priority__ < other_priority: - return True - return False + return self.__array_priority__ < other_priority + else: + # If array_ufunc is not None, it will be called inside the ufunc; + # None explicitly tells us to not call the ufunc, i.e., defer. + return array_ufunc is None def _comparison(self, other, compare): """Compare self with other using operator.eq or operator.ne. |