From 00dff152bb10c810176c7e47ad41e563df602208 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 2 Aug 2011 17:06:20 -0500 Subject: ENH: umath: Make sum, prod, any, or functions use the .reduce method directly This required some extensive changes, like: * Making logical_or, logical_and, and logical_not on object arrays behave like their Python equivalents instead of calling methods on the objects * Changing the units for a fair number of the binary operations to None, so they don't get initialized to their unit values at the start A consequence of this is that multi-dimensional reductions like sum, prod, any, or all no longer need to make copies, so are faster in some cases. --- numpy/core/code_generators/generate_umath.py | 6 +- numpy/core/fromnumeric.py | 40 ++----- numpy/core/src/multiarray/nditer_api.c | 2 +- numpy/core/src/umath/funcs.inc.src | 77 ++++++++++++++ numpy/core/src/umath/ufunc_object.c | 149 ++++++++++++++++++--------- numpy/core/tests/test_ufunc.py | 35 ++++++- numpy/lib/tests/test_function_base.py | 2 +- numpy/ma/tests/test_core.py | 6 +- 8 files changed, 221 insertions(+), 96 deletions(-) diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 82a8cde11..8771a040d 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -414,21 +414,21 @@ defdict = { docstrings.get('numpy.core.umath.logical_and'), 'PyUFunc_SimpleBinaryComparisonTypeResolution', TD(nodatetime_or_obj, out='?'), - TD(P, f='logical_and'), + TD(O, f='npy_ObjectLogicalAnd'), ), 'logical_not' : Ufunc(1, 1, None, docstrings.get('numpy.core.umath.logical_not'), None, TD(nodatetime_or_obj, out='?'), - TD(P, f='logical_not'), + TD(O, f='npy_ObjectLogicalNot'), ), 'logical_or' : Ufunc(2, 1, Zero, docstrings.get('numpy.core.umath.logical_or'), 'PyUFunc_SimpleBinaryComparisonTypeResolution', TD(nodatetime_or_obj, out='?'), - TD(P, f='logical_or'), + TD(O, f='npy_ObjectLogicalOr'), ), 'logical_xor' : Ufunc(2, 1, None, diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 602c0ebc5..ca46e4a59 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1448,12 +1448,7 @@ def sum(a, axis=None, dtype=None, out=None): out[...] = res return out return res - try: - sum = a.sum - except AttributeError: - return _wrapit(a, 'sum', axis, dtype, out) - return sum(axis, dtype, out) - + return um.add.reduce(a, axis=axis, dtype=dtype, out=out) def product (a, axis=None, dtype=None, out=None): """ @@ -1464,11 +1459,7 @@ def product (a, axis=None, dtype=None, out=None): prod : equivalent function; see for details. """ - try: - prod = a.prod - except AttributeError: - return _wrapit(a, 'prod', axis, dtype, out) - return prod(axis, dtype, out) + return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out) def sometrue(a, axis=None, out=None): @@ -1482,11 +1473,7 @@ def sometrue(a, axis=None, out=None): any : equivalent function """ - try: - any = a.any - except AttributeError: - return _wrapit(a, 'any', axis, out) - return any(axis, out) + return um.logical_or.reduce(a, axis=axis, out=out) def alltrue (a, axis=None, out=None): @@ -1498,12 +1485,7 @@ def alltrue (a, axis=None, out=None): numpy.all : Equivalent function; see for details. """ - try: - all = a.all - except AttributeError: - return _wrapit(a, 'all', axis, out) - return all(axis, out) - + return um.logical_and.reduce(a, axis=axis, out=out) def any(a,axis=None, out=None): """ @@ -1569,12 +1551,7 @@ def any(a,axis=None, out=None): (191614240, 191614240) """ - try: - any = a.any - except AttributeError: - return _wrapit(a, 'any', axis, out) - return any(axis, out) - + return um.logical_or.reduce(a, axis=axis, out=out) def all(a,axis=None, out=None): """ @@ -1633,12 +1610,7 @@ def all(a,axis=None, out=None): (28293632, 28293632, array([ True], dtype=bool)) """ - try: - all = a.all - except AttributeError: - return _wrapit(a, 'all', axis, out) - return all(axis, out) - + return um.logical_and.reduce(a, axis=axis, out=out) def cumsum (a, axis=None, dtype=None, out=None): """ diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index 8a90a7d2e..b3402129f 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -2564,7 +2564,7 @@ npyiter_checkreducesize(NpyIter *iter, npy_intp count, *reduce_outerdim = 0; /* If there's only one dimension, no need to calculate anything */ - if (ndim == 1) { + if (ndim == 1 || count == 0) { *reduce_innersize = count; return count; } diff --git a/numpy/core/src/umath/funcs.inc.src b/numpy/core/src/umath/funcs.inc.src index 5dc58e990..a76adaa39 100644 --- a/numpy/core/src/umath/funcs.inc.src +++ b/numpy/core/src/umath/funcs.inc.src @@ -78,6 +78,83 @@ npy_Object@Kind@(PyObject *i1, PyObject *i2) } /**end repeat**/ +/* Emulates Python's 'a or b' behavior */ +static PyObject * +npy_ObjectLogicalOr(PyObject *i1, PyObject *i2) +{ + if (i1 == NULL) { + Py_XINCREF(i2); + return i2; + } + else if (i2 == NULL) { + Py_INCREF(i2); + return i1; + } + else { + int retcode = PyObject_IsTrue(i1); + if (retcode == -1) { + return NULL; + } + else if (retcode) { + Py_INCREF(i1); + return i1; + } + else { + Py_INCREF(i2); + return i2; + } + } +} + +/* Emulates Python's 'a and b' behavior */ +static PyObject * +npy_ObjectLogicalAnd(PyObject *i1, PyObject *i2) +{ + if (i1 == NULL) { + return NULL; + } + else if (i2 == NULL) { + return NULL; + } + else { + int retcode = PyObject_IsTrue(i1); + if (retcode == -1) { + return NULL; + } + else if (!retcode) { + Py_INCREF(i1); + return i1; + } + else { + Py_INCREF(i2); + return i2; + } + } +} + + +/* Emulates Python's 'not b' behavior */ +static PyObject * +npy_ObjectLogicalNot(PyObject *i1) +{ + if (i1 == NULL) { + return NULL; + } + else { + int retcode = PyObject_Not(i1); + if (retcode == -1) { + return NULL; + } + else if (retcode) { + Py_INCREF(Py_True); + return Py_True; + } + else { + Py_INCREF(Py_False); + return Py_False; + } + } +} /* ***************************************************************************** diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 10b5656b8..8de0911e0 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2571,6 +2571,53 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, PyArrayObject *out) return (PyArrayObject *)ret; } +/* Allocate 'result' or conform 'out' to 'self' (in 'result') */ +static PyArrayObject * +allocate_or_conform_reduce_result(PyArrayObject *arr, PyArrayObject *out, + npy_bool *axis_flags, int otype_final, int keepmask) +{ + if (out == NULL) { + PyArrayObject *result; + PyArray_Descr *dtype = NULL; + /* + * Set up the output data type, using the input's exact + * data type if the type number didn't change to preserve + * metadata + */ + if (PyArray_DESCR(arr)->type_num == otype_final) { + if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) { + dtype = PyArray_DESCR(arr); + Py_INCREF(dtype); + } + else { + dtype = PyArray_DescrNewByteorder(PyArray_DESCR(arr), + NPY_NATIVE); + } + } + else { + dtype = PyArray_DescrFromType(otype_final); + } + if (dtype == NULL) { + return NULL; + } + + result = allocate_reduce_result(arr, axis_flags, dtype); + + /* Allocate an NA mask if necessary */ + if (keepmask && result != NULL && PyArray_HASMASKNA(arr)) { + if (PyArray_AllocateMaskNA(result, 1, 0, 1) < 0) { + Py_DECREF(result); + return NULL; + } + } + + return result; + } + else { + return conform_reduce_result(PyArray_NDIM(arr), axis_flags, out); + } +} + /* * Either: * 1) Fills 'result' with the identity, and returns a reference to 'arr'. @@ -2732,7 +2779,7 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out, /* Iterator parameters */ NpyIter *iter = NULL; - PyArrayObject *op[2]; + PyArrayObject *op[3]; PyArray_Descr *op_dtypes[2] = {NULL, NULL}; npy_uint32 flags, op_flags[2]; @@ -2774,48 +2821,16 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out, return NULL; } - /* Allocate 'result' or conform 'out' to 'self' (in 'result') */ - if (out == NULL) { - PyArray_Descr *dtype = NULL; - /* - * Set up the output data type, using the input's exact - * data type if the type number didn't change to preserve - * metadata - */ - if (PyArray_DESCR(arr)->type_num == otype_final) { - if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) { - dtype = PyArray_DESCR(arr); - Py_INCREF(dtype); - } - else { - dtype = PyArray_DescrNewByteorder(PyArray_DESCR(arr), - NPY_NATIVE); - } - } - else { - dtype = PyArray_DescrFromType(otype_final); - } - if (dtype == NULL) { - return NULL; - } - result = allocate_reduce_result(arr, axis_flags, dtype); - if (result == NULL) { - return NULL; - } - - /* Allocate an NA mask if necessary */ - if (!skipna && PyArray_HASMASKNA(arr)) { - if (PyArray_AllocateMaskNA(result, 1, 0, 1) < 0) { - Py_DECREF(result); - return NULL; - } - } + /* If the loop wants the arrays, provide them */ + if (_does_loop_use_arrays(innerloopdata)) { + innerloopdata = (void*)op; } - else { - result = conform_reduce_result(ndim, axis_flags, out); - if (result == NULL) { - return NULL; - } + + /* Allocate an output or conform 'out' to 'self' */ + result = allocate_or_conform_reduce_result(arr, out, + axis_flags, otype_final, !skipna); + if (result == NULL) { + return NULL; } /* @@ -2832,6 +2847,8 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out, /* Now we can do a loop applying the ufunc in a straightforward manner */ op[0] = result; op[1] = arr_view; + /* op is length 3 in case the inner loop wanted these as its data */ + op[2] = result; op_dtypes[0] = PyArray_DescrFromType(otype_final); if (op_dtypes[0] == NULL) { goto fail; @@ -2898,6 +2915,10 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out, if (!needs_api) { NPY_END_THREADS; } + + if (needs_api && PyErr_Occurred()) { + goto fail; + } } /* Strip out the extra 'one' dimensions in the result */ @@ -3768,14 +3789,6 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args, if (mp == NULL) { return NULL; } - /* Check to see if input is zero-dimensional */ - if (PyArray_NDIM(mp) == 0) { - PyErr_Format(PyExc_TypeError, "cannot %s on a scalar", - _reduce_type[operation]); - Py_XDECREF(otype); - Py_DECREF(mp); - return NULL; - } /* Check to see that type (and otype) is not FLEXIBLE */ if (PyArray_ISFLEXIBLE(mp) || (otype && PyTypeNum_ISFLEXIBLE(otype->type_num))) { @@ -3855,6 +3868,35 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args, naxes = 1; } + /* Check to see if input is zero-dimensional. */ + if (PyArray_NDIM(mp) == 0) { + /* A reduction with no axes is still valid but trivial */ + if (operation == UFUNC_REDUCE && naxes == 0) { + Py_XDECREF(otype); + /* If there's an output parameter, copy the value */ + if (out != NULL) { + if (PyArray_CopyInto(out, mp) < 0) { + Py_DECREF(mp); + return NULL; + } + else { + Py_DECREF(mp); + Py_INCREF(out); + return (PyObject *)out; + } + } + /* Otherwise return the array unscathed */ + else { + return (PyObject *)mp; + } + } + PyErr_Format(PyExc_TypeError, "cannot %s on a scalar", + _reduce_type[operation]); + Py_XDECREF(otype); + Py_DECREF(mp); + return NULL; + } + /* * If out is specified it determines otype * unless otype already specified. @@ -3919,9 +3961,16 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args, } Py_DECREF(mp); Py_DECREF(otype); + if (ret == NULL) { return NULL; } + + /* If an output parameter was provided, don't wrap it */ + if (out != NULL) { + return (PyObject *)ret; + } + if (Py_TYPE(op) != Py_TYPE(ret)) { res = PyObject_CallMethod(op, "__array_wrap__", "O", ret); if (res == NULL) { diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 773ce9a3b..c05d21cc1 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -120,9 +120,9 @@ class TestUfunc(TestCase): # class to use in testing object method loops class foo(object): - def logical_not(self) : + def conjugate(self) : return np.bool_(1) - def logical_and(self, obj) : + def logical_xor(self, obj) : return np.bool_(1) # check unary PyUFunc_O_O @@ -134,7 +134,7 @@ class TestUfunc(TestCase): x = np.zeros(10, dtype=np.object)[0::2] for i in range(len(x)) : x[i] = foo() - assert_(np.all(np.logical_not(x) == True), msg) + assert_(np.all(np.conjugate(x) == True), msg) # check binary PyUFunc_OO_O msg = "PyUFunc_OO_O" @@ -145,7 +145,7 @@ class TestUfunc(TestCase): x = np.zeros(10, dtype=np.object)[0::2] for i in range(len(x)) : x[i] = foo() - assert_(np.all(np.logical_and(x,x) == 1), msg) + assert_(np.all(np.logical_xor(x,x)), msg) # check PyUFunc_On_Om # fixme -- I don't know how to do this yet @@ -471,6 +471,33 @@ class TestUfunc(TestCase): assert_equal(ref, True, err_msg="reference check") + def test_object_logical(self): + a = np.array([3, None, True, False, "test", ""], dtype=object) + assert_equal(np.logical_or(a, None), + np.array([x or None for x in a], dtype=object)) + assert_equal(np.logical_or(a, True), + np.array([x or True for x in a], dtype=object)) + assert_equal(np.logical_or(a, 12), + np.array([x or 12 for x in a], dtype=object)) + assert_equal(np.logical_or(a, "blah"), + np.array([x or "blah" for x in a], dtype=object)) + + assert_equal(np.logical_and(a, None), + np.array([x and None for x in a], dtype=object)) + assert_equal(np.logical_and(a, True), + np.array([x and True for x in a], dtype=object)) + assert_equal(np.logical_and(a, 12), + np.array([x and 12 for x in a], dtype=object)) + assert_equal(np.logical_and(a, "blah"), + np.array([x and "blah" for x in a], dtype=object)) + + assert_equal(np.logical_not(a), + np.array([not x for x in a], dtype=object)) + + assert_equal(np.logical_or.reduce(a), 3) + assert_equal(np.logical_and.reduce(a), None) + + def test_casting_out_param(self): # Test that it's possible to do casts on output a = np.ones((200,100), np.int64) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 105389d6d..0b6b1e19d 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -452,7 +452,7 @@ class TestTrapz(TestCase): def test_simple(self): r = trapz(exp(-1.0 / 2 * (arange(-10, 10, .1)) ** 2) / sqrt(2 * pi), dx=0.1) #check integral of normal equals 1 - assert_almost_equal(sum(r, axis=0), 1, 7) + assert_almost_equal(r, 1, 7) def test_ndim(self): x = linspace(0, 1, 3) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 91a3c4021..95219beb1 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -1102,13 +1102,13 @@ class TestMaskedArrayArithmetic(TestCase): output.fill(-9999) result = npfunc(xm, axis=0, out=output) # ... the result should be the given output - self.assertTrue(result is output) + assert_(result is output) assert_equal(result, xmmeth(axis=0, out=output)) # output = empty(4, dtype=int) result = xmmeth(axis=0, out=output) - self.assertTrue(result is output) - self.assertTrue(output[0] is masked) + assert_(result is output) + assert_(output[0] is masked) def test_eq_on_structured(self): -- cgit v1.2.1