ENH: umath: Make sum, prod, any, or functions use the .reduce method directly

This required some extensive changes, like:
 * Making logical_or, logical_and, and logical_not on object arrays behave
   like their Python equivalents instead of calling methods on the objects
 * Changing the units for a fair number of the binary operations to
   None, so they don't get initialized to their unit values at the start

A consequence of this is that multi-dimensional reductions like sum, prod,
any, or all no longer need to make copies, so are faster in some cases.

8 files changed, 221 insertions, 96 deletions

diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py
index 82a8cde11..8771a040d 100644
--- a/numpy/core/code_generators/generate_umath.py
+++ b/numpy/core/code_generators/generate_umath.py
@@ -414,21 +414,21 @@ defdict = {
           docstrings.get('numpy.core.umath.logical_and'),
           'PyUFunc_SimpleBinaryComparisonTypeResolution',
           TD(nodatetime_or_obj, out='?'),
-          TD(P, f='logical_and'),
+          TD(O, f='npy_ObjectLogicalAnd'),
           ),
 'logical_not' :
     Ufunc(1, 1, None,
           docstrings.get('numpy.core.umath.logical_not'),
           None,
           TD(nodatetime_or_obj, out='?'),
-          TD(P, f='logical_not'),
+          TD(O, f='npy_ObjectLogicalNot'),
           ),
 'logical_or' :
     Ufunc(2, 1, Zero,
           docstrings.get('numpy.core.umath.logical_or'),
           'PyUFunc_SimpleBinaryComparisonTypeResolution',
           TD(nodatetime_or_obj, out='?'),
-          TD(P, f='logical_or'),
+          TD(O, f='npy_ObjectLogicalOr'),
           ),
 'logical_xor' :
     Ufunc(2, 1, None,
diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py
index 602c0ebc5..ca46e4a59 100644
--- a/numpy/core/fromnumeric.py
+++ b/numpy/core/fromnumeric.py
@@ -1448,12 +1448,7 @@ def sum(a, axis=None, dtype=None, out=None):
             out[...] = res
             return out
         return res
-    try:
-        sum = a.sum
-    except AttributeError:
-        return _wrapit(a, 'sum', axis, dtype, out)
-    return sum(axis, dtype, out)
-
+    return um.add.reduce(a, axis=axis, dtype=dtype, out=out)
 
 def product (a, axis=None, dtype=None, out=None):
     """
@@ -1464,11 +1459,7 @@ def product (a, axis=None, dtype=None, out=None):
     prod : equivalent function; see for details.
 
     """
-    try:
-        prod = a.prod
-    except AttributeError:
-        return _wrapit(a, 'prod', axis, dtype, out)
-    return prod(axis, dtype, out)
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out)
 
 
 def sometrue(a, axis=None, out=None):
@@ -1482,11 +1473,7 @@ def sometrue(a, axis=None, out=None):
     any : equivalent function
 
     """
-    try:
-        any = a.any
-    except AttributeError:
-        return _wrapit(a, 'any', axis, out)
-    return any(axis, out)
+    return um.logical_or.reduce(a, axis=axis, out=out)
 
 
 def alltrue (a, axis=None, out=None):
@@ -1498,12 +1485,7 @@ def alltrue (a, axis=None, out=None):
     numpy.all : Equivalent function; see for details.
 
     """
-    try:
-        all = a.all
-    except AttributeError:
-        return _wrapit(a, 'all', axis, out)
-    return all(axis, out)
-
+    return um.logical_and.reduce(a, axis=axis, out=out)
 
 def any(a,axis=None, out=None):
     """
@@ -1569,12 +1551,7 @@ def any(a,axis=None, out=None):
     (191614240, 191614240)
 
     """
-    try:
-        any = a.any
-    except AttributeError:
-        return _wrapit(a, 'any', axis, out)
-    return any(axis, out)
-
+    return um.logical_or.reduce(a, axis=axis, out=out)
 
 def all(a,axis=None, out=None):
     """
@@ -1633,12 +1610,7 @@ def all(a,axis=None, out=None):
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    try:
-        all = a.all
-    except AttributeError:
-        return _wrapit(a, 'all', axis, out)
-    return all(axis, out)
-
+    return um.logical_and.reduce(a, axis=axis, out=out)
 
 def cumsum (a, axis=None, dtype=None, out=None):
     """
diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c
index 8a90a7d2e..b3402129f 100644
--- a/numpy/core/src/multiarray/nditer_api.c
+++ b/numpy/core/src/multiarray/nditer_api.c
@@ -2564,7 +2564,7 @@ npyiter_checkreducesize(NpyIter *iter, npy_intp count,
     *reduce_outerdim = 0;
 
     /* If there's only one dimension, no need to calculate anything */
-    if (ndim == 1) {
+    if (ndim == 1 || count == 0) {
         *reduce_innersize = count;
         return count;
     }
diff --git a/numpy/core/src/umath/funcs.inc.src b/numpy/core/src/umath/funcs.inc.src
index 5dc58e990..a76adaa39 100644
--- a/numpy/core/src/umath/funcs.inc.src
+++ b/numpy/core/src/umath/funcs.inc.src
@@ -78,6 +78,83 @@ npy_Object@Kind@(PyObject *i1, PyObject *i2)
 }
 /**end repeat**/
 
+/* Emulates Python's 'a or b' behavior */
+static PyObject *
+npy_ObjectLogicalOr(PyObject *i1, PyObject *i2)
+{
+    if (i1 == NULL) {
+        Py_XINCREF(i2);
+        return i2;
+    }
+    else if (i2 == NULL) {
+        Py_INCREF(i2);
+        return i1;
+    }
+    else {
+        int retcode = PyObject_IsTrue(i1);
+        if (retcode == -1) {
+            return NULL;
+        }
+        else if (retcode) {
+            Py_INCREF(i1);
+            return i1;
+        }
+        else {
+            Py_INCREF(i2);
+            return i2;
+        }
+    }
+}
+
+/* Emulates Python's 'a and b' behavior */
+static PyObject *
+npy_ObjectLogicalAnd(PyObject *i1, PyObject *i2)
+{
+    if (i1 == NULL) {
+        return NULL;
+    }
+    else if (i2 == NULL) {
+        return NULL;
+    }
+    else {
+        int retcode = PyObject_IsTrue(i1);
+        if (retcode == -1) {
+            return NULL;
+        }
+        else if (!retcode) {
+            Py_INCREF(i1);
+            return i1;
+        }
+        else {
+            Py_INCREF(i2);
+            return i2;
+        }
+    }
+}
+
+
+/* Emulates Python's 'not b' behavior */
+static PyObject *
+npy_ObjectLogicalNot(PyObject *i1)
+{
+    if (i1 == NULL) {
+        return NULL;
+    }
+    else {
+        int retcode = PyObject_Not(i1);
+        if (retcode == -1) {
+            return NULL;
+        }
+        else if (retcode) {
+            Py_INCREF(Py_True);
+            return Py_True;
+        }
+        else {
+            Py_INCREF(Py_False);
+            return Py_False;
+        }
+    }
+}
 
 /*
  *****************************************************************************
diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c
index 10b5656b8..8de0911e0 100644
--- a/numpy/core/src/umath/ufunc_object.c
+++ b/numpy/core/src/umath/ufunc_object.c
@@ -2571,6 +2571,53 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, PyArrayObject *out)
     return (PyArrayObject *)ret;
 }
 
+/* Allocate 'result' or conform 'out' to 'self' (in 'result') */
+static PyArrayObject *
+allocate_or_conform_reduce_result(PyArrayObject *arr, PyArrayObject *out,
+                        npy_bool *axis_flags, int otype_final, int keepmask)
+{
+    if (out == NULL) {
+        PyArrayObject *result;
+        PyArray_Descr *dtype = NULL;
+        /*
+         * Set up the output data type, using the input's exact
+         * data type if the type number didn't change to preserve
+         * metadata
+         */
+        if (PyArray_DESCR(arr)->type_num == otype_final) {
+            if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) {
+                dtype = PyArray_DESCR(arr);
+                Py_INCREF(dtype);
+            }
+            else {
+                dtype = PyArray_DescrNewByteorder(PyArray_DESCR(arr),
+                                                        NPY_NATIVE);
+            }
+        }
+        else {
+            dtype = PyArray_DescrFromType(otype_final);
+        }
+        if (dtype == NULL) {
+            return NULL;
+        }
+
+        result = allocate_reduce_result(arr, axis_flags, dtype);
+
+        /* Allocate an NA mask if necessary */
+        if (keepmask && result != NULL && PyArray_HASMASKNA(arr)) {
+            if (PyArray_AllocateMaskNA(result, 1, 0, 1) < 0) {
+                Py_DECREF(result);
+                return NULL;
+            }
+        }
+
+        return result;
+    }
+    else {
+        return conform_reduce_result(PyArray_NDIM(arr), axis_flags, out);
+    }
+}
+
 /*
  * Either:
  *   1) Fills 'result' with the identity, and returns a reference to 'arr'.
@@ -2732,7 +2779,7 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out,
 
     /* Iterator parameters */
     NpyIter *iter = NULL;
-    PyArrayObject *op[2];
+    PyArrayObject *op[3];
     PyArray_Descr *op_dtypes[2] = {NULL, NULL};
     npy_uint32 flags, op_flags[2];
 
@@ -2774,48 +2821,16 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out,
         return NULL;
     }
 
-    /* Allocate 'result' or conform 'out' to 'self' (in 'result') */
-    if (out == NULL) {
-        PyArray_Descr *dtype = NULL;
-        /*
-         * Set up the output data type, using the input's exact
-         * data type if the type number didn't change to preserve
-         * metadata
-         */
-        if (PyArray_DESCR(arr)->type_num == otype_final) {
-            if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) {
-                dtype = PyArray_DESCR(arr);
-                Py_INCREF(dtype);
-            }
-            else {
-                dtype = PyArray_DescrNewByteorder(PyArray_DESCR(arr),
-                                                        NPY_NATIVE);
-            }
-        }
-        else {
-            dtype = PyArray_DescrFromType(otype_final);
-        }
-        if (dtype == NULL) {
-            return NULL;
-        }
-        result = allocate_reduce_result(arr, axis_flags, dtype);
-        if (result == NULL) {
-            return NULL;
-        }
-
-        /* Allocate an NA mask if necessary */
-        if (!skipna && PyArray_HASMASKNA(arr)) {
-            if (PyArray_AllocateMaskNA(result, 1, 0, 1) < 0) {
-                Py_DECREF(result);
-                return NULL;
-            }
-        }
+    /* If the loop wants the arrays, provide them */
+    if (_does_loop_use_arrays(innerloopdata)) {
+        innerloopdata = (void*)op;
     }
-    else {
-        result = conform_reduce_result(ndim, axis_flags, out);
-        if (result == NULL) {
-            return NULL;
-        }
+
+    /* Allocate an output or conform 'out' to 'self' */
+    result = allocate_or_conform_reduce_result(arr, out,
+                                        axis_flags, otype_final, !skipna);
+    if (result == NULL) {
+        return NULL;
     }
 
     /*
@@ -2832,6 +2847,8 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out,
     /* Now we can do a loop applying the ufunc in a straightforward manner */
     op[0] = result;
     op[1] = arr_view;
+    /* op is length 3 in case the inner loop wanted these as its data */
+    op[2] = result;
     op_dtypes[0] = PyArray_DescrFromType(otype_final);
     if (op_dtypes[0] == NULL) {
         goto fail;
@@ -2898,6 +2915,10 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *out,
         if (!needs_api) {
             NPY_END_THREADS;
         }
+
+        if (needs_api && PyErr_Occurred()) {
+            goto fail;
+        }
     }
 
     /* Strip out the extra 'one' dimensions in the result */
@@ -3768,14 +3789,6 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args,
     if (mp == NULL) {
         return NULL;
     }
-    /* Check to see if input is zero-dimensional */
-    if (PyArray_NDIM(mp) == 0) {
-        PyErr_Format(PyExc_TypeError, "cannot %s on a scalar",
-                     _reduce_type[operation]);
-        Py_XDECREF(otype);
-        Py_DECREF(mp);
-        return NULL;
-    }
     /* Check to see that type (and otype) is not FLEXIBLE */
     if (PyArray_ISFLEXIBLE(mp) ||
         (otype && PyTypeNum_ISFLEXIBLE(otype->type_num))) {
@@ -3855,6 +3868,35 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args,
         naxes = 1;
     }
 
+    /* Check to see if input is zero-dimensional. */
+    if (PyArray_NDIM(mp) == 0) {
+        /* A reduction with no axes is still valid but trivial */
+        if (operation == UFUNC_REDUCE && naxes == 0) {
+            Py_XDECREF(otype);
+            /* If there's an output parameter, copy the value */
+            if (out != NULL) {
+                if (PyArray_CopyInto(out, mp) < 0) {
+                    Py_DECREF(mp);
+                    return NULL;
+                }
+                else {
+                    Py_DECREF(mp);
+                    Py_INCREF(out);
+                    return (PyObject *)out;
+                }
+            }
+            /* Otherwise return the array unscathed */
+            else {
+                return (PyObject *)mp;
+            }
+        }
+        PyErr_Format(PyExc_TypeError, "cannot %s on a scalar",
+                     _reduce_type[operation]);
+        Py_XDECREF(otype);
+        Py_DECREF(mp);
+        return NULL;
+    }
+
      /*
       * If out is specified it determines otype
       * unless otype already specified.
@@ -3919,9 +3961,16 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args,
     }
     Py_DECREF(mp);
     Py_DECREF(otype);
+
     if (ret == NULL) {
         return NULL;
     }
+
+    /* If an output parameter was provided, don't wrap it */
+    if (out != NULL) {
+        return (PyObject *)ret;
+    }
+
     if (Py_TYPE(op) != Py_TYPE(ret)) {
         res = PyObject_CallMethod(op, "__array_wrap__", "O", ret);
         if (res == NULL) {
diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py
index 773ce9a3b..c05d21cc1 100644
--- a/numpy/core/tests/test_ufunc.py
+++ b/numpy/core/tests/test_ufunc.py
@@ -120,9 +120,9 @@ class TestUfunc(TestCase):
 
         # class to use in testing object method loops
         class foo(object):
-            def logical_not(self) :
+            def conjugate(self) :
                 return np.bool_(1)
-            def logical_and(self, obj) :
+            def logical_xor(self, obj) :
                 return np.bool_(1)
 
         # check unary PyUFunc_O_O
@@ -134,7 +134,7 @@ class TestUfunc(TestCase):
         x = np.zeros(10, dtype=np.object)[0::2]
         for i in range(len(x)) :
             x[i] = foo()
-        assert_(np.all(np.logical_not(x) == True), msg)
+        assert_(np.all(np.conjugate(x) == True), msg)
 
         # check binary PyUFunc_OO_O
         msg = "PyUFunc_OO_O"
@@ -145,7 +145,7 @@ class TestUfunc(TestCase):
         x = np.zeros(10, dtype=np.object)[0::2]
         for i in range(len(x)) :
             x[i] = foo()
-        assert_(np.all(np.logical_and(x,x) == 1), msg)
+        assert_(np.all(np.logical_xor(x,x)), msg)
 
         # check PyUFunc_On_Om
         # fixme -- I don't know how to do this yet
@@ -471,6 +471,33 @@ class TestUfunc(TestCase):
 
         assert_equal(ref, True, err_msg="reference check")
 
+    def test_object_logical(self):
+        a = np.array([3, None, True, False, "test", ""], dtype=object)
+        assert_equal(np.logical_or(a, None),
+                        np.array([x or None for x in a], dtype=object))
+        assert_equal(np.logical_or(a, True),
+                        np.array([x or True for x in a], dtype=object))
+        assert_equal(np.logical_or(a, 12),
+                        np.array([x or 12 for x in a], dtype=object))
+        assert_equal(np.logical_or(a, "blah"),
+                        np.array([x or "blah" for x in a], dtype=object))
+
+        assert_equal(np.logical_and(a, None),
+                        np.array([x and None for x in a], dtype=object))
+        assert_equal(np.logical_and(a, True),
+                        np.array([x and True for x in a], dtype=object))
+        assert_equal(np.logical_and(a, 12),
+                        np.array([x and 12 for x in a], dtype=object))
+        assert_equal(np.logical_and(a, "blah"),
+                        np.array([x and "blah" for x in a], dtype=object))
+
+        assert_equal(np.logical_not(a),
+                        np.array([not x for x in a], dtype=object))
+
+        assert_equal(np.logical_or.reduce(a), 3)
+        assert_equal(np.logical_and.reduce(a), None)
+
+
     def test_casting_out_param(self):
         # Test that it's possible to do casts on output
         a = np.ones((200,100), np.int64)
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index 105389d6d..0b6b1e19d 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -452,7 +452,7 @@ class TestTrapz(TestCase):
     def test_simple(self):
         r = trapz(exp(-1.0 / 2 * (arange(-10, 10, .1)) ** 2) / sqrt(2 * pi), dx=0.1)
         #check integral of normal equals 1
-        assert_almost_equal(sum(r, axis=0), 1, 7)
+        assert_almost_equal(r, 1, 7)
 
     def test_ndim(self):
         x = linspace(0, 1, 3)
diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
index 91a3c4021..95219beb1 100644
--- a/numpy/ma/tests/test_core.py
+++ b/numpy/ma/tests/test_core.py
@@ -1102,13 +1102,13 @@ class TestMaskedArrayArithmetic(TestCase):
             output.fill(-9999)
             result = npfunc(xm, axis=0, out=output)
             # ... the result should be the given output
-            self.assertTrue(result is output)
+            assert_(result is output)
             assert_equal(result, xmmeth(axis=0, out=output))
             #
             output = empty(4, dtype=int)
             result = xmmeth(axis=0, out=output)
-            self.assertTrue(result is output)
-            self.assertTrue(output[0] is masked)
+            assert_(result is output)
+            assert_(output[0] is masked)
 
 
     def test_eq_on_structured(self):