diff options
22 files changed, 407 insertions, 258 deletions
diff --git a/doc/release/upcoming_changes/22707.compatibility.rst b/doc/release/upcoming_changes/22707.compatibility.rst new file mode 100644 index 000000000..8c9805f37 --- /dev/null +++ b/doc/release/upcoming_changes/22707.compatibility.rst @@ -0,0 +1,4 @@ +* When comparing datetimes and timedelta using ``np.equal`` or ``np.not_equal`` + numpy previously allowed the comparison with ``casting="unsafe"``. + This operation now fails. Forcing the output dtype using the ``dtype`` + kwarg can make the operation succeed, but we do not recommend it. diff --git a/doc/release/upcoming_changes/22707.expired.rst b/doc/release/upcoming_changes/22707.expired.rst new file mode 100644 index 000000000..496752e8d --- /dev/null +++ b/doc/release/upcoming_changes/22707.expired.rst @@ -0,0 +1,13 @@ +``==`` and ``!=`` warnings finalized +------------------------------------ +The ``==`` and ``!=`` operators on arrays now always: + +* raise errors that occur during comparisons such as when the arrays + have incompatible shapes (``np.array([1, 2]) == np.array([1, 2, 3])``). +* return an array of all ``True`` or all ``False`` when values are + fundamentally not comparable (e.g. have different dtypes). An example + is ``np.array(["a"]) == np.array([1])``. + +This mimics the Python behavior of returning ``False`` and ``True`` +when comparing incompatible types like ``"a" == 1`` and ``"a" != 1``. +For a long time these gave ``DeprecationWarning`` or ``FutureWarning``. diff --git a/doc/release/upcoming_changes/22707.improvement.rst b/doc/release/upcoming_changes/22707.improvement.rst new file mode 100644 index 000000000..1b8d4f844 --- /dev/null +++ b/doc/release/upcoming_changes/22707.improvement.rst @@ -0,0 +1,8 @@ +New ``DTypePromotionError`` +--------------------------- +NumPy now has a new ``DTypePromotionError`` which is used when two +dtypes cannot be promoted to a common one, for example:: + + np.result_type("M8[s]", np.complex128) + +raises this new exception. diff --git a/numpy/__init__.py b/numpy/__init__.py index 1e41dc8bf..9f8e60a07 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -107,6 +107,8 @@ import sys import warnings from ._globals import _NoValue, _CopyMode +from . import exceptions +# Note that the following names are imported explicitly for backcompat: from .exceptions import ( ComplexWarning, ModuleDeprecationWarning, VisibleDeprecationWarning, TooHardError, AxisError) @@ -130,7 +132,7 @@ else: raise ImportError(msg) from e __all__ = [ - 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', + 'exceptions', 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', 'ComplexWarning', 'TooHardError', 'AxisError'] # mapping of {name: (value, deprecation_msg)} diff --git a/numpy/core/_exceptions.py b/numpy/core/_exceptions.py index 62579ed0d..87d4213a6 100644 --- a/numpy/core/_exceptions.py +++ b/numpy/core/_exceptions.py @@ -36,36 +36,35 @@ class UFuncTypeError(TypeError): @_display_as_base -class _UFuncBinaryResolutionError(UFuncTypeError): - """ Thrown when a binary resolution fails """ +class _UFuncNoLoopError(UFuncTypeError): + """ Thrown when a ufunc loop cannot be found """ def __init__(self, ufunc, dtypes): super().__init__(ufunc) self.dtypes = tuple(dtypes) - assert len(self.dtypes) == 2 def __str__(self): return ( - "ufunc {!r} cannot use operands with types {!r} and {!r}" + "ufunc {!r} did not contain a loop with signature matching types " + "{!r} -> {!r}" ).format( - self.ufunc.__name__, *self.dtypes + self.ufunc.__name__, + _unpack_tuple(self.dtypes[:self.ufunc.nin]), + _unpack_tuple(self.dtypes[self.ufunc.nin:]) ) @_display_as_base -class _UFuncNoLoopError(UFuncTypeError): - """ Thrown when a ufunc loop cannot be found """ +class _UFuncBinaryResolutionError(_UFuncNoLoopError): + """ Thrown when a binary resolution fails """ def __init__(self, ufunc, dtypes): - super().__init__(ufunc) - self.dtypes = tuple(dtypes) + super().__init__(ufunc, dtypes) + assert len(self.dtypes) == 2 def __str__(self): return ( - "ufunc {!r} did not contain a loop with signature matching types " - "{!r} -> {!r}" + "ufunc {!r} cannot use operands with types {!r} and {!r}" ).format( - self.ufunc.__name__, - _unpack_tuple(self.dtypes[:self.ufunc.nin]), - _unpack_tuple(self.dtypes[self.ufunc.nin:]) + self.ufunc.__name__, *self.dtypes ) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 85076f3e1..c78385880 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -9,6 +9,7 @@ import re import sys import warnings +from ..exceptions import DTypePromotionError from .multiarray import dtype, array, ndarray, promote_types try: import ctypes @@ -454,7 +455,8 @@ def _promote_fields(dt1, dt2): """ # Both must be structured and have the same names in the same order if (dt1.names is None or dt2.names is None) or dt1.names != dt2.names: - raise TypeError("invalid type promotion") + raise DTypePromotionError( + f"field names `{dt1.names}` and `{dt2.names}` mismatch.") # if both are identical, we can (maybe!) just return the same dtype. identical = dt1 is dt2 @@ -467,7 +469,8 @@ def _promote_fields(dt1, dt2): # Check that the titles match (if given): if field1[2:] != field2[2:]: - raise TypeError("invalid type promotion") + raise DTypePromotionError( + f"field titles of field '{name}' mismatch") if len(field1) == 2: new_fields.append((name, new_descr)) else: diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index ceafffd51..08e2cc683 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -875,108 +875,6 @@ DEPRECATE_silence_error(const char *msg) { return 0; } -/* - * Comparisons can fail, but we do not always want to pass on the exception - * (see comment in array_richcompare below), but rather return NotImplemented. - * Here, an exception should be set on entrance. - * Returns either NotImplemented with the exception cleared, or NULL - * with the exception set. - * Raises deprecation warnings for cases where behaviour is meant to change - * (2015-05-14, 1.10) - */ - -NPY_NO_EXPORT PyObject * -_failed_comparison_workaround(PyArrayObject *self, PyObject *other, int cmp_op) -{ - PyObject *exc, *val, *tb; - PyArrayObject *array_other; - int other_is_flexible, ndim_other; - int self_is_flexible = PyTypeNum_ISFLEXIBLE(PyArray_DESCR(self)->type_num); - - PyErr_Fetch(&exc, &val, &tb); - /* - * Determine whether other has a flexible dtype; here, inconvertible - * is counted as inflexible. (This repeats work done in the ufunc, - * but OK to waste some time in an unlikely path.) - */ - array_other = (PyArrayObject *)PyArray_FROM_O(other); - if (array_other) { - other_is_flexible = PyTypeNum_ISFLEXIBLE( - PyArray_DESCR(array_other)->type_num); - ndim_other = PyArray_NDIM(array_other); - Py_DECREF(array_other); - } - else { - PyErr_Clear(); /* we restore the original error if needed */ - other_is_flexible = 0; - ndim_other = 0; - } - if (cmp_op == Py_EQ || cmp_op == Py_NE) { - /* - * note: for == and !=, a structured dtype self cannot get here, - * but a string can. Other can be string or structured. - */ - if (other_is_flexible || self_is_flexible) { - /* - * For scalars, returning NotImplemented is correct. - * For arrays, we emit a future deprecation warning. - * When this warning is removed, a correctly shaped - * array of bool should be returned. - */ - if (ndim_other != 0 || PyArray_NDIM(self) != 0) { - /* 2015-05-14, 1.10 */ - if (DEPRECATE_FUTUREWARNING( - "elementwise comparison failed; returning scalar " - "instead, but in the future will perform " - "elementwise comparison") < 0) { - goto fail; - } - } - } - else { - /* - * If neither self nor other had a flexible dtype, the error cannot - * have been caused by a lack of implementation in the ufunc. - * - * 2015-05-14, 1.10 - */ - if (DEPRECATE( - "elementwise comparison failed; " - "this will raise an error in the future.") < 0) { - goto fail; - } - } - Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - else if (other_is_flexible || self_is_flexible) { - /* - * For LE, LT, GT, GE and a flexible self or other, we return - * NotImplemented, which is the correct answer since the ufuncs do - * not in fact implement loops for those. This will get us the - * desired TypeError. - */ - Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - else { - /* LE, LT, GT, or GE with non-flexible other; just pass on error */ - goto fail; - } - -fail: - /* - * Reraise the original exception, possibly chaining with a new one. - */ - npy_PyErr_ChainExceptionsCause(exc, val, tb); - return NULL; -} NPY_NO_EXPORT PyObject * array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) @@ -1074,33 +972,99 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (result == NULL) { + + /* + * At this point `self` can take control of the operation by converting + * `other` to an array (it would have a chance to take control). + * If we are not in `==` and `!=`, this is an error and we hope that + * the existing error makes sense and derives from `TypeError` (which + * python would raise for `NotImplemented`) when it should. + * + * However, if the issue is no matching loop for the given dtypes and + * we are inside == and !=, then returning an array of True or False + * makes sense (following Python behavior for `==` and `!=`). + * Effectively: Both *dtypes* told us that they cannot be compared. + * + * In theory, the error could be raised from within an object loop, the + * solution to that could be pushing this into the ufunc (where we can + * distinguish the two easily). In practice, it seems like it should not + * but a huge problem: The ufunc loop will itself call `==` which should + * probably never raise a UFuncNoLoopError. + * + * TODO: If/once we correctly push structured comparisons into the ufunc + * we could consider pushing this path into the ufunc itself as a + * fallback loop (which ignores the input arrays). + * This would have the advantage that subclasses implemementing + * `__array_ufunc__` do not explicitly need `__eq__` and `__ne__`. + */ + if (result == NULL + && (cmp_op == Py_EQ || cmp_op == Py_NE) + && PyErr_ExceptionMatches(npy_UFuncNoLoopError)) { + PyErr_Clear(); + + PyArrayObject *array_other = (PyArrayObject *)PyArray_FROM_O(other); + if (PyArray_TYPE(array_other) == NPY_VOID) { + /* + * Void arrays are currently not handled by ufuncs, so if the other + * is a void array, we defer to it (will raise a TypeError). + */ + Py_DECREF(array_other); + Py_RETURN_NOTIMPLEMENTED; + } + + if (PyArray_NDIM(self) == 0 && PyArray_NDIM(array_other) == 0) { + /* + * (seberg) not sure that this is best, but we preserve Python + * bool result for "scalar" inputs for now by returning + * `NotImplemented`. + */ + Py_DECREF(array_other); + Py_RETURN_NOTIMPLEMENTED; + } + + /* Hack warning: using NpyIter to allocate broadcasted result. */ + PyArrayObject *ops[3] = {self, array_other, NULL}; + npy_uint32 flags = NPY_ITER_ZEROSIZE_OK | NPY_ITER_REFS_OK; + npy_uint32 op_flags[3] = { + NPY_ITER_READONLY, NPY_ITER_READONLY, + NPY_ITER_ALLOCATE | NPY_ITER_WRITEONLY}; + + PyArray_Descr *bool_descr = PyArray_DescrFromType(NPY_BOOL); + PyArray_Descr *op_descrs[3] = { + PyArray_DESCR(self), PyArray_DESCR(array_other), bool_descr}; + + NpyIter *iter = NpyIter_MultiNew( + 3, ops, flags, NPY_KEEPORDER, NPY_NO_CASTING, + op_flags, op_descrs); + + Py_CLEAR(bool_descr); + Py_CLEAR(array_other); + if (iter == NULL) { + return NULL; + } + PyArrayObject *res = NpyIter_GetOperandArray(iter)[2]; + Py_INCREF(res); + if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { + Py_DECREF(res); + return NULL; + } + /* - * 2015-05-14, 1.10; updated 2018-06-18, 1.16. - * - * Comparisons can raise errors when element-wise comparison is not - * possible. Some of these, though, should not be passed on. - * In particular, the ufuncs do not have loops for flexible dtype, - * so those should be treated separately. Furthermore, for EQ and NE, - * we should never fail. - * - * Our ideal behaviour would be: - * - * 1. For EQ and NE: - * - If self and other are scalars, return NotImplemented, - * so that python can assign True of False as appropriate. - * - If either is an array, return an array of False or True. - * - * 2. For LT, LE, GE, GT: - * - If self or other was flexible, return NotImplemented - * (as is in fact the case), so python can raise a TypeError. - * - If other is not convertible to an array, pass on the error - * (MHvK, 2018-06-18: not sure about this, but it's what we have). - * - * However, for backwards compatibility, we cannot yet return arrays, - * so we raise warnings instead. + * The array is guaranteed to be newly allocated and thus contiguous, + * so simply fill it with 0 or 1. */ - result = _failed_comparison_workaround(self, other, cmp_op); + memset(PyArray_BYTES(res), cmp_op == Py_EQ ? 0 : 1, PyArray_NBYTES(res)); + + /* Ensure basic subclass support by wrapping: */ + if (!PyArray_CheckExact(self)) { + /* + * If other is also a subclass (with higher priority) we would + * already have deferred. So use `self` for wrapping. If users + * need more, they need to override `==` and `!=`. + */ + Py_SETREF(res, PyArray_SubclassWrap(self, res)); + } + return (PyObject *)res; } return result; } diff --git a/numpy/core/src/multiarray/common_dtype.c b/numpy/core/src/multiarray/common_dtype.c index 3561a905a..38a130221 100644 --- a/numpy/core/src/multiarray/common_dtype.c +++ b/numpy/core/src/multiarray/common_dtype.c @@ -8,6 +8,7 @@ #include "numpy/arrayobject.h" #include "common_dtype.h" +#include "convert_datatype.h" #include "dtypemeta.h" #include "abstractdtypes.h" @@ -61,7 +62,7 @@ PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) } if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { Py_DECREF(Py_NotImplemented); - PyErr_Format(PyExc_TypeError, + PyErr_Format(npy_DTypePromotionError, "The DTypes %S and %S do not have a common DType. " "For example they cannot be stored in a single array unless " "the dtype is `object`.", dtype1, dtype2); @@ -288,7 +289,7 @@ PyArray_PromoteDTypeSequence( Py_INCREF(dtypes_in[l]); PyTuple_SET_ITEM(dtypes_in_tuple, l, (PyObject *)dtypes_in[l]); } - PyErr_Format(PyExc_TypeError, + PyErr_Format(npy_DTypePromotionError, "The DType %S could not be promoted by %S. This means that " "no common DType exists for the given inputs. " "For example they cannot be stored in a single array unless " diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index eeb42df66..3973fc795 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -50,6 +50,9 @@ NPY_NO_EXPORT npy_intp REQUIRED_STR_LEN[] = {0, 3, 5, 10, 10, 20, 20, 20, 20}; */ NPY_NO_EXPORT int npy_promotion_state = NPY_USE_LEGACY_PROMOTION; NPY_NO_EXPORT PyObject *NO_NEP50_WARNING_CTX = NULL; +NPY_NO_EXPORT PyObject *npy_DTypePromotionError = NULL; +NPY_NO_EXPORT PyObject *npy_UFuncNoLoopError = NULL; + static PyObject * PyArray_GetGenericToVoidCastingImpl(void); diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index b6bc7d8a7..1a23965f8 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -14,6 +14,8 @@ extern NPY_NO_EXPORT npy_intp REQUIRED_STR_LEN[]; #define NPY_USE_WEAK_PROMOTION_AND_WARN 2 extern NPY_NO_EXPORT int npy_promotion_state; extern NPY_NO_EXPORT PyObject *NO_NEP50_WARNING_CTX; +extern NPY_NO_EXPORT PyObject *npy_DTypePromotionError; +extern NPY_NO_EXPORT PyObject *npy_UFuncNoLoopError; NPY_NO_EXPORT int npy_give_promotion_warnings(void); diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 2abd68ca2..695b696c2 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1622,6 +1622,12 @@ compute_datetime_metadata_greatest_common_divisor( return 0; + /* + * We do not use `DTypePromotionError` below. The reason this is that a + * `DTypePromotionError` indicates that `arr_dt1 != arr_dt2` for + * all values, but this is wrong for "0". This could be changed but + * for now we consider them errors that occur _while_ promoting. + */ incompatible_units: { PyObject *umeta1 = metastr_to_unicode(meta1, 0); if (umeta1 == NULL) { diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 6c33da729..edc07bc92 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -404,7 +404,7 @@ void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) if (descr1->subarray == NULL && descr1->names == NULL && descr2->subarray == NULL && descr2->names == NULL) { if (descr1->elsize != descr2->elsize) { - PyErr_SetString(PyExc_TypeError, + PyErr_SetString(npy_DTypePromotionError, "Invalid type promotion with void datatypes of different " "lengths. Use the `np.bytes_` datatype instead to pad the " "shorter value with trailing zero bytes."); @@ -443,7 +443,7 @@ void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) return NULL; } if (!cmp) { - PyErr_SetString(PyExc_TypeError, + PyErr_SetString(npy_DTypePromotionError, "invalid type promotion with subarray datatypes " "(shape mismatch)."); return NULL; @@ -473,7 +473,7 @@ void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) return new_descr; } - PyErr_SetString(PyExc_TypeError, + PyErr_SetString(npy_DTypePromotionError, "invalid type promotion with structured datatype(s)."); return NULL; } @@ -617,6 +617,12 @@ string_unicode_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) static PyArray_DTypeMeta * datetime_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) { + /* + * Timedelta/datetime shouldn't actuall promote at all. That they + * currently do means that we need additional hacks in the comparison + * type resolver. For comparisons we have to make sure we reject it + * nicely in order to return an array of True/False values. + */ if (cls->type_num == NPY_DATETIME && other->type_num == NPY_TIMEDELTA) { /* * TODO: We actually currently do allow promotion here. This is diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index ca4fdfeca..5da3d66df 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4811,6 +4811,38 @@ intern_strings(void) return 0; } + +/* + * Initializes global constants. At some points these need to be cleaned + * up, and sometimes we also import them where they are needed. But for + * some things, adding an `npy_cache_import` everywhere seems inconvenient. + * + * These globals should not need the C-layer at all and will be imported + * before anything on the C-side is initialized. + */ +static int +initialize_static_globals(void) +{ + assert(npy_DTypePromotionError == NULL); + npy_cache_import( + "numpy.exceptions", "DTypePromotionError", + &npy_DTypePromotionError); + if (npy_DTypePromotionError == NULL) { + return -1; + } + + assert(npy_UFuncNoLoopError == NULL); + npy_cache_import( + "numpy.core._exceptions", "_UFuncNoLoopError", + &npy_UFuncNoLoopError); + if (npy_UFuncNoLoopError == NULL) { + return -1; + } + + return 0; +} + + static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_multiarray_umath", @@ -4861,6 +4893,14 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } + if (intern_strings() < 0) { + goto err; + } + + if (initialize_static_globals() < 0) { + goto err; + } + if (PyType_Ready(&PyUFunc_Type) < 0) { goto err; } @@ -5033,10 +5073,6 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } - if (intern_strings() < 0) { - goto err; - } - if (set_typeinfo(d) != 0) { goto err; } diff --git a/numpy/core/src/umath/dispatching.c b/numpy/core/src/umath/dispatching.c index 2de5a5670..6d6c481fb 100644 --- a/numpy/core/src/umath/dispatching.c +++ b/numpy/core/src/umath/dispatching.c @@ -43,6 +43,7 @@ #include <convert_datatype.h> #include "numpy/ndarraytypes.h" +#include "numpy/npy_3kcompat.h" #include "common.h" #include "dispatching.h" @@ -947,7 +948,7 @@ promote_and_get_ufuncimpl(PyUFuncObject *ufunc, int cacheable = 1; /* unused, as we modify the original `op_dtypes` */ if (legacy_promote_using_legacy_type_resolver(ufunc, ops, signature, op_dtypes, &cacheable, NPY_FALSE) < 0) { - return NULL; + goto handle_error; } } @@ -959,10 +960,7 @@ promote_and_get_ufuncimpl(PyUFuncObject *ufunc, npy_promotion_state = old_promotion_state; if (info == NULL) { - if (!PyErr_Occurred()) { - raise_no_loop_found_error(ufunc, (PyObject **)op_dtypes); - } - return NULL; + goto handle_error; } PyArrayMethodObject *method = (PyArrayMethodObject *)PyTuple_GET_ITEM(info, 1); @@ -984,7 +982,7 @@ promote_and_get_ufuncimpl(PyUFuncObject *ufunc, /* Reset the promotion state: */ npy_promotion_state = NPY_USE_WEAK_PROMOTION_AND_WARN; if (res < 0) { - return NULL; + goto handle_error; } } @@ -1018,12 +1016,29 @@ promote_and_get_ufuncimpl(PyUFuncObject *ufunc, * If signature is forced the cache may contain an incompatible * loop found via promotion (signature not enforced). Reject it. */ - raise_no_loop_found_error(ufunc, (PyObject **)op_dtypes); - return NULL; + goto handle_error; } } return method; + + handle_error: + /* We only set the "no loop found error here" */ + if (!PyErr_Occurred()) { + raise_no_loop_found_error(ufunc, (PyObject **)op_dtypes); + } + /* + * Otherwise an error occurred, but if the error was DTypePromotionError + * then we chain it, because DTypePromotionError effectively means that there + * is no loop available. (We failed finding a loop by using promotion.) + */ + else if (PyErr_ExceptionMatches(npy_DTypePromotionError)) { + PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; + PyErr_Fetch(&err_type, &err_value, &err_traceback); + raise_no_loop_found_error(ufunc, (PyObject **)op_dtypes); + npy_PyErr_ChainExceptionsCause(err_type, err_value, err_traceback); + } + return NULL; } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 707f39e94..a0a16a0f9 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -358,13 +358,24 @@ PyUFunc_SimpleBinaryComparisonTypeResolver(PyUFuncObject *ufunc, } if (type_tup == NULL) { + if (PyArray_ISDATETIME(operands[0]) + && PyArray_ISDATETIME(operands[1]) + && type_num1 != type_num2) { + /* + * Reject mixed datetime and timedelta explictly, this was always + * implicitly rejected because casting fails (except with + * `casting="unsafe"` admittedly). + * This is required to ensure that `==` and `!=` can correctly + * detect that they should return a result array of False/True. + */ + return raise_binary_type_reso_error(ufunc, operands); + } /* - * DEPRECATED NumPy 1.20, 2020-12. - * This check is required to avoid the FutureWarning that - * ResultType will give for number->string promotions. + * This check is required to avoid a potential FutureWarning that + * ResultType would give for number->string promotions. * (We never supported flexible dtypes here.) */ - if (!PyArray_ISFLEXIBLE(operands[0]) && + else if (!PyArray_ISFLEXIBLE(operands[0]) && !PyArray_ISFLEXIBLE(operands[1])) { out_dtypes[0] = PyArray_ResultType(2, operands, 0, NULL); if (out_dtypes[0] == NULL) { diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 3a8db40df..4ec1f83d4 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -138,80 +138,6 @@ class _VisibleDeprecationTestCase(_DeprecationTestCase): warning_cls = np.VisibleDeprecationWarning -class TestComparisonDeprecations(_DeprecationTestCase): - """This tests the deprecation, for non-element-wise comparison logic. - This used to mean that when an error occurred during element-wise comparison - (i.e. broadcasting) NotImplemented was returned, but also in the comparison - itself, False was given instead of the error. - - Also test FutureWarning for the None comparison. - """ - - message = "elementwise.* comparison failed; .*" - - def test_normal_types(self): - for op in (operator.eq, operator.ne): - # Broadcasting errors: - self.assert_deprecated(op, args=(np.zeros(3), [])) - a = np.zeros(3, dtype='i,i') - # (warning is issued a couple of times here) - self.assert_deprecated(op, args=(a, a[:-1]), num=None) - - # ragged array comparison returns True/False - a = np.array([1, np.array([1,2,3])], dtype=object) - b = np.array([1, np.array([1,2,3])], dtype=object) - self.assert_deprecated(op, args=(a, b), num=None) - - def test_string(self): - # For two string arrays, strings always raised the broadcasting error: - a = np.array(['a', 'b']) - b = np.array(['a', 'b', 'c']) - assert_warns(FutureWarning, lambda x, y: x == y, a, b) - - # The empty list is not cast to string, and this used to pass due - # to dtype mismatch; now (2018-06-21) it correctly leads to a - # FutureWarning. - assert_warns(FutureWarning, lambda: a == []) - - def test_void_dtype_equality_failures(self): - class NotArray: - def __array__(self): - raise TypeError - - # Needed so Python 3 does not raise DeprecationWarning twice. - def __ne__(self, other): - return NotImplemented - - self.assert_deprecated(lambda: np.arange(2) == NotArray()) - self.assert_deprecated(lambda: np.arange(2) != NotArray()) - - def test_array_richcompare_legacy_weirdness(self): - # It doesn't really work to use assert_deprecated here, b/c part of - # the point of assert_deprecated is to check that when warnings are - # set to "error" mode then the error is propagated -- which is good! - # But here we are testing a bunch of code that is deprecated *because* - # it has the habit of swallowing up errors and converting them into - # different warnings. So assert_warns will have to be sufficient. - assert_warns(FutureWarning, lambda: np.arange(2) == "a") - assert_warns(FutureWarning, lambda: np.arange(2) != "a") - # No warning for scalar comparisons - with warnings.catch_warnings(): - warnings.filterwarnings("error") - assert_(not (np.array(0) == "a")) - assert_(np.array(0) != "a") - assert_(not (np.int16(0) == "a")) - assert_(np.int16(0) != "a") - - for arg1 in [np.asarray(0), np.int16(0)]: - struct = np.zeros(2, dtype="i4,i4") - for arg2 in [struct, "a"]: - for f in [operator.lt, operator.le, operator.gt, operator.ge]: - with warnings.catch_warnings() as l: - warnings.filterwarnings("always") - assert_raises(TypeError, f, arg1, arg2) - assert_(not l) - - class TestDatetime64Timezone(_DeprecationTestCase): """Parsing of datetime64 with timezones deprecated in 1.11.0, because datetime64 is now timezone naive rather than UTC only. diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 4d3996d86..63ac32f20 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1271,6 +1271,13 @@ class TestStructured: a = np.zeros((1, 0, 1), [('a', '<f8', (1, 1))]) assert_equal(a, a) + @pytest.mark.parametrize("op", [operator.eq, operator.ne]) + def test_structured_array_comparison_bad_broadcasts(self, op): + a = np.zeros(3, dtype='i,i') + b = np.array([], dtype="i,i") + with pytest.raises(ValueError): + op(a, b) + def test_structured_comparisons_with_promotion(self): # Check that structured arrays can be compared so long as their # dtypes promote fine: @@ -1291,7 +1298,10 @@ class TestStructured: assert_equal(a == b, [False, True]) assert_equal(a != b, [True, False]) - def test_void_comparison_failures(self): + @pytest.mark.parametrize("op", [ + operator.eq, lambda x, y: operator.eq(y, x), + operator.ne, lambda x, y: operator.ne(y, x)]) + def test_void_comparison_failures(self, op): # In principle, one could decide to return an array of False for some # if comparisons are impossible. But right now we return TypeError # when "void" dtype are involved. @@ -1299,18 +1309,18 @@ class TestStructured: y = np.zeros(3) # Cannot compare non-structured to structured: with pytest.raises(TypeError): - x == y + op(x, y) # Added title prevents promotion, but casts are OK: y = np.zeros(3, dtype=[(('title', 'a'), 'i1')]) assert np.can_cast(y.dtype, x.dtype) with pytest.raises(TypeError): - x == y + op(x, y) x = np.zeros(3, dtype="V7") y = np.zeros(3, dtype="V8") with pytest.raises(TypeError): - x == y + op(x, y) def test_casting(self): # Check that casting a structured array to change its byte order @@ -9493,6 +9503,105 @@ def test_equal_override(): assert_equal(array != my_always_equal, 'ne') +@pytest.mark.parametrize("op", [operator.eq, operator.ne]) +@pytest.mark.parametrize(["dt1", "dt2"], [ + ([("f", "i")], [("f", "i")]), # structured comparison (successfull) + ("M8", "d"), # impossible comparison: result is all True or False + ("d", "d"), # valid comparison + ]) +def test_equal_subclass_no_override(op, dt1, dt2): + # Test how the three different possible code-paths deal with subclasses + + class MyArr(np.ndarray): + called_wrap = 0 + + def __array_wrap__(self, new): + type(self).called_wrap += 1 + return super().__array_wrap__(new) + + numpy_arr = np.zeros(5, dtype=dt1) + my_arr = np.zeros(5, dtype=dt2).view(MyArr) + + assert type(op(numpy_arr, my_arr)) is MyArr + assert type(op(my_arr, numpy_arr)) is MyArr + # We expect 2 calls (more if there were more fields): + assert MyArr.called_wrap == 2 + + +@pytest.mark.parametrize(["dt1", "dt2"], [ + ("M8[ns]", "d"), + ("M8[s]", "l"), + ("m8[ns]", "d"), + # Missing: ("m8[ns]", "l") as timedelta currently promotes ints + ("M8[s]", "m8[s]"), + ("S5", "U5"), + # Structured/void dtypes have explicit paths not tested here. +]) +def test_no_loop_gives_all_true_or_false(dt1, dt2): + # Make sure they broadcast to test result shape, use random values, since + # the actual value should be ignored + arr1 = np.random.randint(5, size=100).astype(dt1) + arr2 = np.random.randint(5, size=99)[:, np.newaxis].astype(dt2) + + res = arr1 == arr2 + assert res.shape == (99, 100) + assert res.dtype == bool + assert not res.any() + + res = arr1 != arr2 + assert res.shape == (99, 100) + assert res.dtype == bool + assert res.all() + + # incompatible shapes raise though + arr2 = np.random.randint(5, size=99).astype(dt2) + with pytest.raises(ValueError): + arr1 == arr2 + + with pytest.raises(ValueError): + arr1 != arr2 + + # Basic test with another operation: + with pytest.raises(np.core._exceptions._UFuncNoLoopError): + arr1 > arr2 + + +@pytest.mark.parametrize("op", [ + operator.eq, operator.ne, operator.le, operator.lt, operator.ge, + operator.gt]) +def test_comparisons_forwards_error(op): + class NotArray: + def __array__(self): + raise TypeError("run you fools") + + with pytest.raises(TypeError, match="run you fools"): + op(np.arange(2), NotArray()) + + with pytest.raises(TypeError, match="run you fools"): + op(NotArray(), np.arange(2)) + + +def test_richcompare_scalar_boolean_singleton_return(): + # These are currently guaranteed to be the boolean singletons, but maybe + # returning NumPy booleans would also be OK: + assert (np.array(0) == "a") is False + assert (np.array(0) != "a") is True + assert (np.int16(0) == "a") is False + assert (np.int16(0) != "a") is True + + +@pytest.mark.parametrize("op", [ + operator.eq, operator.ne, operator.le, operator.lt, operator.ge, + operator.gt]) +def test_ragged_comparison_fails(op): + # This needs to convert the internal array to True/False, which fails: + a = np.array([1, np.array([1, 2, 3])], dtype=object) + b = np.array([1, np.array([1, 2, 3])], dtype=object) + + with pytest.raises(ValueError, match="The truth value.*ambiguous"): + op(a, b) + + @pytest.mark.parametrize( ["fun", "npfun"], [ diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 160e4a3a4..f638284de 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -128,10 +128,7 @@ class TestRegression: assert_(a[1] == 'auto') assert_(a[0] != 'auto') b = np.linspace(0, 10, 11) - # This should return true for now, but will eventually raise an error: - with suppress_warnings() as sup: - sup.filter(FutureWarning) - assert_(b != 'auto') + assert_array_equal(b != 'auto', np.ones(11, dtype=bool)) assert_(b[0] != 'auto') def test_unicode_swapping(self): diff --git a/numpy/core/tests/test_unicode.py b/numpy/core/tests/test_unicode.py index 2d7c2818e..e5454bd48 100644 --- a/numpy/core/tests/test_unicode.py +++ b/numpy/core/tests/test_unicode.py @@ -36,10 +36,10 @@ def test_string_cast(): uni_arr1 = str_arr.astype('>U') uni_arr2 = str_arr.astype('<U') - with pytest.warns(FutureWarning): - assert str_arr != uni_arr1 - with pytest.warns(FutureWarning): - assert str_arr != uni_arr2 + assert_array_equal(str_arr != uni_arr1, np.ones(2, dtype=bool)) + assert_array_equal(uni_arr1 != str_arr, np.ones(2, dtype=bool)) + assert_array_equal(str_arr == uni_arr1, np.zeros(2, dtype=bool)) + assert_array_equal(uni_arr1 == str_arr, np.zeros(2, dtype=bool)) assert_array_equal(uni_arr1, uni_arr2) diff --git a/numpy/exceptions.py b/numpy/exceptions.py index 81a2f3c65..721b8102e 100644 --- a/numpy/exceptions.py +++ b/numpy/exceptions.py @@ -25,8 +25,9 @@ Exceptions .. autosummary:: :toctree: generated/ - AxisError Given when an axis was invalid. - TooHardError Error specific to `numpy.shares_memory`. + AxisError Given when an axis was invalid. + DTypePromotionError Given when no common dtype could be found. + TooHardError Error specific to `numpy.shares_memory`. """ @@ -35,7 +36,7 @@ from ._utils import set_module as _set_module __all__ = [ "ComplexWarning", "VisibleDeprecationWarning", - "TooHardError", "AxisError"] + "TooHardError", "AxisError", "DTypePromotionError"] # Disallow reloading this module so as to preserve the identities of the @@ -195,3 +196,48 @@ class AxisError(ValueError, IndexError): if self._msg is not None: msg = f"{self._msg}: {msg}" return msg + + +class DTypePromotionError(TypeError): + """Multiple DTypes could not be converted to a common one. + + This exception derives from ``TypeError`` and is raised whenever dtypes + cannot be converted to a single common one. This can be because they + are of a different category/class or incompatible instances of the same + one (see Examples). + + Notes + ----- + Many functions will use promotion to find the correct result and + implementation. For these functions the error will typically be chained + with a more specific error indicating that no implementation was found + for the input dtypes. + + Typically promotion should be considered "invalid" between the dtypes of + two arrays when `arr1 == arr2` can safely return all ``False`` because the + dtypes are fundamentally different. + + Examples + -------- + Datetimes and complex numbers are incompatible classes and cannot be + promoted: + + >>> np.result_type(np.dtype("M8[s]"), np.complex128) + DTypePromotionError: The DType <class 'numpy.dtype[datetime64]'> could not + be promoted by <class 'numpy.dtype[complex128]'>. This means that no common + DType exists for the given inputs. For example they cannot be stored in a + single array unless the dtype is `object`. The full list of DTypes is: + (<class 'numpy.dtype[datetime64]'>, <class 'numpy.dtype[complex128]'>) + + For example for structured dtypes, the structure can mismatch and the + same ``DTypePromotionError`` is given when two structured dtypes with + a mismatch in their number of fields is given: + + >>> dtype1 = np.dtype([("field1", np.float64), ("field2", np.int64)]) + >>> dtype2 = np.dtype([("field1", np.float64)]) + >>> np.promote_types(dtype1, dtype2) + DTypePromotionError: field names `('field1', 'field2')` and `('field1',)` + mismatch. + + """ + pass diff --git a/numpy/linalg/tests/test_regression.py b/numpy/linalg/tests/test_regression.py index 7ed932bc9..af38443a9 100644 --- a/numpy/linalg/tests/test_regression.py +++ b/numpy/linalg/tests/test_regression.py @@ -107,10 +107,7 @@ class TestRegression: assert_raises(ValueError, linalg.norm, testvector, ord='nuc') assert_raises(ValueError, linalg.norm, testvector, ord=np.inf) assert_raises(ValueError, linalg.norm, testvector, ord=-np.inf) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - assert_raises((AttributeError, DeprecationWarning), - linalg.norm, testvector, ord=0) + assert_raises(ValueError, linalg.norm, testvector, ord=0) assert_raises(ValueError, linalg.norm, testvector, ord=-1) assert_raises(ValueError, linalg.norm, testvector, ord=-2) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 4cd602510..8f654f3d9 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -361,6 +361,7 @@ SKIP_LIST_2 = [ 'numpy.matlib.char', 'numpy.matlib.rec', 'numpy.matlib.emath', + 'numpy.matlib.exceptions', 'numpy.matlib.math', 'numpy.matlib.linalg', 'numpy.matlib.fft', |