diff options
Diffstat (limited to 'numpy/core')
22 files changed, 998 insertions, 436 deletions
diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 4a4f3b6e7..c2458c2b5 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -31,6 +31,7 @@ API_FILES = [join('multiarray', 'alloc.c'), join('multiarray', 'arraytypes.c.src'), join('multiarray', 'buffer.c'), join('multiarray', 'calculation.c'), + join('multiarray', 'common_dtype.c'), join('multiarray', 'conversion_utils.c'), join('multiarray', 'convert.c'), join('multiarray', 'convert_datatype.c'), diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index dacb72022..d1acfdf26 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -856,6 +856,17 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); */ #define NPY_ARRAY_ENSUREARRAY 0x0040 +#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD + /* + * Dual use of the ENSUREARRAY flag, to indicate that this was converted + * from a python float, int, or complex. + * An array using this flag must be a temporary array that can never + * leave the C internals of NumPy. Even if it does, ENSUREARRAY is + * absolutely safe to abuse, since it already is a base class array :). + */ + #define _NPY_ARRAY_WAS_PYSCALAR 0x0040 +#endif /* NPY_INTERNAL_BUILD */ + /* * Make sure that the strides are in units of the element size Needed * for some operations with record-arrays. @@ -1867,6 +1878,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls); typedef PyArray_DTypeMeta *(common_dtype_function)( PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2); + typedef PyArray_DTypeMeta *(common_dtype_with_value_function)( + PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2, PyObject *value); typedef PyArray_Descr *(common_instance_function)( PyArray_Descr *dtype1, PyArray_Descr *dtyep2); @@ -1925,6 +1938,7 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, is_known_scalar_type_function *is_known_scalar_type; default_descr_function *default_descr; common_dtype_function *common_dtype; + common_dtype_with_value_function *common_dtype_with_value; common_instance_function *common_instance; /* * The casting implementation (ArrayMethod) to convert between two diff --git a/numpy/core/setup.py b/numpy/core/setup.py index d1229ee8f..b03e9f990 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -23,11 +23,6 @@ NPY_RELAXED_STRIDES_CHECKING = (os.environ.get('NPY_RELAXED_STRIDES_CHECKING', " NPY_RELAXED_STRIDES_DEBUG = (os.environ.get('NPY_RELAXED_STRIDES_DEBUG', "0") != "0") NPY_RELAXED_STRIDES_DEBUG = NPY_RELAXED_STRIDES_DEBUG and NPY_RELAXED_STRIDES_CHECKING -# Set to True to use the new casting implementation as much as implemented. -# Allows running the full test suit to exercise the new machinery until -# it is used as default and the old version is eventually deleted. -NPY_USE_NEW_CASTINGIMPL = os.environ.get('NPY_USE_NEW_CASTINGIMPL', "0") != "0" - # XXX: ugly, we use a class to avoid calling twice some expensive functions in # config.h/numpyconfig.h. I don't see a better way because distutils force # config.h generation inside an Extension class, and as such sharing @@ -472,12 +467,6 @@ def configuration(parent_package='',top_path=None): else: moredefs.append(('NPY_RELAXED_STRIDES_DEBUG', 0)) - # Use the new experimental casting implementation in NumPy 1.20: - if NPY_USE_NEW_CASTINGIMPL: - moredefs.append(('NPY_USE_NEW_CASTINGIMPL', 1)) - else: - moredefs.append(('NPY_USE_NEW_CASTINGIMPL', 0)) - # Get long double representation rep = check_long_double_representation(config_cmd) moredefs.append(('HAVE_LDOUBLE_%s' % rep, 1)) @@ -787,6 +776,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'npy_buffer.h'), join('src', 'multiarray', 'calculation.h'), join('src', 'multiarray', 'common.h'), + join('src', 'multiarray', 'common_dtype.h'), join('src', 'multiarray', 'convert_datatype.h'), join('src', 'multiarray', 'convert.h'), join('src', 'multiarray', 'conversion_utils.h'), @@ -849,6 +839,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'calculation.c'), join('src', 'multiarray', 'compiled_base.c'), join('src', 'multiarray', 'common.c'), + join('src', 'multiarray', 'common_dtype.c'), join('src', 'multiarray', 'convert.c'), join('src', 'multiarray', 'convert_datatype.c'), join('src', 'multiarray', 'conversion_utils.c'), diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index ba10573d9..bfdeae079 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -2141,17 +2141,6 @@ getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) static PyObject * -uses_new_casts(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) -{ -#if NPY_USE_NEW_CASTINGIMPL - Py_RETURN_TRUE; -#else - Py_RETURN_FALSE; -#endif -} - - -static PyObject * run_byteorder_converter(PyObject* NPY_UNUSED(self), PyObject *args) { char byteorder; @@ -2407,9 +2396,6 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"getset_numericops", getset_numericops, METH_NOARGS, NULL}, - {"uses_new_casts", - uses_new_casts, - METH_NOARGS, NULL}, /**begin repeat * #name = cabs, carg# */ diff --git a/numpy/core/src/multiarray/abstractdtypes.c b/numpy/core/src/multiarray/abstractdtypes.c index 02c0eac53..587d91c49 100644 --- a/numpy/core/src/multiarray/abstractdtypes.c +++ b/numpy/core/src/multiarray/abstractdtypes.c @@ -13,6 +13,12 @@ #include "common.h" +static NPY_INLINE PyArray_Descr * +int_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_LONG); +} + static PyArray_Descr * discover_descriptor_from_pyint( PyArray_DTypeMeta *NPY_UNUSED(cls), PyObject *obj) @@ -45,6 +51,13 @@ discover_descriptor_from_pyint( } +static NPY_INLINE PyArray_Descr * +float_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_DOUBLE); +} + + static PyArray_Descr* discover_descriptor_from_pyfloat( PyArray_DTypeMeta* NPY_UNUSED(cls), PyObject *obj) @@ -53,6 +66,11 @@ discover_descriptor_from_pyfloat( return PyArray_DescrFromType(NPY_DOUBLE); } +static NPY_INLINE PyArray_Descr * +complex_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_CDOUBLE); +} static PyArray_Descr* discover_descriptor_from_pycomplex( @@ -66,21 +84,17 @@ discover_descriptor_from_pycomplex( NPY_NO_EXPORT int initialize_and_map_pytypes_to_dtypes() { - PyArrayAbstractObjDTypeMeta_Type.tp_base = &PyArrayDTypeMeta_Type; - if (PyType_Ready(&PyArrayAbstractObjDTypeMeta_Type) < 0) { - return -1; - } - ((PyTypeObject *)&PyArray_PyIntAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyIntAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyIntAbstractDType.scalar_type = &PyLong_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyIntAbstractDType) < 0) { return -1; } - ((PyTypeObject *)&PyArray_PyFloatAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyFloatAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyFloatAbstractDType.scalar_type = &PyFloat_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyFloatAbstractDType) < 0) { return -1; } - ((PyTypeObject *)&PyArray_PyComplexAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyComplexAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyComplexAbstractDType.scalar_type = &PyComplex_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyComplexAbstractDType) < 0) { return -1; @@ -126,43 +140,147 @@ initialize_and_map_pytypes_to_dtypes() } +/* + * The following functions define the "common DType" for the abstract dtypes. + * + * Note that the logic with respect to the "higher" dtypes such as floats + * could likely be more logically defined for them, but since NumPy dtypes + * largely "know" each other, that is not necessary. + */ +static PyArray_DTypeMeta * +int_common_dtype(PyArray_DTypeMeta *NPY_UNUSED(cls), PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL) { + /* Use the default integer for bools: */ + return PyArray_DTypeFromTypeNum(NPY_LONG); + } + else if (PyTypeNum_ISNUMBER(other->type_num) || + other->type_num == NPY_TIMEDELTA) { + /* All other numeric types (ant timdelta) are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_UINT8); + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} -/* Note: This is currently largely not used, but will be required eventually. */ -NPY_NO_EXPORT PyTypeObject PyArrayAbstractObjDTypeMeta_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "numpy._AbstractObjDTypeMeta", - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Helper MetaClass for value based casting AbstractDTypes.", -}; +static PyArray_DTypeMeta * +float_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL || PyTypeNum_ISINTEGER(other->type_num)) { + /* Use the default integer for bools and ints: */ + return PyArray_DTypeFromTypeNum(NPY_DOUBLE); + } + else if (PyTypeNum_ISNUMBER(other->type_num)) { + /* All other numeric types (float+complex) are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other == &PyArray_PyIntAbstractDType) { + Py_INCREF(cls); + return cls; + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_HALF); + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} + + +static PyArray_DTypeMeta * +complex_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL || + PyTypeNum_ISINTEGER(other->type_num)) { + /* Use the default integer for bools and ints: */ + return PyArray_DTypeFromTypeNum(NPY_CDOUBLE); + } + else if (PyTypeNum_ISFLOAT(other->type_num)) { + /* + * For floats we choose the equivalent precision complex, although + * there is no CHALF, so half also goes to CFLOAT. + */ + if (other->type_num == NPY_HALF || other->type_num == NPY_FLOAT) { + return PyArray_DTypeFromTypeNum(NPY_CFLOAT); + } + if (other->type_num == NPY_DOUBLE) { + return PyArray_DTypeFromTypeNum(NPY_CDOUBLE); + } + assert(other->type_num == NPY_LONGDOUBLE); + return PyArray_DTypeFromTypeNum(NPY_CLONGDOUBLE); + } + else if (PyTypeNum_ISCOMPLEX(other->type_num)) { + /* All other numeric types are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_CFLOAT); + } + else if (other == &PyArray_PyIntAbstractDType || + other == &PyArray_PyFloatAbstractDType) { + Py_INCREF(cls); + return cls; + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} + + +/* + * TODO: These abstract DTypes also carry the dual role of representing + * `Floating`, `Complex`, and `Integer` (both signed and unsigned). + * They will have to be renamed and exposed in that capacity. + */ NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyIntAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyIntBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._IntegerAbstractDType", },}, .abstract = 1, + .default_descr = int_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pyint, + .common_dtype = int_common_dtype, .kind = 'i', }; NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyFloatAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyFloatBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._FloatAbstractDType", },}, .abstract = 1, + .default_descr = float_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pyfloat, + .common_dtype = float_common_dtype, .kind = 'f', }; NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyComplexAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyComplexBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._ComplexAbstractDType", },}, .abstract = 1, + .default_descr = complex_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pycomplex, + .common_dtype = complex_common_dtype, .kind = 'c', }; - diff --git a/numpy/core/src/multiarray/abstractdtypes.h b/numpy/core/src/multiarray/abstractdtypes.h index 3a982cd38..a6c526717 100644 --- a/numpy/core/src/multiarray/abstractdtypes.h +++ b/numpy/core/src/multiarray/abstractdtypes.h @@ -3,12 +3,12 @@ #include "dtypemeta.h" + /* * These are mainly needed for value based promotion in ufuncs. It * may be necessary to make them (partially) public, to allow user-defined * dtypes to perform value based casting. */ -NPY_NO_EXPORT extern PyTypeObject PyArrayAbstractObjDTypeMeta_Type; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyIntAbstractDType; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyFloatAbstractDType; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyComplexAbstractDType; diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 6b7c3888d..22050a56f 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -11,6 +11,7 @@ #include "descriptor.h" #include "convert_datatype.h" +#include "common_dtype.h" #include "dtypemeta.h" #include "array_coercion.h" @@ -205,7 +206,7 @@ _PyArray_MapPyTypeToDType( * @return DType, None if it a known non-scalar, or NULL if an unknown object. */ static NPY_INLINE PyArray_DTypeMeta * -discover_dtype_from_pytype(PyTypeObject *pytype) +npy_discover_dtype_from_pytype(PyTypeObject *pytype) { PyObject *DType; @@ -263,7 +264,7 @@ discover_dtype_from_pyobject( } } - PyArray_DTypeMeta *DType = discover_dtype_from_pytype(Py_TYPE(obj)); + PyArray_DTypeMeta *DType = npy_discover_dtype_from_pytype(Py_TYPE(obj)); if (DType != NULL) { return DType; } diff --git a/numpy/core/src/multiarray/array_coercion.h b/numpy/core/src/multiarray/array_coercion.h index 90ce0355a..c5ccad225 100644 --- a/numpy/core/src/multiarray/array_coercion.h +++ b/numpy/core/src/multiarray/array_coercion.h @@ -15,7 +15,6 @@ typedef struct coercion_cache_obj { int depth; /* the dimension at which this object was found. */ } coercion_cache_obj; - NPY_NO_EXPORT int _PyArray_MapPyTypeToDType( PyArray_DTypeMeta *DType, PyTypeObject *pytype, npy_bool userdef); diff --git a/numpy/core/src/multiarray/array_method.c b/numpy/core/src/multiarray/array_method.c index 2cc075141..e13da12de 100644 --- a/numpy/core/src/multiarray/array_method.c +++ b/numpy/core/src/multiarray/array_method.c @@ -34,6 +34,7 @@ #include "arrayobject.h" #include "array_method.h" #include "dtypemeta.h" +#include "common_dtype.h" #include "convert_datatype.h" diff --git a/numpy/core/src/multiarray/common_dtype.c b/numpy/core/src/multiarray/common_dtype.c new file mode 100644 index 000000000..a88085f6f --- /dev/null +++ b/numpy/core/src/multiarray/common_dtype.c @@ -0,0 +1,318 @@ +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE +#include <numpy/npy_common.h> +#include "numpy/arrayobject.h" + +#include "common_dtype.h" +#include "dtypemeta.h" +#include "abstractdtypes.h" + + +/* + * This file defines all logic necessary for generic "common dtype" + * operations. This is unfortunately surprisingly complicated to get right + * due to the value based logic NumPy uses and the fact that NumPy has + * no clear (non-transitive) type promotion hierarchy. + * Unlike most languages `int32 + float32 -> float64` instead of `float32`. + * The other complicated thing is value-based-promotion, which means that + * in many cases a Python 1, may end up as an `int8` or `uint8`. + * + * This file implements the necessary logic so that `np.result_type(...)` + * can give the correct result for any order of inputs and can further + * generalize to user DTypes. + */ + + +/** + * This function defines the common DType operator. + * + * Note that the common DType will not be "object" (unless one of the dtypes + * is object), even though object can technically represent all values + * correctly. + * + * TODO: Before exposure, we should review the return value (e.g. no error + * when no common DType is found). + * + * @param dtype1 DType class to find the common type for. + * @param dtype2 Second DType class. + * @return The common DType or NULL with an error set + */ +NPY_NO_EXPORT NPY_INLINE PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) +{ + if (dtype1 == dtype2) { + Py_INCREF(dtype1); + return dtype1; + } + + PyArray_DTypeMeta *common_dtype; + + common_dtype = dtype1->common_dtype(dtype1, dtype2); + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(common_dtype); + common_dtype = dtype2->common_dtype(dtype2, dtype1); + } + if (common_dtype == NULL) { + return NULL; + } + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + PyErr_Format(PyExc_TypeError, + "The DTypes %S and %S do not have a common DType. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`.", dtype1, dtype2); + return NULL; + } + return common_dtype; +} + + +/** + * This function takes a list of dtypes and "reduces" them (in a sense, + * it finds the maximal dtype). Note that "maximum" here is defined by + * knowledge (or category or domain). A user DType must always "know" + * about all NumPy dtypes, floats "know" about integers, integers "know" + * about unsigned integers. + * + * c + * / \ + * a \ <-- The actual promote(a, b) may be c or unknown. + * / \ \ + * a b c + * + * The reduction is done "pairwise". In the above `a.__common_dtype__(b)` + * has a result (so `a` knows more) and `a.__common_dtype__(c)` returns + * NotImplemented (so `c` knows more). You may notice that the result + * `res = a.__common_dtype__(b)` is not important. We could try to use it + * to remove the whole branch if `res is c` or by checking if + * `c.__common_dtype(res) is c`. + * Right now, we only clear initial elements in the most simple case where + * `a.__common_dtype(b) is a` (and thus `b` cannot alter the end-result). + * Clearing means, we do not have to worry about them later. + * + * There is one further subtlety. If we have an abstract DType and a + * non-abstract one, we "prioritize" the non-abstract DType here. + * In this sense "prioritizing" means that we use: + * abstract.__common_dtype__(other) + * If both return NotImplemented (which is acceptable and even expected in + * this case, see later) then `other` will be considered to know more. + * + * The reason why this may be acceptable for abstract DTypes, is that + * the value-dependent abstract DTypes may provide default fall-backs. + * The priority inversion effectively means that abstract DTypes are ordered + * just below their concrete counterparts. + * (This fall-back is convenient but not perfect, it can lead to + * non-minimal promotions: e.g. `np.uint24 + 2**20 -> int32`. And such + * cases may also be possible in some mixed type scenarios; they can be + * avoided by defining the promotion explicitly in the user DType.) + * + * @param length Number of DTypes + * @param dtypes + */ +static PyArray_DTypeMeta * +reduce_dtypes_to_most_knowledgeable( + npy_intp length, PyArray_DTypeMeta **dtypes) +{ + assert(length >= 2); + npy_intp half = length / 2; + + PyArray_DTypeMeta *res = NULL; + + for (npy_intp low = 0; low < half; low++) { + npy_intp high = length - 1 - low; + if (dtypes[high] == dtypes[low]) { + Py_INCREF(dtypes[low]); + Py_XSETREF(res, dtypes[low]); + } + else { + if (dtypes[high]->abstract) { + /* + * Priority inversion, start with abstract, because if it + * returns `other`, we can let other pass instead. + */ + PyArray_DTypeMeta *tmp = dtypes[low]; + dtypes[low] = dtypes[high]; + dtypes[high] = tmp; + } + + Py_XSETREF(res, dtypes[low]->common_dtype(dtypes[low], dtypes[high])); + if (res == NULL) { + return NULL; + } + } + + if (res == (PyArray_DTypeMeta *)Py_NotImplemented) { + PyArray_DTypeMeta *tmp = dtypes[low]; + dtypes[low] = dtypes[high]; + dtypes[high] = tmp; + } + if (res == dtypes[low]) { + /* `dtypes[high]` cannot influence the final result, so clear: */ + dtypes[high] = NULL; + } + } + + if (length == 2) { + return res; + } + Py_DECREF(res); + return reduce_dtypes_to_most_knowledgeable(length - half, dtypes); +} + + +/** + * Promotes a list of DTypes with each other in a way that should guarantee + * stable results even when changing the order. + * + * In general this approach always works as long as the most generic dtype + * is either strictly larger, or compatible with all other dtypes. + * For example promoting float16 with any other float, integer, or unsigned + * integer again gives a floating point number. And any floating point number + * promotes in the "same way" as `float16`. + * If a user inserts more than one type into the NumPy type hierarchy, this + * can break. Given: + * uint24 + int32 -> int48 # Promotes to a *new* dtype! + * + * The following becomes problematic (order does not matter): + * uint24 + int16 + uint32 -> int64 + * <== (uint24 + int16) + (uint24 + uint32) -> int64 + * <== int32 + uint32 -> int64 + * + * It is impossible to achieve an `int48` result in the above. + * + * This is probably only resolvable by asking `uint24` to take over the + * whole reduction step; which we currently do not do. + * (It may be possible to notice the last up-cast and implement use something + * like: `uint24.nextafter(int32).__common_dtype__(uint32)`, but that seems + * even harder to grasp.) + * + * Note that a case where two dtypes are mixed (and know nothing about each + * other) will always generate an error: + * uint24 + int48 + int64 -> Error + * + * Even though `int64` is a safe solution, since `uint24 + int64 -> int64` and + * `int48 + int64 -> int64` and `int64` and there cannot be a smaller solution. + * + * //TODO: Maybe this function should allow not setting an error? + * + * @param length Number of dtypes (and values) must be at least 1 + * @param dtypes The concrete or abstract DTypes to promote + * @return NULL or the promoted DType. + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_PromoteDTypeSequence( + npy_intp length, PyArray_DTypeMeta **dtypes_in) +{ + if (length == 1) { + Py_INCREF(dtypes_in[0]); + return dtypes_in[0]; + } + PyArray_DTypeMeta *result = NULL; + + /* Copy dtypes so that we can reorder them (only allocate when many) */ + PyObject *_scratch_stack[NPY_MAXARGS]; + PyObject **_scratch_heap = NULL; + PyArray_DTypeMeta **dtypes = (PyArray_DTypeMeta **)_scratch_stack; + + if (length > NPY_MAXARGS) { + _scratch_heap = PyMem_Malloc(length * sizeof(PyObject *)); + if (_scratch_heap == NULL) { + PyErr_NoMemory(); + return NULL; + } + dtypes = (PyArray_DTypeMeta **)_scratch_heap; + } + + memcpy(dtypes, dtypes_in, length * sizeof(PyObject *)); + + /* + * `result` is the last promotion result, which can usually be reused if + * it is not NotImplemneted. + * The passed in dtypes are partially sorted (and cleared, when clearly + * not relevant anymore). + * `dtypes[0]` will be the most knowledgeable (highest category) which + * we consider the "main_dtype" here. + */ + result = reduce_dtypes_to_most_knowledgeable(length, dtypes); + if (result == NULL) { + goto finish; + } + PyArray_DTypeMeta *main_dtype = dtypes[0]; + + npy_intp reduce_start = 1; + if (result == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_SETREF(result, NULL); + } + else { + /* (new) first value is already taken care of in `result` */ + reduce_start = 2; + } + /* + * At this point, we have only looked at every DType at most once. + * The `main_dtype` must know all others (or it will be a failure) and + * all dtypes returned by its `common_dtype` must be guaranteed to succeed + * promotion with one another. + * It is the job of the "main DType" to ensure that at this point order + * is irrelevant. + * If this turns out to be a limitation, this "reduction" will have to + * become a default version and we have to allow DTypes to override it. + */ + PyArray_DTypeMeta *prev = NULL; + for (npy_intp i = reduce_start; i < length; i++) { + if (dtypes[i] == NULL || dtypes[i] == prev) { + continue; + } + /* + * "Promote" the current dtype with the main one (which should be + * a higher category). We assume that the result is not in a lower + * category. + */ + PyArray_DTypeMeta *promotion = main_dtype->common_dtype( + main_dtype, dtypes[i]); + if (promotion == NULL) { + Py_XSETREF(result, NULL); + goto finish; + } + else if ((PyObject *)promotion == Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + Py_XSETREF(result, NULL); + PyObject *dtypes_in_tuple = PyTuple_New(length); + if (dtypes_in_tuple == NULL) { + goto finish; + } + for (npy_intp l=0; l < length; l++) { + Py_INCREF(dtypes_in[l]); + PyTuple_SET_ITEM(dtypes_in_tuple, l, (PyObject *)dtypes_in[l]); + } + PyErr_Format(PyExc_TypeError, + "The DType %S could not be promoted by %S. This means that " + "no common DType exists for the given inputs. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`. The full list of DTypes is: %S", + dtypes[i], main_dtype, dtypes_in_tuple); + Py_DECREF(dtypes_in_tuple); + goto finish; + } + if (result == NULL) { + result = promotion; + continue; + } + + /* + * The above promoted, now "reduce" with the current result; note that + * in the typical cases we expect this step to be a no-op. + */ + Py_SETREF(result, PyArray_CommonDType(result, promotion)); + Py_DECREF(promotion); + if (result == NULL) { + goto finish; + } + } + + finish: + PyMem_Free(_scratch_heap); + return result; +} diff --git a/numpy/core/src/multiarray/common_dtype.h b/numpy/core/src/multiarray/common_dtype.h new file mode 100644 index 000000000..b3666531a --- /dev/null +++ b/numpy/core/src/multiarray/common_dtype.h @@ -0,0 +1,17 @@ +#ifndef _NPY_COMMON_DTYPE_H_ +#define _NPY_COMMON_DTYPE_H_ + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include <numpy/ndarraytypes.h> +#include "dtypemeta.h" + +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); + +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_PromoteDTypeSequence( + npy_intp length, PyArray_DTypeMeta **dtypes_in); + +#endif /* _NPY_COMMON_DTYPE_H_ */ diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 8dca184cb..01ee56d16 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -17,10 +17,12 @@ #include "common.h" #include "ctors.h" #include "dtypemeta.h" +#include "common_dtype.h" #include "scalartypes.h" #include "mapping.h" #include "legacy_dtype_implementation.h" +#include "abstractdtypes.h" #include "convert_datatype.h" #include "_datetime.h" #include "datetime_strings.h" @@ -98,7 +100,7 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) else if (from->type_num < NPY_NTYPES && to->type_num < NPY_NTYPES) { /* All builtin dtypes have their casts explicitly defined. */ PyErr_Format(PyExc_RuntimeError, - "builtin cast from %S to %s not found, this should not " + "builtin cast from %S to %S not found, this should not " "be possible.", from, to); return NULL; } @@ -431,7 +433,24 @@ PyArray_GetCastSafety( NPY_NO_EXPORT int PyArray_CanCastSafely(int fromtype, int totype) { -#if NPY_USE_NEW_CASTINGIMPL + /* Identity */ + if (fromtype == totype) { + return 1; + } + /* + * As a micro-optimization, keep the cast table around. This can probably + * be removed as soon as the ufunc loop lookup is modified (presumably + * before the 1.21 release). It does no harm, but the main user of this + * function is the ufunc-loop lookup calling it until a loop matches! + * + * (The table extends further, but is not strictly correct for void). + * TODO: Check this! + */ + if ((unsigned int)fromtype <= NPY_CLONGDOUBLE && + (unsigned int)totype <= NPY_CLONGDOUBLE) { + return _npy_can_cast_safely_table[fromtype][totype]; + } + PyArray_DTypeMeta *from = PyArray_DTypeFromTypeNum(fromtype); if (from == NULL) { PyErr_WriteUnraisable(NULL); @@ -458,9 +477,6 @@ PyArray_CanCastSafely(int fromtype, int totype) int res = PyArray_MinCastSafety(safety, NPY_SAFE_CASTING) == NPY_SAFE_CASTING; Py_DECREF(castingimpl); return res; -#else - return PyArray_LegacyCanCastSafely(fromtype, totype); -#endif } @@ -474,11 +490,7 @@ PyArray_CanCastSafely(int fromtype, int totype) NPY_NO_EXPORT npy_bool PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to) { -#if NPY_USE_NEW_CASTINGIMPL return PyArray_CanCastTypeTo(from, to, NPY_SAFE_CASTING); -#else - return PyArray_LegacyCanCastTo(from, to); -#endif } @@ -553,7 +565,6 @@ NPY_NO_EXPORT npy_bool PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting) { -#if NPY_USE_NEW_CASTINGIMPL /* * NOTE: This code supports U and S, this is identical to the code * in `ctors.c` which does not allow these dtypes to be attached @@ -580,9 +591,6 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, } /* If casting is the smaller (or equal) safety we match */ return PyArray_MinCastSafety(safety, casting) == casting; -#else - return PyArray_LegacyCanCastTypeTo(from, to, casting); -#endif } @@ -590,18 +598,15 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, static int min_scalar_type_num(char *valueptr, int type_num, int *is_small_unsigned); + +/* + * NOTE: This function uses value based casting logic for scalars. It will + * require updates when we phase out value-based-casting. + */ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting) { - int swap; - int is_small_unsigned = 0, type_num; - npy_bool ret; - PyArray_Descr *dtype; - - /* An aligned memory buffer large enough to hold any type */ - npy_longlong value[4]; - /* * If the two dtypes are actually references to the same object * or if casting type is forced unsafe then always OK. @@ -610,16 +615,42 @@ can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, return 1; } + /* NOTE: This is roughly the same code as `PyArray_CanCastTypeTo`: */ + NPY_CASTING safety; + if (PyDataType_ISUNSIZED(to) && to->subarray == NULL) { + safety = PyArray_GetCastSafety(scal_type, NULL, NPY_DTYPE(to)); + } + else { + safety = PyArray_GetCastSafety(scal_type, to, NPY_DTYPE(to)); + } + if (safety < 0) { + PyErr_Clear(); + return 0; + } + safety = PyArray_MinCastSafety(safety, casting); + if (safety == casting) { + /* This is definitely a valid cast. */ + return 1; + } + /* - * If the scalar isn't a number, or the rule is stricter than - * NPY_SAFE_CASTING, use the straight type-based rules + * If the scalar isn't a number, value-based casting cannot kick in and + * we must not attempt it. + * (Additional fast-checks would be possible, but probably unnecessary.) */ - if (!PyTypeNum_ISNUMBER(scal_type->type_num) || - casting < NPY_SAFE_CASTING) { - return PyArray_CanCastTypeTo(scal_type, to, casting); + if (!PyTypeNum_ISNUMBER(scal_type->type_num)) { + return 0; } - swap = !PyArray_ISNBO(scal_type->byteorder); + /* + * At this point we have to check value-based casting. + */ + PyArray_Descr *dtype; + int is_small_unsigned = 0, type_num; + /* An aligned memory buffer large enough to hold any builtin numeric type */ + npy_longlong value[4]; + + int swap = !PyArray_ISNBO(scal_type->byteorder); scal_type->f->copyswap(&value, scal_data, swap, NULL); type_num = min_scalar_type_num((char *)&value, scal_type->type_num, @@ -645,7 +676,7 @@ can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyObject_Print(to, stdout, 0); printf("\n"); #endif - ret = PyArray_CanCastTypeTo(dtype, to, casting); + npy_bool ret = PyArray_CanCastTypeTo(dtype, to, casting); Py_DECREF(dtype); return ret; } @@ -842,7 +873,6 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) return descr; } -#if NPY_USE_NEW_CASTINGIMPL PyObject *tmp = PyArray_GetCastingImpl(NPY_DTYPE(descr), given_DType); if (tmp == NULL || tmp == Py_None) { Py_XDECREF(tmp); @@ -865,23 +895,10 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) error:; /* (; due to compiler limitations) */ PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; PyErr_Fetch(&err_type, &err_value, &err_traceback); - PyErr_Format(PyExc_ValueError, + PyErr_Format(PyExc_TypeError, "cannot cast dtype %S to %S.", descr, given_DType); - npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); + npy_PyErr_ChainExceptionsCause(err_type, err_value, err_traceback); return NULL; - -#else /* NPY_USE_NEW_CASTS */ - if (!given_DType->legacy) { - PyErr_SetString(PyExc_NotImplementedError, - "Must use casting to find the correct DType for a parametric " - "user DType. This is not yet implemented (this error should be " - "unreachable)."); - return NULL; - } - - PyArray_Descr *flex_dtype = PyArray_DescrNew(given_DType->singleton); - return PyArray_AdaptFlexibleDType(descr, flex_dtype); -#endif /* NPY_USE_NEW_CASTS */ } @@ -901,7 +918,7 @@ PyArray_FindConcatenationDescriptor( npy_intp n, PyArrayObject **arrays, PyObject *requested_dtype) { if (requested_dtype == NULL) { - return PyArray_ResultType(n, arrays, 0, NULL); + return PyArray_LegacyResultType(n, arrays, 0, NULL); } PyArray_DTypeMeta *common_dtype; @@ -921,16 +938,16 @@ PyArray_FindConcatenationDescriptor( goto finish; } assert(n > 0); /* concatenate requires at least one array input. */ + + /* + * NOTE: This code duplicates `PyArray_CastToDTypeAndPromoteDescriptors` + * to use arrays, copying the descriptors seems not better. + */ PyArray_Descr *descr = PyArray_DESCR(arrays[0]); result = PyArray_CastDescrToDType(descr, common_dtype); if (result == NULL || n == 1) { goto finish; } - /* - * This could short-cut a bit, calling `common_instance` directly and/or - * returning the `default_descr()` directly. Avoiding that (for now) as - * it would duplicate code from `PyArray_PromoteTypes`. - */ for (npy_intp i = 1; i < n; i++) { descr = PyArray_DESCR(arrays[i]); PyArray_Descr *curr = PyArray_CastDescrToDType(descr, common_dtype); @@ -938,7 +955,7 @@ PyArray_FindConcatenationDescriptor( Py_SETREF(result, NULL); goto finish; } - Py_SETREF(result, PyArray_PromoteTypes(result, curr)); + Py_SETREF(result, common_dtype->common_instance(result, curr)); Py_DECREF(curr); if (result == NULL) { goto finish; @@ -951,50 +968,6 @@ PyArray_FindConcatenationDescriptor( } -/** - * This function defines the common DType operator. - * - * Note that the common DType will not be "object" (unless one of the dtypes - * is object), even though object can technically represent all values - * correctly. - * - * TODO: Before exposure, we should review the return value (e.g. no error - * when no common DType is found). - * - * @param dtype1 DType class to find the common type for. - * @param dtype2 Second DType class. - * @return The common DType or NULL with an error set - */ -NPY_NO_EXPORT PyArray_DTypeMeta * -PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) -{ - if (dtype1 == dtype2) { - Py_INCREF(dtype1); - return dtype1; - } - - PyArray_DTypeMeta *common_dtype; - - common_dtype = dtype1->common_dtype(dtype1, dtype2); - if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { - Py_DECREF(common_dtype); - common_dtype = dtype2->common_dtype(dtype2, dtype1); - } - if (common_dtype == NULL) { - return NULL; - } - if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { - Py_DECREF(Py_NotImplemented); - PyErr_Format(PyExc_TypeError, - "The DTypes %S and %S do not have a common DType. " - "For example they cannot be stored in a single array unless " - "the dtype is `object`.", dtype1, dtype2); - return NULL; - } - return common_dtype; -} - - /*NUMPY_API * Produces the smallest size and lowest kind type to which both * input types can be cast. @@ -1017,6 +990,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) } if (!common_dtype->parametric) { + /* Note that this path loses all metadata */ res = common_dtype->default_descr(common_dtype); Py_DECREF(common_dtype); return res; @@ -1050,28 +1024,17 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) * Produces the smallest size and lowest kind type to which all * input types can be cast. * - * Equivalent to functools.reduce(PyArray_PromoteTypes, types) + * Roughly equivalent to functools.reduce(PyArray_PromoteTypes, types) + * but uses a more complex pairwise approach. */ NPY_NO_EXPORT PyArray_Descr * PyArray_PromoteTypeSequence(PyArray_Descr **types, npy_intp ntypes) { - npy_intp i; - PyArray_Descr *ret = NULL; if (ntypes == 0) { PyErr_SetString(PyExc_TypeError, "at least one type needed to promote"); return NULL; } - ret = types[0]; - Py_INCREF(ret); - for (i = 1; i < ntypes; ++i) { - PyArray_Descr *tmp = PyArray_PromoteTypes(types[i], ret); - Py_DECREF(ret); - ret = tmp; - if (ret == NULL) { - return NULL; - } - } - return ret; + return PyArray_ResultType(0, NULL, ntypes, types); } /* @@ -1437,6 +1400,8 @@ dtype_kind_to_simplified_ordering(char kind) * If the scalars are of a lower or same category as the arrays, they may be * demoted to a lower type within their category (the lowest type they can * be cast to safely according to scalar casting rules). + * + * If any new style dtype is involved (non-legacy), always returns 0. */ NPY_NO_EXPORT int should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, @@ -1453,6 +1418,9 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, /* Compute the maximum "kinds" and whether everything is scalar */ for (npy_intp i = 0; i < narrs; ++i) { + if (!NPY_DTYPE(PyArray_DESCR(arr[i]))->legacy) { + return 0; + } if (PyArray_NDIM(arr[i]) == 0) { int kind = dtype_kind_to_simplified_ordering( PyArray_DESCR(arr[i])->kind); @@ -1474,6 +1442,9 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, * finish computing the max array kind */ for (npy_intp i = 0; i < ndtypes; ++i) { + if (!NPY_DTYPE(dtypes[i])->legacy) { + return 0; + } int kind = dtype_kind_to_simplified_ordering(dtypes[i]->kind); if (kind > max_array_kind) { max_array_kind = kind; @@ -1490,6 +1461,206 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, /*NUMPY_API + * + * Produces the result type of a bunch of inputs, using the same rules + * as `np.result_type`. + * + * NOTE: This function is expected to through a transitional period or + * change behaviour. DTypes should always be strictly enforced for + * 0-D arrays, while "weak DTypes" will be used to represent Python + * integers, floats, and complex in all cases. + * (Within this function, these are currently flagged on the array + * object to work through `np.result_type`, this may change.) + * + * Until a time where this transition is complete, we probably cannot + * add new "weak DTypes" or allow users to create their own. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_ResultType( + npy_intp narrs, PyArrayObject *arrs[], + npy_intp ndtypes, PyArray_Descr *descrs[]) +{ + PyArray_Descr *result = NULL; + + if (narrs + ndtypes <= 1) { + /* If the input is a single value, skip promotion. */ + if (narrs == 1) { + result = PyArray_DTYPE(arrs[0]); + } + else if (ndtypes == 1) { + result = descrs[0]; + } + else { + PyErr_SetString(PyExc_TypeError, + "no arrays or types available to calculate result type"); + return NULL; + } + return ensure_dtype_nbo(result); + } + + void **info_on_heap = NULL; + void *_info_on_stack[NPY_MAXARGS * 2]; + PyArray_DTypeMeta **all_DTypes; + PyArray_Descr **all_descriptors; + + if (narrs + ndtypes > NPY_MAXARGS) { + info_on_heap = PyMem_Malloc(2 * (narrs+ndtypes) * sizeof(PyObject *)); + if (info_on_heap == NULL) { + PyErr_NoMemory(); + return NULL; + } + all_DTypes = (PyArray_DTypeMeta **)info_on_heap; + all_descriptors = (PyArray_Descr **)(info_on_heap + narrs + ndtypes); + } + else { + all_DTypes = (PyArray_DTypeMeta **)_info_on_stack; + all_descriptors = (PyArray_Descr **)(_info_on_stack + narrs + ndtypes); + } + + /* Copy all dtypes into a single array defining non-value-based behaviour */ + for (npy_intp i=0; i < ndtypes; i++) { + all_DTypes[i] = NPY_DTYPE(descrs[i]); + Py_INCREF(all_DTypes[i]); + all_descriptors[i] = descrs[i]; + } + + int at_least_one_scalar = 0; + int all_pyscalar = ndtypes == 0; + for (npy_intp i=0, i_all=ndtypes; i < narrs; i++, i_all++) { + /* Array descr is also the correct "default" for scalars: */ + if (PyArray_NDIM(arrs[i]) == 0) { + at_least_one_scalar = 1; + } + + if (!(PyArray_FLAGS(arrs[i]) & _NPY_ARRAY_WAS_PYSCALAR)) { + /* This was not a scalar with an abstract DType */ + all_descriptors[i_all] = PyArray_DTYPE(arrs[i]); + all_DTypes[i_all] = NPY_DTYPE(all_descriptors[i_all]); + Py_INCREF(all_DTypes[i_all]); + all_pyscalar = 0; + continue; + } + + /* + * The original was a Python scalar with an abstract DType. + * In a future world, this type of code may need to work on the + * DType level first and discover those from the original value. + * But, right now we limit the logic to int, float, and complex + * and do it here to allow for a transition without losing all of + * our remaining sanity. + */ + if (PyArray_ISFLOAT(arrs[i])) { + all_DTypes[i_all] = &PyArray_PyFloatAbstractDType; + } + else if (PyArray_ISCOMPLEX(arrs[i])) { + all_DTypes[i_all] = &PyArray_PyComplexAbstractDType; + } + else { + /* N.B.: Could even be an object dtype here for large ints */ + all_DTypes[i_all] = &PyArray_PyIntAbstractDType; + } + Py_INCREF(all_DTypes[i_all]); + /* + * Leave the decriptor empty, if we need it, we will have to go + * to more extreme lengths unfortunately. + */ + all_descriptors[i_all] = NULL; + } + + PyArray_DTypeMeta *common_dtype = PyArray_PromoteDTypeSequence( + narrs+ndtypes, all_DTypes); + for (npy_intp i=0; i < narrs+ndtypes; i++) { + Py_DECREF(all_DTypes[i]); + } + if (common_dtype == NULL) { + goto finish; + } + + if (common_dtype->abstract) { + /* (ab)use default descriptor to define a default */ + PyArray_Descr *tmp_descr = common_dtype->default_descr(common_dtype); + if (tmp_descr == NULL) { + goto finish; + } + Py_INCREF(NPY_DTYPE(tmp_descr)); + Py_SETREF(common_dtype, NPY_DTYPE(tmp_descr)); + Py_DECREF(tmp_descr); + } + + /* + * NOTE: Code duplicates `PyArray_CastToDTypeAndPromoteDescriptors`, but + * supports special handling of the abstract values. + */ + if (!common_dtype->parametric) { + /* Note that this "fast" path loses all metadata */ + result = common_dtype->default_descr(common_dtype); + } + else { + result = PyArray_CastDescrToDType(all_descriptors[0], common_dtype); + + for (npy_intp i = 1; i < ndtypes+narrs; i++) { + PyArray_Descr *curr; + if (NPY_LIKELY(i < ndtypes || + !(PyArray_FLAGS(arrs[i-ndtypes]) & _NPY_ARRAY_WAS_PYSCALAR))) { + curr = PyArray_CastDescrToDType(all_descriptors[i], common_dtype); + } + else { + /* + * Unlike `PyArray_CastToDTypeAndPromoteDescriptors` deal with + * plain Python values "graciously". This recovers the original + * value the long route, but it should almost never happen... + */ + PyObject *tmp = PyArray_GETITEM( + arrs[i-ndtypes], PyArray_BYTES(arrs[i-ndtypes])); + if (tmp == NULL) { + Py_SETREF(result, NULL); + goto finish; + } + curr = common_dtype->discover_descr_from_pyobject(common_dtype, tmp); + Py_DECREF(tmp); + } + if (curr == NULL) { + Py_SETREF(result, NULL); + goto finish; + } + Py_SETREF(result, common_dtype->common_instance(result, curr)); + Py_DECREF(curr); + if (result == NULL) { + goto finish; + } + } + } + + /* + * Unfortunately, when 0-D "scalar" arrays are involved and mixed, we + * have to use the value-based logic. The intention is to move away from + * the complex logic arising from it. We thus fall back to the legacy + * version here. + * It may be possible to micro-optimize this to skip some of the above + * logic when this path is necessary. + */ + if (at_least_one_scalar && !all_pyscalar && result->type_num < NPY_NTYPES) { + PyArray_Descr *legacy_result = PyArray_LegacyResultType( + narrs, arrs, ndtypes, descrs); + if (legacy_result == NULL) { + /* + * Going from error to success should not really happen, but is + * probably OK if it does. + */ + Py_SETREF(result, NULL); + goto finish; + } + /* Return the old "legacy" result (could warn here if different) */ + Py_SETREF(result, legacy_result); + } + + finish: + PyMem_Free(info_on_heap); + return result; +} + + +/* * Produces the result type of a bunch of inputs, using the UFunc * type promotion rules. Use this function when you have a set of * input arrays, and need to determine an output array dtype. @@ -1501,11 +1672,11 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, * Otherwise, does a type promotion on the MinScalarType * of all the inputs. Data types passed directly are treated as array * types. - * */ NPY_NO_EXPORT PyArray_Descr * -PyArray_ResultType(npy_intp narrs, PyArrayObject **arr, - npy_intp ndtypes, PyArray_Descr **dtypes) +PyArray_LegacyResultType( + npy_intp narrs, PyArrayObject **arr, + npy_intp ndtypes, PyArray_Descr **dtypes) { npy_intp i; @@ -1604,6 +1775,49 @@ PyArray_ResultType(npy_intp narrs, PyArrayObject **arr, } } +/** + * Promotion of descriptors (of arbitrary DType) to their correctly + * promoted instances of the given DType. + * I.e. the given DType could be a string, which then finds the correct + * string length, given all `descrs`. + * + * @param ndescrs number of descriptors to cast and find the common instance. + * At least one must be passed in. + * @param descrs The descriptors to work with. + * @param DType The DType of the desired output descriptor. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_CastToDTypeAndPromoteDescriptors( + npy_intp ndescr, PyArray_Descr *descrs[], PyArray_DTypeMeta *DType) +{ + assert(ndescr > 0); + + PyArray_Descr *result = PyArray_CastDescrToDType(descrs[0], DType); + if (result == NULL || ndescr == 1) { + return result; + } + if (!DType->parametric) { + /* Note that this "fast" path loses all metadata */ + Py_DECREF(result); + return DType->default_descr(DType); + } + + for (npy_intp i = 1; i < ndescr; i++) { + PyArray_Descr *curr = PyArray_CastDescrToDType(descrs[i], DType); + if (curr == NULL) { + Py_DECREF(result); + return NULL; + } + Py_SETREF(result, DType->common_instance(result, curr)); + Py_DECREF(curr); + if (result == NULL) { + return NULL; + } + } + return result; +} + + /*NUMPY_API * Is the typenum valid? */ diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index 33517b8ca..ba16d4d1b 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -20,8 +20,10 @@ PyArray_ObjectType(PyObject *op, int minimum_type); NPY_NO_EXPORT PyArrayObject ** PyArray_ConvertToCommonType(PyObject *op, int *retn); -NPY_NO_EXPORT PyArray_DTypeMeta * -PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); +NPY_NO_EXPORT PyArray_Descr * +PyArray_LegacyResultType( + npy_intp narrs, PyArrayObject **arr, + npy_intp ndtypes, PyArray_Descr **dtypes); NPY_NO_EXPORT int PyArray_ValidType(int type); @@ -29,7 +31,7 @@ PyArray_ValidType(int type); NPY_NO_EXPORT int dtype_kind_to_ordering(char kind); -/* Like PyArray_CanCastArrayTo */ +/* Used by PyArray_CanCastArrayTo and in the legacy ufunc type resolution */ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting); diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 26b16b15b..40ca9ee2a 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -32,6 +32,14 @@ dtypemeta_dealloc(PyArray_DTypeMeta *self) { } static PyObject * +dtypemeta_alloc(PyTypeObject *NPY_UNUSED(type), Py_ssize_t NPY_UNUSED(items)) +{ + PyErr_SetString(PyExc_TypeError, + "DTypes can only be created using the NumPy API."); + return NULL; +} + +static PyObject * dtypemeta_new(PyTypeObject *NPY_UNUSED(type), PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) { @@ -690,6 +698,7 @@ NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = { .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", .tp_members = dtypemeta_members, .tp_base = NULL, /* set to PyType_Type at import time */ + .tp_alloc = dtypemeta_alloc, .tp_init = (initproc)dtypemeta_init, .tp_new = dtypemeta_new, .tp_is_gc = dtypemeta_is_gc, diff --git a/numpy/core/src/multiarray/legacy_dtype_implementation.c b/numpy/core/src/multiarray/legacy_dtype_implementation.c index d2e95348d..9b4946da3 100644 --- a/numpy/core/src/multiarray/legacy_dtype_implementation.c +++ b/numpy/core/src/multiarray/legacy_dtype_implementation.c @@ -1,10 +1,10 @@ /* - * This file hosts legacy implementations of certain functions for - * which alternatives exists, but the old functions are still required - * in certain code paths, or until the code transition is finalized. + * The only function exported here is `PyArray_LegacyCanCastTypeTo`, which + * is currently still in use when first registering a userdtype. * - * This code should typically not require modification, and if modified - * similar changes may be necessary in the new version. + * The extremely limited use means that it can probably remain unmaintained + * until such a time where legay user dtypes are deprecated and removed + * entirely. */ #define NPY_NO_DEPRECATED_API NPY_API_VERSION @@ -78,7 +78,7 @@ _equivalent_subarrays(PyArray_ArrayDescr *sub1, PyArray_ArrayDescr *sub2) } -NPY_NO_EXPORT unsigned char +static unsigned char PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) { int type_num1, type_num2, size1, size2; @@ -116,7 +116,7 @@ PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } -NPY_NO_EXPORT unsigned char +static unsigned char PyArray_LegacyEquivTypenums(int typenum1, int typenum2) { PyArray_Descr *d1, *d2; @@ -135,7 +135,7 @@ PyArray_LegacyEquivTypenums(int typenum1, int typenum2) } -NPY_NO_EXPORT int +static int PyArray_LegacyCanCastSafely(int fromtype, int totype) { PyArray_Descr *from; @@ -171,7 +171,7 @@ PyArray_LegacyCanCastSafely(int fromtype, int totype) } -NPY_NO_EXPORT npy_bool +static npy_bool PyArray_LegacyCanCastTo(PyArray_Descr *from, PyArray_Descr *to) { int from_type_num = from->type_num; @@ -551,168 +551,3 @@ PyArray_LegacyCanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, } } - -/* - * Legacy function to find the correct dtype when casting from any built-in - * dtype to NPY_STRING, NPY_UNICODE, NPY_VOID, and NPY_DATETIME with generic - * units. - * - * This function returns a dtype based on flex_dtype and the values in - * data_dtype. It also calls Py_DECREF on the flex_dtype. If the - * flex_dtype is not flexible, it returns it as-is. - * - * Usually, if data_obj is not an array, dtype should be the result - * given by the PyArray_GetArrayParamsFromObject function. - * - * If *flex_dtype is NULL, returns immediately, without setting an - * exception, leaving any previous error handling intact. - */ -NPY_NO_EXPORT PyArray_Descr * -PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype) -{ - PyArray_DatetimeMetaData *meta; - PyArray_Descr *retval = NULL; - int flex_type_num; - - if (flex_dtype == NULL) { - return retval; - } - - flex_type_num = flex_dtype->type_num; - - /* Flexible types with expandable size */ - if (PyDataType_ISUNSIZED(flex_dtype)) { - /* First replace the flex_dtype */ - retval = PyArray_DescrNew(flex_dtype); - Py_DECREF(flex_dtype); - if (retval == NULL) { - return retval; - } - - if (data_dtype->type_num == flex_type_num || - flex_type_num == NPY_VOID) { - (retval)->elsize = data_dtype->elsize; - } - else if (flex_type_num == NPY_STRING || flex_type_num == NPY_UNICODE) { - npy_intp size = 8; - - /* - * Get a string-size estimate of the input. These - * are generallly the size needed, rounded up to - * a multiple of eight. - */ - switch (data_dtype->type_num) { - case NPY_BOOL: - case NPY_UBYTE: - case NPY_BYTE: - case NPY_USHORT: - case NPY_SHORT: - case NPY_UINT: - case NPY_INT: - case NPY_ULONG: - case NPY_LONG: - case NPY_ULONGLONG: - case NPY_LONGLONG: - if (data_dtype->kind == 'b') { - /* 5 chars needed for cast to 'True' or 'False' */ - size = 5; - } - else if (data_dtype->elsize > 8 || - data_dtype->elsize < 0) { - /* - * Element size should never be greater than 8 or - * less than 0 for integer type, but just in case... - */ - break; - } - else if (data_dtype->kind == 'u') { - size = REQUIRED_STR_LEN[data_dtype->elsize]; - } - else if (data_dtype->kind == 'i') { - /* Add character for sign symbol */ - size = REQUIRED_STR_LEN[data_dtype->elsize] + 1; - } - break; - case NPY_HALF: - case NPY_FLOAT: - case NPY_DOUBLE: - size = 32; - break; - case NPY_LONGDOUBLE: - size = 48; - break; - case NPY_CFLOAT: - case NPY_CDOUBLE: - size = 2 * 32; - break; - case NPY_CLONGDOUBLE: - size = 2 * 48; - break; - case NPY_OBJECT: - size = 64; - break; - case NPY_STRING: - case NPY_VOID: - size = data_dtype->elsize; - break; - case NPY_UNICODE: - size = data_dtype->elsize / 4; - break; - case NPY_DATETIME: - meta = get_datetime_metadata_from_dtype(data_dtype); - if (meta == NULL) { - Py_DECREF(retval); - return NULL; - } - size = get_datetime_iso_8601_strlen(0, meta->base); - break; - case NPY_TIMEDELTA: - size = 21; - break; - } - - if (flex_type_num == NPY_STRING) { - retval->elsize = size; - } - else if (flex_type_num == NPY_UNICODE) { - retval->elsize = size * 4; - } - } - else { - /* - * We should never get here, but just in case someone adds - * a new flex dtype... - */ - PyErr_SetString(PyExc_TypeError, - "don't know how to adapt flex dtype"); - Py_DECREF(retval); - return NULL; - } - } - /* Flexible type with generic time unit that adapts */ - else if (flex_type_num == NPY_DATETIME || - flex_type_num == NPY_TIMEDELTA) { - meta = get_datetime_metadata_from_dtype(flex_dtype); - retval = flex_dtype; - if (meta == NULL) { - return NULL; - } - - if (meta->base == NPY_FR_GENERIC) { - if (data_dtype->type_num == NPY_DATETIME || - data_dtype->type_num == NPY_TIMEDELTA) { - meta = get_datetime_metadata_from_dtype(data_dtype); - if (meta == NULL) { - return NULL; - } - - retval = create_datetime_dtype(flex_type_num, meta); - Py_DECREF(flex_dtype); - } - } - } - else { - retval = flex_dtype; - } - return retval; -} diff --git a/numpy/core/src/multiarray/legacy_dtype_implementation.h b/numpy/core/src/multiarray/legacy_dtype_implementation.h index ca171d773..b36eb019a 100644 --- a/numpy/core/src/multiarray/legacy_dtype_implementation.h +++ b/numpy/core/src/multiarray/legacy_dtype_implementation.h @@ -1,40 +1,8 @@ #ifndef _NPY_LEGACY_DTYPE_IMPLEMENTATION_H #define _NPY_LEGACY_DTYPE_IMPLEMENTATION_H - -NPY_NO_EXPORT unsigned char -PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2); - -NPY_NO_EXPORT unsigned char -PyArray_LegacyEquivTypenums(int typenum1, int typenum2); - -NPY_NO_EXPORT int -PyArray_LegacyCanCastSafely(int fromtype, int totype); - -NPY_NO_EXPORT npy_bool -PyArray_LegacyCanCastTo(PyArray_Descr *from, PyArray_Descr *to); - NPY_NO_EXPORT npy_bool PyArray_LegacyCanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting); -/* - * This function calls Py_DECREF on flex_dtype, and replaces it with - * a new dtype that has been adapted based on the values in data_dtype - * and data_obj. If the flex_dtype is not flexible, it returns it as-is. - * - * Usually, if data_obj is not an array, dtype should be the result - * given by the PyArray_GetArrayParamsFromObject function. - * - * The data_obj may be NULL if just a dtype is known for the source. - * - * If *flex_dtype is NULL, returns immediately, without setting an - * exception, leaving any previous error handling intact. - * - * The current flexible dtypes include NPY_STRING, NPY_UNICODE, NPY_VOID, - * and NPY_DATETIME with generic units. - */ -NPY_NO_EXPORT PyArray_Descr * -PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype); - #endif /*_NPY_LEGACY_DTYPE_IMPLEMENTATION_H*/ diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index ff5a5d8bc..251e527a6 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -867,44 +867,44 @@ array_astype(PyArrayObject *self, Py_INCREF(self); return (PyObject *)self; } - else if (PyArray_CanCastArrayTo(self, dtype, casting)) { - PyArrayObject *ret; - - /* This steals the reference to dtype, so no DECREF of dtype */ - ret = (PyArrayObject *)PyArray_NewLikeArray( - self, order, dtype, subok); - if (ret == NULL) { - return NULL; - } - /* NumPy 1.20, 2020-10-01 */ - if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && - DEPRECATE_FUTUREWARNING( - "casting an array to a subarray dtype " - "will not using broadcasting in the future, but cast each " - "element to the new dtype and then append the dtype's shape " - "to the new array. You can opt-in to the new behaviour, by " - "additional field to the cast: " - "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" - "This may lead to a different result or to current failures " - "succeeding. " - "(FutureWarning since NumPy 1.20)") < 0) { - Py_DECREF(ret); - return NULL; - } - - if (PyArray_CopyInto(ret, self) < 0) { - Py_DECREF(ret); - return NULL; - } - - return (PyObject *)ret; - } - else { + if (!PyArray_CanCastArrayTo(self, dtype, casting)) { + PyErr_Clear(); npy_set_invalid_cast_error( PyArray_DESCR(self), dtype, casting, PyArray_NDIM(self) == 0); Py_DECREF(dtype); return NULL; } + + PyArrayObject *ret; + + /* This steals the reference to dtype, so no DECREF of dtype */ + ret = (PyArrayObject *)PyArray_NewLikeArray( + self, order, dtype, subok); + if (ret == NULL) { + return NULL; + } + /* NumPy 1.20, 2020-10-01 */ + if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && + DEPRECATE_FUTUREWARNING( + "casting an array to a subarray dtype " + "will not use broadcasting in the future, but cast each " + "element to the new dtype and then append the dtype's shape " + "to the new array. You can opt-in to the new behaviour, by " + "additional field to the cast: " + "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" + "This may lead to a different result or to current failures " + "succeeding. " + "(FutureWarning since NumPy 1.20)") < 0) { + Py_DECREF(ret); + return NULL; + } + + if (PyArray_CopyInto(ret, self) < 0) { + Py_DECREF(ret); + return NULL; + } + + return (PyObject *)ret; } /* default sub-type implementation */ diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index f3c1b7f98..f7c3ea093 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1470,7 +1470,9 @@ array_putmask(PyObject *NPY_UNUSED(module), PyObject *args, PyObject *kwds) NPY_NO_EXPORT unsigned char PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) { -#if NPY_USE_NEW_CASTINGIMPL + if (type1 == type2) { + return 1; + } /* * Do not use PyArray_CanCastTypeTo because it supports legacy flexible * dtypes as input. @@ -1482,9 +1484,6 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } /* If casting is "no casting" this dtypes are considered equivalent. */ return PyArray_MinCastSafety(safety, NPY_NO_CASTING) == NPY_NO_CASTING; -#else - return PyArray_LegacyEquivTypes(type1, type2); -#endif } @@ -3357,7 +3356,7 @@ array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *from_obj = NULL; PyArray_Descr *d1 = NULL; PyArray_Descr *d2 = NULL; - npy_bool ret; + int ret; PyObject *retobj = NULL; NPY_CASTING casting = NPY_SAFE_CASTING; static char *kwlist[] = {"from_", "to", "casting", NULL}; @@ -3487,6 +3486,10 @@ array_result_type(PyObject *NPY_UNUSED(dummy), PyObject *args) if (arr[narr] == NULL) { goto finish; } + if (PyLong_CheckExact(obj) || PyFloat_CheckExact(obj) || + PyComplex_CheckExact(obj)) { + ((PyArrayObject_fields *)arr[narr])->flags |= _NPY_ARRAY_WAS_PYSCALAR; + } ++narr; } else { diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 7e9b93782..a62776748 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -16,6 +16,9 @@ #include "binop_override.h" #include "ufunc_override.h" +#include "abstractdtypes.h" +#include "common_dtype.h" +#include "convert_datatype.h" /************************************************************************* **************** Implement Number Protocol **************************** diff --git a/numpy/core/tests/test_casting_unittests.py b/numpy/core/tests/test_casting_unittests.py index c8fcd4b42..2cec1acd3 100644 --- a/numpy/core/tests/test_casting_unittests.py +++ b/numpy/core/tests/test_casting_unittests.py @@ -17,7 +17,6 @@ from numpy.lib.stride_tricks import as_strided from numpy.testing import assert_array_equal from numpy.core._multiarray_umath import _get_castingimpl as get_castingimpl -from numpy.core._multiarray_tests import uses_new_casts # Simple skips object, parametric and long double (unsupported by struct) @@ -135,11 +134,7 @@ class TestChanges: def test_float_to_string(self, floating, string): assert np.can_cast(floating, string) # 100 is long enough to hold any formatted floating - if uses_new_casts(): - assert np.can_cast(floating, f"{string}100") - else: - assert not np.can_cast(floating, f"{string}100") - assert np.can_cast(floating, f"{string}100", casting="same_kind") + assert np.can_cast(floating, f"{string}100") def test_to_void(self): # But in general, we do consider these safe: @@ -147,17 +142,11 @@ class TestChanges: assert np.can_cast("S20", "V") # Do not consider it a safe cast if the void is too smaller: - if uses_new_casts(): - assert not np.can_cast("d", "V1") - assert not np.can_cast("S20", "V1") - assert not np.can_cast("U1", "V1") - # Structured to unstructured is just like any other: - assert np.can_cast("d,i", "V", casting="same_kind") - else: - assert np.can_cast("d", "V1") - assert np.can_cast("S20", "V1") - assert np.can_cast("U1", "V1") - assert not np.can_cast("d,i", "V", casting="same_kind") + assert not np.can_cast("d", "V1") + assert not np.can_cast("S20", "V1") + assert not np.can_cast("U1", "V1") + # Structured to unstructured is just like any other: + assert np.can_cast("d,i", "V", casting="same_kind") class TestCasting: diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 53e4821ae..8a6b7dcd5 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1088,6 +1088,103 @@ class TestPickling: assert roundtrip_DType is DType +class TestPromotion: + """Test cases related to more complex DType promotions. Further promotion + tests are defined in `test_numeric.py` + """ + @pytest.mark.parametrize(["other", "expected"], + [(2**16-1, np.complex64), + (2**32-1, np.complex128), + (np.float16(2), np.complex64), + (np.float32(2), np.complex64), + (np.longdouble(2), np.complex64), + # Base of the double value to sidestep any rounding issues: + (np.longdouble(np.nextafter(1.7e308, 0.)), np.complex128), + # Additionally use "nextafter" so the cast can't round down: + (np.longdouble(np.nextafter(1.7e308, np.inf)), np.clongdouble), + # repeat for complex scalars: + (np.complex64(2), np.complex64), + (np.clongdouble(2), np.complex64), + # Base of the double value to sidestep any rounding issues: + (np.clongdouble(np.nextafter(1.7e308, 0.) * 1j), np.complex128), + # Additionally use "nextafter" so the cast can't round down: + (np.clongdouble(np.nextafter(1.7e308, np.inf)), np.clongdouble), + ]) + def test_complex_other_value_based(self, other, expected): + # This would change if we modify the value based promotion + min_complex = np.dtype(np.complex64) + + res = np.result_type(other, min_complex) + assert res == expected + # Check the same for a simple ufunc call that uses the same logic: + res = np.minimum(other, np.ones(3, dtype=min_complex)).dtype + assert res == expected + + @pytest.mark.parametrize(["other", "expected"], + [(np.bool_, np.complex128), + (np.int64, np.complex128), + (np.float16, np.complex64), + (np.float32, np.complex64), + (np.float64, np.complex128), + (np.longdouble, np.clongdouble), + (np.complex64, np.complex64), + (np.complex128, np.complex128), + (np.clongdouble, np.clongdouble), + ]) + def test_complex_scalar_value_based(self, other, expected): + # This would change if we modify the value based promotion + complex_scalar = 1j + + res = np.result_type(other, complex_scalar) + assert res == expected + # Check the same for a simple ufunc call that uses the same logic: + res = np.minimum(np.ones(3, dtype=other), complex_scalar).dtype + assert res == expected + + def test_complex_pyscalar_promote_rational(self): + with pytest.raises(TypeError, + match=r".* do not have a common DType"): + np.result_type(1j, rational) + + with pytest.raises(TypeError, + match=r".* no common DType exists for the given inputs"): + np.result_type(1j, rational(1, 2)) + + @pytest.mark.parametrize(["other", "expected"], + [(1, rational), (1., np.float64)]) + def test_float_int_pyscalar_promote_rational(self, other, expected): + # Note that rationals are a bit akward as they promote with float64 + # or default ints, but not float16 or uint8/int8 (which looks + # inconsistent here) + with pytest.raises(TypeError, + match=r".* do not have a common DType"): + np.result_type(other, rational) + + assert np.result_type(other, rational(1, 2)) == expected + + @pytest.mark.parametrize(["dtypes", "expected"], [ + # These promotions are not associative/commutative: + ([np.uint16, np.int16, np.float16], np.float32), + ([np.uint16, np.int8, np.float16], np.float32), + ([np.uint8, np.int16, np.float16], np.float32), + # The following promotions are not ambiguous, but cover code + # paths of abstract promotion (no particular logic being tested) + ([1, 1, np.float64], np.float64), + ([1, 1., np.complex128], np.complex128), + ([1, 1j, np.float64], np.complex128), + ([1., 1., np.int64], np.float64), + ([1., 1j, np.float64], np.complex128), + ([1j, 1j, np.float64], np.complex128), + ([1, True, np.bool_], np.int_), + ]) + def test_permutations_do_not_influence_result(self, dtypes, expected): + # Tests that most permutations do not influence the result. In the + # above some uint and int combintations promote to a larger integer + # type, which would then promote to a larger than necessary float. + for perm in permutations(dtypes): + assert np.result_type(*perm) == expected + + def test_rational_dtype(): # test for bug gh-5719 a = np.array([1111], dtype=rational).astype diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index a0c72f9d0..679e3c036 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -382,13 +382,9 @@ class TestConcatenate: @pytest.mark.parametrize("axis", [None, 0]) def test_string_dtype_does_not_inspect(self, axis): - # The error here currently depends on NPY_USE_NEW_CASTINGIMPL as - # the new version rejects using the "default string length" of 64. - # The new behaviour is better, `np.array()` and `arr.astype()` would - # have to be used instead. (currently only raises due to unsafe cast) - with pytest.raises((ValueError, TypeError)): + with pytest.raises(TypeError): np.concatenate(([None], [1]), dtype="S", axis=axis) - with pytest.raises((ValueError, TypeError)): + with pytest.raises(TypeError): np.concatenate(([None], [1]), dtype="U", axis=axis) @pytest.mark.parametrize("axis", [None, 0]) |