diff options
| -rw-r--r-- | numpy/core/include/numpy/ndarraytypes.h | 2 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/common.h | 8 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/na_singleton.c | 295 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/na_singleton.h | 25 | ||||
| -rw-r--r-- | numpy/core/tests/test_na.py | 153 |
5 files changed, 467 insertions, 16 deletions
diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 370c899ec..e99b3fb55 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -927,6 +927,8 @@ typedef struct tagNpyNA { PyObject_HEAD } NpyNA; +#define NpyNA_Check(op) PyObject_TypeCheck(op, &NpyNA_Type) + /********************************** * The nditer object, added in 1.6 **********************************/ diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 8242a0d18..fc39ddeeb 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -6,6 +6,10 @@ NPY_NO_EXPORT PyArray_Descr * _array_find_type(PyObject *op, PyArray_Descr *minitype, int max); +/* + * Returns NULL without setting an exception if no scalar is matched, a + * new dtype reference otherwise. + */ NPY_NO_EXPORT PyArray_Descr * _array_find_python_scalar_type(PyObject *op); @@ -13,7 +17,7 @@ NPY_NO_EXPORT PyArray_Descr * _array_typedescr_fromstr(char *str); NPY_NO_EXPORT char * -index2ptr(PyArrayObject *mp, intp i); +index2ptr(PyArrayObject *mp, npy_intp i); NPY_NO_EXPORT int _zerofill(PyArrayObject *ret); @@ -21,7 +25,7 @@ _zerofill(PyArrayObject *ret); NPY_NO_EXPORT int _IsAligned(PyArrayObject *ap); -NPY_NO_EXPORT Bool +NPY_NO_EXPORT npy_bool _IsWriteable(PyArrayObject *ap); #ifndef Py_UNICODE_WIDE diff --git a/numpy/core/src/multiarray/na_singleton.c b/numpy/core/src/multiarray/na_singleton.c index 7f866ec19..b0c7ec3fe 100644 --- a/numpy/core/src/multiarray/na_singleton.c +++ b/numpy/core/src/multiarray/na_singleton.c @@ -13,11 +13,13 @@ #define NPY_NO_DEPRECATED_API #define _MULTIARRAYMODULE #include <numpy/arrayobject.h> +#include <numpy/arrayscalars.h> #include "npy_config.h" #include "numpy/npy_3kcompat.h" #include "descriptor.h" +#include "common.h" #include "na_singleton.h" static PyObject * @@ -147,8 +149,16 @@ na_str(NpyNA_fieldaccess *self) static PyObject * na_richcompare(NpyNA_fieldaccess *self, PyObject *other, int cmp_op) { - Py_INCREF(Npy_NA); - return Npy_NA; + /* If an ndarray is compared directly with NA, let the array handle it */ + if (PyArray_Check(other)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + /* Otherwise always return the NA singleton */ + else { + Py_INCREF(Npy_NA); + return Npy_NA; + } } static PyObject * @@ -259,6 +269,222 @@ static PyGetSetDef na_getsets[] = { {NULL, NULL, NULL, NULL, NULL} }; +/* Combines two NA values together, merging their payloads and dtypes. */ +NPY_NO_EXPORT NpyNA * +NpyNA_CombineNA(NpyNA *na1, NpyNA *na2) +{ + NpyNA_fieldaccess *ret, *fna1, *fna2; + + fna1 = (NpyNA_fieldaccess *)na1; + fna2 = (NpyNA_fieldaccess *)na2; + + ret = (NpyNA_fieldaccess *)na_new(&NpyNA_Type, NULL, NULL); + if (ret == NULL) { + return NULL; + } + + /* Combine the payloads */ + ret->payload = NpyNA_CombinePayloads(fna1->payload, fna2->payload); + + /* Combine the dtypes */ + Py_XDECREF(ret->dtype); + ret->dtype = NULL; + if (fna1->dtype != NULL && fna2->dtype != NULL) { + ret->dtype = PyArray_PromoteTypes(fna1->dtype, fna2->dtype); + if (ret->dtype == NULL) { + Py_DECREF(ret); + return NULL; + } + } + else if (fna1->dtype != NULL) { + ret->dtype = fna1->dtype; + Py_INCREF(ret->dtype); + } + else if (fna2->dtype != NULL) { + ret->dtype = fna2->dtype; + Py_INCREF(ret->dtype); + } + + return (NpyNA *)ret; +} + +/* + * Combines an NA with an object, raising an error if the object has + * no extractable NumPy dtype. + */ +NPY_NO_EXPORT NpyNA * +NpyNA_CombineNAWithObject(NpyNA *na, PyObject *obj) +{ + NpyNA_fieldaccess *ret, *fna; + PyArray_Descr *dtype = NULL; + + fna = (NpyNA_fieldaccess *)na; + + /* If 'obj' is NA, handle it specially */ + if (NpyNA_Check(obj)) { + return NpyNA_CombineNA(na, (NpyNA *)obj); + } + + /* Extract a dtype from 'obj' */ + if (PyArray_IsScalar(obj, Generic)) { + dtype = PyArray_DescrFromScalar(obj); + if (dtype == NULL) { + return NULL; + } + } + else if (PyArray_Check(obj)) { + /* TODO: This needs to be more complicated... */ + dtype = PyArray_DESCR((PyArrayObject *)obj); + Py_INCREF(dtype); + } + else { + dtype = _array_find_python_scalar_type(obj); + if (dtype == NULL) { + PyErr_SetString(PyExc_TypeError, + "numpy.NA only supports operations with scalars " + "and NumPy arrays"); + return NULL; + } + } + + ret = (NpyNA_fieldaccess *)na_new(&NpyNA_Type, NULL, NULL); + if (ret == NULL) { + return NULL; + } + + /* Copy the payload */ + ret->payload = fna->payload; + + /* Combine the dtypes */ + Py_XDECREF(ret->dtype); + if (fna->dtype == NULL) { + ret->dtype = dtype; + } + else { + ret->dtype = PyArray_PromoteTypes(fna->dtype, dtype); + Py_DECREF(dtype); + if (ret->dtype == NULL) { + Py_DECREF(ret); + return NULL; + } + } + + return (NpyNA *)ret; +} + +/* An NA unary op simply passes along the same NA */ +static PyObject * +na_unaryop(PyObject *self) +{ + Py_INCREF(self); + return self; +} + +static PyObject * +na_binaryop(PyObject *op1, PyObject *op2) +{ + /* If an ndarray is operated on with NA, let the array handle it */ + if (PyArray_Check(op1) || PyArray_Check(op2)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + /* Combine NAs according to standard rules */ + else { + if (NpyNA_Check(op1)) { + return (PyObject *)NpyNA_CombineNAWithObject((NpyNA *)op1, op2); + } + else if (NpyNA_Check(op2)) { + return (PyObject *)NpyNA_CombineNAWithObject((NpyNA *)op2, op1); + } + else { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + } +} + +static PyObject * +na_power(PyObject *op1, PyObject *op2, PyObject *NPY_UNUSED(op3)) +{ + return na_binaryop(op1, op2); +} + +/* Special case bitwise <and> with a boolean 'other' */ +static PyObject * +na_and(PyObject *op1, PyObject *op2) +{ + NpyNA *na; + PyObject *other; + + if (NpyNA_Check(op1)) { + na = (NpyNA *)op1; + other = op2; + } + else if (NpyNA_Check(op2)) { + na = (NpyNA *)op2; + other = op1; + } + else { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + /* If an ndarray is operated on with NA, let the array handle it */ + if (PyArray_Check(other)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + /* NA & False is False */ + else if (other == Py_False || + ((Py_TYPE(other) == &PyBoolArrType_Type) && + ((PyBoolScalarObject *)other)->obval == 0)) { + Py_INCREF(Py_False); + return Py_False; + } + /* Combine NAs according to standard rules */ + else { + return (PyObject *)NpyNA_CombineNAWithObject(na, other); + } +} + +/* Special case bitwise <or> with a boolean 'other' */ +static PyObject * +na_or(PyObject *op1, PyObject *op2) +{ + NpyNA *na; + PyObject *other; + + if (NpyNA_Check(op1)) { + na = (NpyNA *)op1; + other = op2; + } + else if (NpyNA_Check(op2)) { + na = (NpyNA *)op2; + other = op1; + } + else { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + /* If an ndarray is operated on with NA, let the array handle it */ + if (PyArray_Check(other)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + /* NA & True is True */ + else if (other == Py_True || + ((Py_TYPE(other) == &PyBoolArrType_Type) && + ((PyBoolScalarObject *)other)->obval != 0)) { + Py_INCREF(Py_True); + return Py_True; + } + /* Combine NAs according to standard rules */ + else { + return (PyObject *)NpyNA_CombineNAWithObject(na, other); + } +} + /* Using NA in an if statement is always an error */ static int na_nonzero(PyObject *NPY_UNUSED(self)) @@ -270,20 +496,63 @@ na_nonzero(PyObject *NPY_UNUSED(self)) } NPY_NO_EXPORT PyNumberMethods na_as_number = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ + (binaryfunc)na_binaryop, /*nb_add*/ + (binaryfunc)na_binaryop, /*nb_subtract*/ + (binaryfunc)na_binaryop, /*nb_multiply*/ #if defined(NPY_PY3K) #else - 0, /*nb_divide*/ + (binaryfunc)na_binaryop, /*nb_divide*/ #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_neg*/ - 0, /*nb_pos*/ - 0, /*(unaryfunc)array_abs,*/ + (binaryfunc)na_binaryop, /*nb_remainder*/ + (binaryfunc)na_binaryop, /*nb_divmod*/ + (ternaryfunc)na_power, /*nb_power*/ + (unaryfunc)na_unaryop, /*nb_neg*/ + (unaryfunc)na_unaryop, /*nb_pos*/ + (unaryfunc)na_unaryop, /*nb_abs,*/ (inquiry)na_nonzero, /*nb_nonzero*/ + (unaryfunc)na_unaryop, /*nb_invert*/ + (binaryfunc)na_binaryop, /*nb_lshift*/ + (binaryfunc)na_binaryop, /*nb_rshift*/ + (binaryfunc)na_and, /*nb_and*/ + (binaryfunc)na_binaryop, /*nb_xor*/ + (binaryfunc)na_or, /*nb_or*/ +#if defined(NPY_PY3K) +#else + 0, /*nb_coerce*/ +#endif + 0, /*nb_int*/ +#if defined(NPY_PY3K) + 0, /*nb_reserved*/ +#else + 0, /*nb_long*/ +#endif + 0, /*nb_float*/ +#if defined(NPY_PY3K) +#else + 0, /*nb_oct*/ + 0, /*nb_hex*/ +#endif + 0, /*inplace_add*/ + 0, /*inplace_subtract*/ + 0, /*inplace_multiply*/ +#if defined(NPY_PY3K) +#else + 0, /*inplace_divide*/ +#endif + 0, /*inplace_remainder*/ + 0, /*inplace_power*/ + 0, /*inplace_lshift*/ + 0, /*inplace_rshift*/ + 0, /*inplace_and*/ + 0, /*inplace_xor*/ + 0, /*inplace_or*/ + (binaryfunc)na_binaryop, /*nb_floor_divide*/ + (binaryfunc)na_binaryop, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ +#if PY_VERSION_HEX >= 0x02050000 + 0, /*nb_index*/ +#endif }; NPY_NO_EXPORT PyTypeObject NpyNA_Type = { @@ -316,7 +585,7 @@ NPY_NO_EXPORT PyTypeObject NpyNA_Type = { 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES, /* tp_flags */ 0, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ diff --git a/numpy/core/src/multiarray/na_singleton.h b/numpy/core/src/multiarray/na_singleton.h index a0a36430a..dbf918e0e 100644 --- a/numpy/core/src/multiarray/na_singleton.h +++ b/numpy/core/src/multiarray/na_singleton.h @@ -13,7 +13,30 @@ typedef struct { } NpyNA_fieldaccess; NPY_NO_EXPORT NpyNA_fieldaccess _Npy_NASingleton; - #define Npy_NA ((PyObject *)&_Npy_NASingleton) +#define NPY_NA_NOPAYLOAD (255) + +static NPY_INLINE npy_uint8 +NpyNA_CombinePayloads(npy_uint p1, npy_uint p2) +{ + if (p1 == NPY_NA_NOPAYLOAD || p2 == NPY_NA_NOPAYLOAD) { + return NPY_NA_NOPAYLOAD; + } + else { + return (p1 < p2) ? p1 : p2; + } +} + +/* Combines two NA values together, merging their payloads and dtypes. */ +NPY_NO_EXPORT NpyNA * +NpyNA_CombineNA(NpyNA *na1, NpyNA *na2); + +/* + * Combines an NA with an object, raising an error if the object has + * no extractable NumPy dtype. + */ +NPY_NO_EXPORT NpyNA * +NpyNA_CombineNAWithObject(NpyNA *na, PyObject *obj); + #endif diff --git a/numpy/core/tests/test_na.py b/numpy/core/tests/test_na.py new file mode 100644 index 000000000..676377ca1 --- /dev/null +++ b/numpy/core/tests/test_na.py @@ -0,0 +1,153 @@ +import numpy as np +from numpy.compat import asbytes +from numpy.testing import * +import sys, warnings + +def test_na_construction(): + # construct a new NA object + v = np.NA() + assert_(not v is np.NA) + assert_equal(v.payload, None) + assert_equal(v.dtype, None) + + # Construct with a payload + v = np.NA(3) + assert_equal(v.payload, 3) + assert_equal(v.dtype, None) + + # Construct with a dtype + v = np.NA(dtype='f4') + assert_equal(v.payload, None) + assert_equal(v.dtype, np.dtype('f4')) + + # Construct with both a payload and a dtype + v = np.NA(5, dtype='f4,i2') + assert_equal(v.payload, 5) + assert_equal(v.dtype, np.dtype('f4,i2')) + + # min and max payload values + v = np.NA(0) + assert_equal(v.payload, 0) + v = np.NA(127) + assert_equal(v.payload, 127) + + # Out of bounds payload values + assert_raises(ValueError, np.NA, -1) + assert_raises(ValueError, np.NA, 128) + +def test_na_str(): + # With no payload or dtype + assert_equal(str(np.NA), 'NA') + assert_equal(str(np.NA()), 'NA') + + # With a payload + assert_equal(str(np.NA(10)), 'NA(10)') + + # With just a dtype + assert_equal(str(np.NA(dtype='c16')), 'NA') + + # With a payload and a dtype + assert_equal(str(np.NA(10, dtype='f4')), 'NA(10)') + +def test_na_repr(): + # With no payload or dtype + assert_equal(repr(np.NA), 'NA') + assert_equal(repr(np.NA()), 'NA') + + # With a payload + assert_equal(repr(np.NA(10)), 'NA(10)') + + # With just a dtype + assert_equal(repr(np.NA(dtype='>c16')), "NA(dtype='>c16')") + + # With a payload and a dtype + assert_equal(repr(np.NA(10, dtype='>f4')), "NA(10, dtype='>f4')") + +def test_na_comparison(): + # NA cannot be converted to a boolean + assert_raises(ValueError, bool, np.NA) + + # Comparison with different objects produces the singleton NA + assert_((np.NA < 3) is np.NA) + assert_((np.NA <= 3) is np.NA) + assert_((np.NA == 3) is np.NA) + assert_((np.NA != 3) is np.NA) + assert_((np.NA >= 3) is np.NA) + assert_((np.NA > 3) is np.NA) + + # Should work with NA on the other side too + assert_((3 < np.NA) is np.NA) + assert_((3 <= np.NA) is np.NA) + assert_((3 == np.NA) is np.NA) + assert_((3 != np.NA) is np.NA) + assert_((3 >= np.NA) is np.NA) + assert_((3 > np.NA) is np.NA) + +def test_na_operations(): + # The minimum of the payload is taken + assert_equal((np.NA + np.NA(3)).payload, None) + assert_equal((np.NA(12) + np.NA()).payload, None) + assert_equal((np.NA(2) - np.NA(6)).payload, 2) + assert_equal((np.NA(5) - np.NA(1)).payload, 1) + + # The dtypes are promoted like np.promote_types + assert_equal((np.NA(dtype='f4') * np.NA(dtype='f8')).dtype, + np.dtype('f8')) + assert_equal((np.NA(dtype='c8') * np.NA(dtype='f8')).dtype, + np.dtype('c16')) + assert_equal((np.NA * np.NA(dtype='i8')).dtype, + np.dtype('i8')) + assert_equal((np.NA(dtype='i2') / np.NA).dtype, + np.dtype('i2')) + +def test_na_other_operations(): + # Make sure we get NAs for all these operations + assert_equal(type(np.NA + 3), np.NAType) + assert_equal(type(3 + np.NA), np.NAType) + assert_equal(type(np.NA - 3.0), np.NAType) + assert_equal(type(3.0 - np.NA), np.NAType) + assert_equal(type(np.NA * 2j), np.NAType) + assert_equal(type(2j * np.NA), np.NAType) + assert_equal(type(np.NA / 2j), np.NAType) + assert_equal(type(2j / np.NA), np.NAType) + assert_equal(type(np.NA // 2j), np.NAType) + assert_equal(type(2j // np.NA), np.NAType) + assert_equal(type(np.NA % 6), np.NAType) + assert_equal(type(6 % np.NA), np.NAType) + assert_equal(type(np.NA ** 2), np.NAType) + assert_equal(type(2 ** np.NA), np.NAType) + assert_equal(type(np.NA & 2), np.NAType) + assert_equal(type(2 & np.NA), np.NAType) + assert_equal(type(np.NA | 2), np.NAType) + assert_equal(type(2 | np.NA), np.NAType) + assert_equal(type(np.NA << 2), np.NAType) + assert_equal(type(2 << np.NA), np.NAType) + assert_equal(type(np.NA >> 2), np.NAType) + assert_equal(type(2 >> np.NA), np.NAType) + assert_(abs(np.NA) is np.NA) + assert_((-np.NA) is np.NA) + assert_((+np.NA) is np.NA) + assert_((~np.NA) is np.NA) + + # The NA should get the dtype from the other operand + assert_equal((np.NA + 3).dtype, np.array(3).dtype) + assert_equal((np.NA - 3.0).dtype, np.array(3.0).dtype) + assert_equal((np.NA * 2j).dtype, np.array(2j).dtype) + + # Should have type promotion if the NA already has a dtype + assert_equal((np.NA(dtype='f4') ** 3.0).dtype, np.dtype('f8')) + + # Bitwise and/or are specialized slightly + # NOTE: The keywords 'and' and 'or' coerce to boolean, so we cannot + # properly support them. + assert_equal(np.NA & False, False) + assert_equal(False & np.NA, False) + assert_equal(np.NA | True, True) + assert_equal(True | np.NA, True) + assert_equal(type(np.NA | False), np.NAType) + assert_equal(type(np.NA & True), np.NAType) + assert_equal((np.NA | False).dtype, np.array(False).dtype) + assert_equal((np.NA & True).dtype, np.array(True).dtype) + +if __name__ == "__main__": + run_module_suite() |