diff options
| -rw-r--r-- | numpy/core/src/multiarray/iterators.c | 4 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/mapping.c | 150 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/na_singleton.c | 34 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/na_singleton.h | 8 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/shape.c | 3 | ||||
| -rw-r--r-- | numpy/core/tests/test_maskna.py | 85 |
6 files changed, 255 insertions, 29 deletions
diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index f59cdb7d1..45fc40366 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -105,6 +105,8 @@ parse_index(PyArrayObject *self, PyObject *op, PyObject *op1 = NULL; int is_slice; + printf("parsing index...\n"); + if (PySlice_Check(op) || op == Py_Ellipsis || op == Py_None) { n = 1; op1 = op; @@ -173,6 +175,7 @@ parse_index(PyArrayObject *self, PyObject *op, } } else { + printf("index %d %d %d\n", (int)start, (int)n_steps, (int)step_size); if (nd_old >= PyArray_NDIM(self)) { PyErr_SetString(PyExc_IndexError, "too many indices"); return -1; @@ -209,6 +212,7 @@ parse_index(PyArrayObject *self, PyObject *op, } *out_offset = offset; if (out_maskna_offset != NULL) { + printf ("maskna offset %d\n", (int)maskna_offset); *out_maskna_offset = maskna_offset; } return nd_new; diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 0477b633e..4608c2604 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -89,8 +89,8 @@ array_big_item(PyArrayObject *self, npy_intp i) fa->maskna_data = PyArray_MASKNA_DATA(self) + i * PyArray_MASKNA_STRIDES(self)[0]; if (fa->nd > 0) { - memcpy(fa->maskna_strides, &(PyArray_MASKNA_STRIDES(self)[1]), - (fa->nd - 1) * sizeof(npy_intp)); + memcpy(fa->maskna_strides, PyArray_MASKNA_STRIDES(self)+1, + fa->nd * sizeof(npy_intp)); } fa->flags |= NPY_ARRAY_MASKNA; } @@ -1392,12 +1392,7 @@ array_ass_sub_simple(PyArrayObject *self, PyObject *index, PyObject *op) tmp = (PyArrayObject *)tmp0; } - if (PyArray_ISOBJECT(self) && (PyArray_NDIM(tmp) == 0)) { - ret = PyArray_DESCR(tmp)->f->setitem(op, PyArray_DATA(tmp), tmp); - } - else { - ret = PyArray_CopyObject(tmp, op); - } + ret = PyArray_CopyObject(tmp, op); Py_DECREF(tmp); return ret; } @@ -1521,7 +1516,7 @@ array_ass_sub(PyArrayObject *self, PyObject *index, PyObject *op) (PyArray_DIMS((PyArrayObject *)index)==0) && PyArray_ISBOOL((PyArrayObject *)index))) { if (PyObject_IsTrue(index)) { - return PyArray_DESCR(self)->f->setitem(op, PyArray_DATA(self), self); + return PyArray_CopyObject(self, op); } else { /* don't do anything */ return 0; @@ -1532,25 +1527,66 @@ array_ass_sub(PyArrayObject *self, PyObject *index, PyObject *op) } /* Integer-tuple */ - if (PyTuple_Check(index) && (PyTuple_GET_SIZE(index) == PyArray_NDIM(self)) - && (_tuple_of_integers(index, vals, PyArray_NDIM(self)) >= 0)) { - int i; - char *item; - - for (i = 0; i < PyArray_NDIM(self); i++) { - if (vals[i] < 0) { - vals[i] += PyArray_DIMS(self)[i]; + if (PyTuple_Check(index) && + (PyTuple_GET_SIZE(index) == PyArray_NDIM(self)) && + (_tuple_of_integers(index, vals, PyArray_NDIM(self)) >= 0)) { + int idim, ndim = PyArray_NDIM(self); + npy_intp *shape = PyArray_DIMS(self); + npy_intp *strides = PyArray_STRIDES(self); + char *item = PyArray_DATA(self); + + if (!PyArray_HASMASKNA(self)) { + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (v < 0) { + v += shape[idim]; + } + if (v < 0 || v >= shape[idim]) { + PyErr_Format(PyExc_IndexError, + "index (%"INTP_FMT") out of range "\ + "(0<=index<%"INTP_FMT") in dimension %d", + vals[idim], PyArray_DIMS(self)[idim], idim); + return -1; + } + else { + item += v * strides[idim]; + } } - if ((vals[i] < 0) || (vals[i] >= PyArray_DIMS(self)[i])) { - PyErr_Format(PyExc_IndexError, - "index (%"INTP_FMT") out of range "\ - "(0<=index<%"INTP_FMT") in dimension %d", - vals[i], PyArray_DIMS(self)[i], i); - return -1; + return PyArray_DESCR(self)->f->setitem(op, item, self); + } + else { + char *maskna_item = PyArray_MASKNA_DATA(self); + npy_intp *maskna_strides = PyArray_MASKNA_STRIDES(self); + NpyNA *na; + + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (v < 0) { + v += shape[idim]; + } + if (v < 0 || v >= shape[idim]) { + PyErr_Format(PyExc_IndexError, + "index (%"INTP_FMT") out of range "\ + "(0<=index<%"INTP_FMT") in dimension %d", + vals[idim], PyArray_DIMS(self)[idim], idim); + return -1; + } + else { + item += v * strides[idim]; + maskna_item += v * maskna_strides[idim]; + } + } + na = NpyNA_FromObject(op, 1); + if (na == NULL) { + *maskna_item = 1; + return PyArray_DESCR(self)->f->setitem(op, item, self); + } + else { + *maskna_item = NpyNA_AsMaskValue(na); + Py_DECREF(na); + return 0; } } - item = PyArray_GetPtr(self, vals); - return PyArray_DESCR(self)->f->setitem(op, item, self); } PyErr_Clear(); @@ -1634,9 +1670,66 @@ array_subscript_nice(PyArrayObject *self, PyObject *op) } } /* optimization for a tuple of integers */ - if (PyArray_NDIM(self) > 1 && PyTuple_Check(op) && - (PyTuple_GET_SIZE(op) == PyArray_NDIM(self)) - && (_tuple_of_integers(op, vals, PyArray_NDIM(self)) >= 0)) { + if (PyArray_NDIM(self) > 1 && + PyTuple_Check(op) && + (PyTuple_GET_SIZE(op) == PyArray_NDIM(self)) && + (_tuple_of_integers(op, vals, PyArray_NDIM(self)) >= 0)) { + int idim, ndim = PyArray_NDIM(self); + npy_intp *shape = PyArray_DIMS(self); + npy_intp *strides = PyArray_STRIDES(self); + char *item = PyArray_DATA(self); + + if (!PyArray_HASMASKNA(self)) { + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (v < 0) { + v += shape[idim]; + } + if (v < 0 || v >= shape[idim]) { + PyErr_Format(PyExc_IndexError, + "index (%"INTP_FMT") out of range "\ + "(0<=index<%"INTP_FMT") in dimension %d", + vals[idim], PyArray_DIMS(self)[idim], idim); + return NULL; + } + else { + item += v * strides[idim]; + } + } + return PyArray_Scalar(item, PyArray_DESCR(self), (PyObject *)self); + } + else { + char *maskna_item = PyArray_MASKNA_DATA(self); + npy_intp *maskna_strides = PyArray_MASKNA_STRIDES(self); + + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (v < 0) { + v += shape[idim]; + } + if (v < 0 || v >= shape[idim]) { + PyErr_Format(PyExc_IndexError, + "index (%"INTP_FMT") out of range "\ + "(0<=index<%"INTP_FMT") in dimension %d", + vals[idim], PyArray_DIMS(self)[idim], idim); + return NULL; + } + else { + item += v * strides[idim]; + maskna_item += v * maskna_strides[idim]; + } + } + if (NpyMaskValue_IsExposed((npy_mask)*maskna_item)) { + return PyArray_Scalar(item, PyArray_DESCR(self), + (PyObject *)self); + } + else { + return (PyObject *)NpyNA_FromDTypeAndMaskValue( + PyArray_DESCR(self), (npy_mask)*maskna_item); + } + } + +#if 0 int i; char *item; @@ -1654,6 +1747,7 @@ array_subscript_nice(PyArrayObject *self, PyObject *op) } item = PyArray_GetPtr(self, vals); return PyArray_Scalar(item, PyArray_DESCR(self), (PyObject *)self); +#endif } PyErr_Clear(); diff --git a/numpy/core/src/multiarray/na_singleton.c b/numpy/core/src/multiarray/na_singleton.c index f3cb1ac16..dd59b5c69 100644 --- a/numpy/core/src/multiarray/na_singleton.c +++ b/numpy/core/src/multiarray/na_singleton.c @@ -434,6 +434,40 @@ NpyNA_FromObject(PyObject *obj, int suppress_error) } /* + * Converts a dtype reference and mask value into an NA. + * Doesn't steal the 'dtype' reference. Raises an error + * if 'maskvalue' represents an exposed mask. + */ +NPY_NO_EXPORT NpyNA * +NpyNA_FromDTypeAndMaskValue(PyArray_Descr *dtype, npy_mask maskvalue) +{ + NpyNA_fields *fna; + + if (dtype == NULL && maskvalue == 0) { + Py_INCREF(Npy_NA); + return (NpyNA *)Npy_NA; + } + + if (NpyMaskValue_IsExposed(maskvalue)) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert exposed mask value into NA"); + return NULL; + } + + fna = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); + if (fna == NULL) { + return NULL; + } + + fna->dtype = dtype; + Py_XINCREF(fna->dtype); + + fna->payload = NpyMaskValue_GetPayload(maskvalue); + + return (NpyNA *)fna; +} + +/* * Returns a mask value corresponding to the NA. */ NPY_NO_EXPORT npy_mask diff --git a/numpy/core/src/multiarray/na_singleton.h b/numpy/core/src/multiarray/na_singleton.h index b911c2364..069c457b6 100644 --- a/numpy/core/src/multiarray/na_singleton.h +++ b/numpy/core/src/multiarray/na_singleton.h @@ -49,6 +49,14 @@ NPY_NO_EXPORT NpyNA * NpyNA_FromObject(PyObject *obj, int suppress_error); /* + * Converts a dtype reference and mask value into an NA. + * Doesn't steal the 'dtype' reference. Raises an error + * if 'maskvalue' represents an exposed mask. + */ +NPY_NO_EXPORT NpyNA * +NpyNA_FromDTypeAndMaskValue(PyArray_Descr *dtype, npy_mask maskvalue); + +/* * Returns a mask value corresponding to the NA. */ NPY_NO_EXPORT npy_mask diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 56c08304c..f9c079208 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -346,11 +346,12 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, printf("building fortran strides\n"); fflush(stdout); for (i = 0; i < ndim; ++i) { fa->maskna_strides[i] = stride; - printf("stride %d\n", stride); + printf("stride %d\n", (int)stride); stride *= fa->dimensions[i]; } } else { + printf("building C strides\n"); fflush(stdout); for (i = ndim; i >= 0; --i) { fa->maskna_strides[i] = stride; stride *= fa->dimensions[i]; diff --git a/numpy/core/tests/test_maskna.py b/numpy/core/tests/test_maskna.py index dbd435c8c..d1d79c86a 100644 --- a/numpy/core/tests/test_maskna.py +++ b/numpy/core/tests/test_maskna.py @@ -120,6 +120,47 @@ def test_array_maskna_isna_1D(): # TODO: fancy indexing is next... +def test_array_maskna_isna_2D(): + a = np.zeros((3,4)) + + # With no mask, it returns all False + assert_equal(np.isna(a), False) + assert_equal(np.isna(a).shape, (3,4)) + + # With a mask but no NAs, it still returns all False + a.flags.maskna = True + assert_equal(np.isna(a), False) + assert_equal(np.isna(a).shape, (3,4)) + + # Checking isna of a single value + assert_equal(np.isna(a[1,2]), False) + # Assigning NA to a single value + a[1,2] = np.NA + assert_equal(np.isna(a), [[0,0,0,0],[0,0,1,0],[0,0,0,0]]) + # Checking isna of a single value + assert_equal(np.isna(a[1,2]), True) + + # Checking isna of a slice + assert_equal(np.isna(a[1:4,1:3]), [[0,1],[0,0]]) + # Assigning NA to a slice + a[1:3,0:2] = np.NA + assert_equal(np.isna(a), [[0,0,0,0],[1,1,1,0],[1,1,0,0]]) + + # Checking isna of a strided slice + assert_equal(np.isna(a[1:,1:5:2]), [[1,0],[1,0]]) + # Assigning NA to a strided slice + a[::2,::2] = np.NA + assert_equal(np.isna(a), [[1,0,1,0],[1,1,1,0],[1,1,1,0]]) + + # Checking isna of a boolean mask index + mask = np.array([[1,1,0,0],[0,1,0,1],[0,0,1,0]], dtype='?') + assert_equal(np.isna(a[mask]), [1,0,1,0,1]) + # Assigning NA to a boolean masked index + a[mask] = np.NA + assert_equal(np.isna(a), [[1,1,1,0],[1,1,1,1],[1,1,1,0]]) + + # TODO: fancy indexing is next... + def test_array_maskna_view_function(): a = np.arange(10) @@ -282,6 +323,50 @@ def test_array_maskna_view_NA_assignment_1D(): # TODO: fancy indexing is next... +def test_array_maskna_view_NA_assignment_2D(): + a = np.arange(6).reshape(2,3) + a_ref = a.copy() + + # Make sure that assigning NA doesn't affect the original data + b = a.view(maskna=True) + b[...] = np.NA + assert_equal(np.isna(b), True) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + b[:] = np.NA + assert_equal(np.isna(b), True) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + b[0,:] = np.NA + assert_equal(np.isna(b[0]), True) + assert_equal(np.isna(b[1]), False) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + b[1:,1:3] = np.NA + assert_equal(np.isna(b), [[0,0,0],[0,1,1]]) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + b[1,::2] = np.NA + assert_equal(np.isna(b), [[0,0,0],[1,0,1]]) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + b[0,2] = np.NA + assert_equal(np.isna(b), [[0,0,1],[0,0,0]]) + assert_equal(a, a_ref) + + b = a.view(maskna=True) + mask = np.array([[1,0,1],[1,1,0]], dtype='?') + b[mask] = np.NA + assert_equal(np.isna(b), mask) + assert_equal(a, a_ref) + + # TODO: fancy indexing is next... + def test_array_maskna_view_array_assignment_1D(): a = np.arange(5) b = a.view(maskna=True) |