diff options
| -rw-r--r-- | numpy/core/code_generators/numpy_api.py | 3 | ||||
| -rw-r--r-- | numpy/core/include/numpy/ndarraytypes.h | 72 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/ctors.c | 180 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/dtype_transfer.c | 124 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/flagsobject.c | 110 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/multiarraymodule.c | 68 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/na_mask.c | 135 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/nditer_api.c | 1 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/shape.c | 11 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/shape.h | 3 | ||||
| -rw-r--r-- | numpy/core/src/private/lowlevel_strided_loops.h | 16 |
11 files changed, 605 insertions, 118 deletions
diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index c8a7bdb01..ec8388311 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -69,7 +69,7 @@ multiarray_types_api = { 'PyHalfArrType_Type': 217, 'NpyIter_Type': 218, # End 1.6 API - 'NpyNA_Type': 286, + 'NpyNA_Type': 287, } #define NPY_NUMUSERTYPES (*(int *)PyArray_API[6]) @@ -323,6 +323,7 @@ multiarray_funcs_api = { 'PyArray_SetBaseObject': 283, 'PyArray_HasNASupport': 284, 'PyArray_ContainsNA': 285, + 'PyArray_AllocateMaskNA': 286, } ufunc_types_api = { diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index e99b3fb55..1b3d0b2a1 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -662,7 +662,7 @@ typedef struct tagPyArrayObject_fieldaccess { * If no mask: NULL * If mask : bool/uint8/structured dtype of mask dtypes */ - PyArray_Descr *maskna_descr; + PyArray_Descr *maskna_dtype; /* * Raw data buffer for mask. If the array has the flag * NPY_ARRAY_OWNMASKNA enabled, it owns this memory and @@ -672,7 +672,8 @@ typedef struct tagPyArrayObject_fieldaccess { /* * Just like dimensions and strides point into the same memory * buffer, we now just make that buffer 3x the nd instead of 2x - * and use the same buffer. + * and use the same buffer. This is always allocated, regardless + * of whether there is an NA mask or not. */ npy_intp *maskna_strides; } PyArrayObject_fieldaccess; @@ -746,6 +747,9 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); /* * Means c-style contiguous (last index varies the fastest). The data * elements right after each other. + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). */ #define NPY_ARRAY_C_CONTIGUOUS 0x0001 @@ -753,6 +757,9 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); * Set if array is a contiguous Fortran array: the first index varies * the fastest in memory (strides array is reverse of C-contiguous * array) + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). */ #define NPY_ARRAY_F_CONTIGUOUS 0x0002 @@ -764,12 +771,16 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); /* * If set, the array owns the data: it will be free'd when the array * is deleted. + * + * This flag may be tested for in PyArray_FLAGS(arr). */ #define NPY_ARRAY_OWNDATA 0x0004 /* * An array never has the next four set; they're only used as parameter * flags to the the various FromAny functions + * + * This flag may be requested in constructor functions. */ /* Cause a cast to occur regardless of whether or not it is safe. */ @@ -778,15 +789,23 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); /* * Always copy the array. Returned arrays are always CONTIGUOUS, * ALIGNED, and WRITEABLE. + * + * This flag may be requested in constructor functions. */ #define NPY_ARRAY_ENSURECOPY 0x0020 -/* Make sure the returned array is a base-class ndarray */ +/* + * Make sure the returned array is a base-class ndarray + * + * This flag may be requested in constructor functions. + */ #define NPY_ARRAY_ENSUREARRAY 0x0040 /* * Make sure that the strides are in units of the element size Needed * for some operations with record-arrays. + * + * This flag may be requested in constructor functions. */ #define NPY_ARRAY_ELEMENTSTRIDES 0x0080 @@ -795,27 +814,64 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); * stored according to how the compiler would align things (e.g., an * array of integers (4 bytes each) starts on a memory address that's * a multiple of 4) + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). */ #define NPY_ARRAY_ALIGNED 0x0100 -/* Array data has the native endianness */ +/* + * Array data has the native endianness + * + * This flag may be requested in constructor functions. + */ #define NPY_ARRAY_NOTSWAPPED 0x0200 -/* Array data is writeable */ +/* + * Array data is writeable + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). + */ #define NPY_ARRAY_WRITEABLE 0x0400 /* * If this flag is set, then base contains a pointer to an array of * the same size that should be updated with the current contents of * this array when this array is deallocated + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). */ #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* + * If this flag is set, then the array has an NA mask corresponding + * to the array data. If the flag NPY_ARRAY_OWNMASKNA is requested + * in a constructor, this flag is also implied even if it is not set. + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). + */ +#define NPY_ARRAY_MASKNA 0x2000 + +/* * If this flag is set, then the array owns the memory for the * missing values NA mask. + * + * This flag may be requested in constructor functions. + * This flag may be tested for in PyArray_FLAGS(arr). + */ +#define NPY_ARRAY_OWNMASKNA 0x4000 + +/* + * If this flag is set, then arrays which have an NA mask, or arrays + * which have an NA dtype are permitted to pass through. If not, + * a array with NA support causes an error to be thrown. + * + * This flag may be requested in constructor functions. */ -#define NPY_ARRAY_OWNMASKNA 0x2000 +#define NPY_ARRAY_ALLOWNA 0x8000 #define NPY_ARRAY_BEHAVED (NPY_ARRAY_ALIGNED | \ @@ -1511,9 +1567,9 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags) /* Access to the missing values NA mask, added in 1.7 */ static NPY_INLINE PyArray_Descr * -PyArray_MASKNA_DESCR(PyArrayObject *arr) +PyArray_MASKNA_DTYPE(PyArrayObject *arr) { - return ((PyArrayObject_fieldaccess *)arr)->maskna_descr; + return ((PyArrayObject_fieldaccess *)arr)->maskna_dtype; } static NPY_INLINE npy_mask * diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index b527a1074..7d2c0c488 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -770,7 +770,7 @@ discover_itemsize(PyObject *s, int nd, int *itemsize) static int discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, int stop_at_string, int stop_at_tuple, - int *out_is_object) + int *out_is_object, int *out_contains_na) { PyObject *e; int r, n, i; @@ -802,6 +802,11 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } + if (NpyNA_Check(obj)) { + *out_contains_na = 1; + return 0; + } + /* obj is not a Sequence */ if (!PySequence_Check(obj) || #if defined(NPY_PY3K) @@ -969,7 +974,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, } r = discover_dimensions(e, &maxndim_m1, d + 1, check_it, stop_at_string, stop_at_tuple, - out_is_object); + out_is_object, out_contains_na); Py_DECREF(e); if (r < 0) { return r; @@ -993,7 +998,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, } r = discover_dimensions(e, &maxndim_m1, dtmp, check_it, stop_at_string, stop_at_tuple, - out_is_object); + out_is_object, out_contains_na); Py_DECREF(e); if (r < 0) { return r; @@ -1144,12 +1149,13 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd, fa->weakreflist = (PyObject *)NULL; if (nd > 0) { - fa->dimensions = PyDimMem_NEW(2*nd); + fa->dimensions = PyDimMem_NEW(3*nd); if (fa->dimensions == NULL) { PyErr_NoMemory(); goto fail; } fa->strides = fa->dimensions + nd; + fa->maskna_strides = fa->dimensions + 2 * nd; memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd); if (strides == NULL) { /* fill it in */ sd = _array_fill_strides(fa->strides, dims, nd, sd, @@ -1323,7 +1329,9 @@ PyArray_NewLikeArray(PyArrayObject *prototype, NPY_ORDER order, _npy_stride_sort_item strideperm[NPY_MAXDIMS]; int i; - PyArray_CreateSortedStridePerm(prototype, strideperm); + PyArray_CreateSortedStridePerm(PyArray_NDIM(prototype), + PyArray_STRIDES(prototype), + strideperm); /* Build the new strides */ stride = dtype->elsize; @@ -1473,72 +1481,26 @@ fail: #endif } -/*NUMPY_API - * Retrieves the array parameters for viewing/converting an arbitrary - * PyObject* to a NumPy array. This allows the "innate type and shape" - * of Python list-of-lists to be discovered without - * actually converting to an array. - * - * In some cases, such as structured arrays and the __array__ interface, - * a data type needs to be used to make sense of the object. When - * this is needed, provide a Descr for 'requested_dtype', otherwise - * provide NULL. This reference is not stolen. Also, if the requested - * dtype doesn't modify the interpretation of the input, out_dtype will - * still get the "innate" dtype of the object, not the dtype passed - * in 'requested_dtype'. - * - * If writing to the value in 'op' is desired, set the boolean - * 'writeable' to 1. This raises an error when 'op' is a scalar, list - * of lists, or other non-writeable 'op'. - * - * Result: When success (0 return value) is returned, either out_arr - * is filled with a non-NULL PyArrayObject and - * the rest of the parameters are untouched, or out_arr is - * filled with NULL, and the rest of the parameters are - * filled. - * - * Typical usage: - * - * PyArrayObject *arr = NULL; - * PyArray_Descr *dtype = NULL; - * int ndim = 0; - * npy_intp dims[NPY_MAXDIMS]; +/* + * A slight generalization of PyArray_GetArrayParamsFromObject, + * which also returns whether the input data contains any numpy.NA + * values. * - * if (PyArray_GetArrayParamsFromObject(op, NULL, 1, &dtype, - * &ndim, &dims, &arr, NULL) < 0) { - * return NULL; - * } - * if (arr == NULL) { - * ... validate/change dtype, validate flags, ndim, etc ... - * // Could make custom strides here too - * arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, - * dims, NULL, - * is_f_order ? NPY_ARRAY_F_CONTIGUOUS : 0, - * NULL); - * if (arr == NULL) { - * return NULL; - * } - * if (PyArray_CopyObject(arr, op) < 0) { - * Py_DECREF(arr); - * return NULL; - * } - * } - * else { - * ... in this case the other parameters weren't filled, just - * validate and possibly copy arr itself ... - * } - * ... use arr ... + * This isn't exposed in the public API. */ NPY_NO_EXPORT int -PyArray_GetArrayParamsFromObject(PyObject *op, +PyArray_GetArrayParamsFromObjectEx(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyArray_Descr **out_dtype, int *out_ndim, npy_intp *out_dims, + int *out_contains_na, PyArrayObject **out_arr, PyObject *context) { PyObject *tmp; + *out_contains_na = 0; + /* If op is an array */ if (PyArray_Check(op)) { if (writeable && !PyArray_ISWRITEABLE((PyArrayObject *)op)) { @@ -1687,7 +1649,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op, is_object = 0; if (discover_dimensions(op, out_ndim, out_dims, check_it, stop_at_string, stop_at_tuple, - &is_object) < 0) { + &is_object, out_contains_na) < 0) { Py_DECREF(*out_dtype); if (PyErr_Occurred()) { return -1; @@ -1763,6 +1725,76 @@ PyArray_GetArrayParamsFromObject(PyObject *op, } /*NUMPY_API + * Retrieves the array parameters for viewing/converting an arbitrary + * PyObject* to a NumPy array. This allows the "innate type and shape" + * of Python list-of-lists to be discovered without + * actually converting to an array. + * + * In some cases, such as structured arrays and the __array__ interface, + * a data type needs to be used to make sense of the object. When + * this is needed, provide a Descr for 'requested_dtype', otherwise + * provide NULL. This reference is not stolen. Also, if the requested + * dtype doesn't modify the interpretation of the input, out_dtype will + * still get the "innate" dtype of the object, not the dtype passed + * in 'requested_dtype'. + * + * If writing to the value in 'op' is desired, set the boolean + * 'writeable' to 1. This raises an error when 'op' is a scalar, list + * of lists, or other non-writeable 'op'. + * + * Result: When success (0 return value) is returned, either out_arr + * is filled with a non-NULL PyArrayObject and + * the rest of the parameters are untouched, or out_arr is + * filled with NULL, and the rest of the parameters are + * filled. + * + * Typical usage: + * + * PyArrayObject *arr = NULL; + * PyArray_Descr *dtype = NULL; + * int ndim = 0; + * npy_intp dims[NPY_MAXDIMS]; + * + * if (PyArray_GetArrayParamsFromObject(op, NULL, 1, &dtype, + * &ndim, &dims, &arr, NULL) < 0) { + * return NULL; + * } + * if (arr == NULL) { + * ... validate/change dtype, validate flags, ndim, etc ... + * // Could make custom strides here too + * arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, + * dims, NULL, + * is_f_order ? NPY_ARRAY_F_CONTIGUOUS : 0, + * NULL); + * if (arr == NULL) { + * return NULL; + * } + * if (PyArray_CopyObject(arr, op) < 0) { + * Py_DECREF(arr); + * return NULL; + * } + * } + * else { + * ... in this case the other parameters weren't filled, just + * validate and possibly copy arr itself ... + * } + * ... use arr ... + */ +NPY_NO_EXPORT int +PyArray_GetArrayParamsFromObject(PyObject *op, + PyArray_Descr *requested_dtype, + npy_bool writeable, + PyArray_Descr **out_dtype, + int *out_ndim, npy_intp *out_dims, + PyArrayObject **out_arr, PyObject *context) +{ + int contains_na = 0; + return PyArray_GetArrayParamsFromObjectEx(op, requested_dtype, + writeable, out_dtype, out_ndim, out_dims, + &contains_na, out_arr, context); +} + +/*NUMPY_API * Does not check for NPY_ARRAY_ENSURECOPY and NPY_ARRAY_NOTSWAPPED in flags * Steals a reference to newtype --- which can be NULL */ @@ -1776,13 +1808,13 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, */ PyArrayObject *arr = NULL, *ret; PyArray_Descr *dtype = NULL; - int ndim = 0; + int ndim = 0, contains_na = 0; npy_intp dims[NPY_MAXDIMS]; /* Get either the array or its parameters if it isn't an array */ - if (PyArray_GetArrayParamsFromObject(op, newtype, + if (PyArray_GetArrayParamsFromObjectEx(op, newtype, 0, &dtype, - &ndim, dims, &arr, context) < 0) { + &ndim, dims, &contains_na, &arr, context) < 0) { Py_XDECREF(newtype); return NULL; } @@ -1851,9 +1883,21 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create an array and copy the data */ ret = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, newtype, - ndim, dims, - NULL, NULL, - flags&NPY_ARRAY_F_CONTIGUOUS, NULL); + ndim, dims, + NULL, NULL, + flags&NPY_ARRAY_F_CONTIGUOUS, NULL); + /* + * Add an NA mask if requested, or if allowed and the data + * has NAs + */ + if ((flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) != 0 || + (contains_na && (flags & NPY_ARRAY_ALLOWNA))) { + if (PyArray_AllocateMaskNA(ret, + (flags&NPY_ARRAY_OWNMASKNA) != 0, 0) < 0) { + Py_DECREF(ret); + return NULL; + } + } if (ret != NULL) { if (ndim > 0) { if (PyArray_AssignFromSequence(ret, op) < 0) { @@ -1952,7 +1996,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, PyArray_DESCR_REPLACE(descr); } if (descr) { - descr->byteorder = PyArray_NATIVE; + descr->byteorder = NPY_NATIVE; } } diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 045e6c4b5..a14c7de36 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -25,6 +25,7 @@ #include "_datetime.h" #include "datetime_strings.h" +#include "shape.h" #include "lowlevel_strided_loops.h" #define NPY_LOWLEVEL_BUFFER_BLOCKSIZE 128 @@ -3870,3 +3871,126 @@ PyArray_CastRawArrays(npy_intp count, /* If needs_api was set to 1, it may have raised a Python exception */ return (needs_api && PyErr_Occurred()) ? NPY_FAIL : NPY_SUCCEED; } + +/* + * Casts the elements from one n-dimensional array to another n-dimensional + * array with identical shape but possibly different strides and dtypes. + * Does not account for overlap. + * + * Returns NPY_SUCCEED or NPY_FAIL. + */ +NPY_NO_EXPORT int +PyArray_CastRawNDimArrays(int ndim, npy_intp *shape, + char *src, char *dst, + npy_intp *src_strides, npy_intp *dst_strides, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references) +{ + PyArray_StridedTransferFn *stransfer = NULL; + NpyAuxData *transferdata = NULL; + int i, j; + npy_intp src_align, dst_align; + int aligned, needs_api = 0; + npy_intp coords[NPY_MAXDIMS]; + npy_intp shape_copy[NPY_MAXDIMS]; + npy_intp src_strides_copy[NPY_MAXDIMS]; + npy_intp dst_strides_copy[NPY_MAXDIMS]; + _npy_stride_sort_item strideperm[NPY_MAXDIMS]; + + /* Use the simpler function for 0 and 1 dimensional transfers */ + if (ndim <= 1) { + if (ndim == 0) { + return PyArray_CastRawArrays(1, src, dst, 0, 0, + src_dtype, dst_dtype, move_references); + } + else { + return PyArray_CastRawArrays(shape[0], src, dst, + src_strides[0], dst_strides[0], + src_dtype, dst_dtype, move_references); + } + } + + /* Determine data alignment */ + src_align = (npy_intp)src; + for (i = 0; i < ndim; ++i) { + src_align |= src_strides[i]; + } + dst_align = (npy_intp)dst; + for (i = 0; i < ndim; ++i) { + dst_align |= dst_strides[i]; + } + aligned = (src_align & (src_dtype->alignment - 1)) == 0 && + (dst_align & (dst_dtype->alignment - 1)) == 0; + + /* Sort the axes based on the destination strides */ + PyArray_CreateSortedStridePerm(ndim, dst_strides, strideperm); + for (i = 0; i < ndim; ++i) { + int iperm = strideperm[i].perm; + shape_copy[i] = shape[iperm]; + src_strides_copy[i] = src_strides[iperm]; + dst_strides_copy[i] = dst_strides[iperm]; + } + + /* Coalesce dimensions where it's possible */ + i = 0; + for (j = 1; j < ndim; ++j) { + if (shape[i] == 1) { + /* Drop axis i */ + shape[i] = shape[j]; + src_strides_copy[i] = src_strides_copy[j]; + dst_strides_copy[i] = dst_strides_copy[j]; + } + else if (shape[j] == 1) { + /* Drop axis j */ + } + else if (src_strides_copy[i] == src_strides_copy[j] * shape[j] && + dst_strides_copy[i] == dst_strides_copy[j] * shape[j]) { + /* Coalesce axes i and j */ + shape[i] *= shape[j]; + src_strides_copy[i] = src_strides_copy[j]; + dst_strides_copy[i] = dst_strides_copy[j]; + } + else { + /* Can't coalesce, go to next i */ + ++i; + } + } + ndim = i+1; + + /* Get the function to do the casting */ + if (PyArray_GetDTypeTransferFunction(aligned, + src_strides[ndim-1], dst_strides[ndim-1], + src_dtype, dst_dtype, + move_references, + &stransfer, &transferdata, + &needs_api) != NPY_SUCCEED) { + return NPY_FAIL; + } + + /* Do the copying */ + memset(coords, 0, ndim * sizeof(npy_intp)); + do { + /* Copy along the last dimension */ + i = ndim - 1; + stransfer(dst, dst_strides_copy[i], src, src_strides_copy[i], shape[i], + src_dtype->elsize, transferdata); + --i; + /* Increment to the next n-dimensional coordinate */ + for (;i > 0; --i) { + if (++coords[i] == shape[i]) { + coords[i] = 0; + src -= (shape[i] - 1) * src_strides_copy[i]; + } + else { + src += src_strides_copy[i]; + break; + } + } + } while (i > 0); + + /* Cleanup */ + NPY_AUXDATA_FREE(transferdata); + + /* If needs_api was set to 1, it may have raised a Python exception */ + return (needs_api && PyErr_Occurred()) ? NPY_FAIL : NPY_SUCCEED; +} diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index 31a7d041e..72acfd6ee 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -181,14 +181,14 @@ arrayflags_dealloc(PyArrayFlagsObject *self) } -#define _define_get(UPPER, lower) \ - static PyObject * \ - arrayflags_ ## lower ## _get(PyArrayFlagsObject *self) \ - { \ - PyObject *item; \ +#define _define_get(UPPER, lower) \ + static PyObject * \ + arrayflags_ ## lower ## _get(PyArrayFlagsObject *self) \ + { \ + PyObject *item; \ item = ((self->flags & (UPPER)) == (UPPER)) ? Py_True : Py_False; \ - Py_INCREF(item); \ - return item; \ + Py_INCREF(item); \ + return item; \ } _define_get(NPY_ARRAY_C_CONTIGUOUS, contiguous) @@ -260,6 +260,80 @@ arrayflags_num_get(PyArrayFlagsObject *self) return PyInt_FromLong(self->flags); } +static PyObject * +arrayflags_maskna_get(PyArrayFlagsObject *self) +{ + PyObject *item; + if (self->flags & NPY_ARRAY_MASKNA) { + item = Py_True; + } + else { + item = Py_False; + } + Py_INCREF(item); + return item; +} + +static int +arrayflags_maskna_set(PyArrayFlagsObject *self, PyObject *obj) +{ + if (self->arr == NULL) { + PyErr_SetString(PyExc_ValueError, "Cannot set flags on array scalars."); + return -1; + } + + if (PyObject_IsTrue(obj)) { + return PyArray_AllocateMaskNA(self->arr, 0, 0); + } + else { + if (self->flags & NPY_ARRAY_MASKNA) { + PyErr_SetString(PyExc_ValueError, + "Cannot remove a NumPy array's NA mask"); + return -1; + } + else { + return 0; + } + } +} + +static PyObject * +arrayflags_ownmaskna_get(PyArrayFlagsObject *self) +{ + PyObject *item; + if (self->flags & NPY_ARRAY_MASKNA) { + item = Py_True; + } + else { + item = Py_False; + } + Py_INCREF(item); + return item; +} + +static int +arrayflags_ownmaskna_set(PyArrayFlagsObject *self, PyObject *obj) +{ + if (self->arr == NULL) { + PyErr_SetString(PyExc_ValueError, "Cannot set flags on array scalars."); + return -1; + } + + if (PyObject_IsTrue(obj)) { + return PyArray_AllocateMaskNA(self->arr, 1, 0); + } + else { + if (self->flags & NPY_ARRAY_OWNMASKNA) { + PyErr_SetString(PyExc_ValueError, + "Cannot remove a NumPy array's NA mask"); + return -1; + } + else { + return 0; + } + } +} + /* relies on setflags order being write, align, uic */ static int arrayflags_updateifcopy_set(PyArrayFlagsObject *self, PyObject *obj) @@ -348,6 +422,14 @@ static PyGetSetDef arrayflags_getsets[] = { (getter)arrayflags_writeable_get, (setter)arrayflags_writeable_set, NULL, NULL}, + {"maskna", + (getter)arrayflags_maskna_get, + (setter)arrayflags_maskna_set, + NULL, NULL}, + {"ownmaskna", + (getter)arrayflags_ownmaskna_get, + (setter)arrayflags_ownmaskna_set, + NULL, NULL}, {"fnc", (getter)arrayflags_fnc_get, NULL, @@ -450,6 +532,9 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) if (strncmp(key, "FARRAY", n) == 0) { return arrayflags_farray_get(self); } + if (strncmp(key, "MASKNA", n) == 0) { + return arrayflags_maskna_get(self); + } break; case 7: if (strncmp(key,"FORTRAN",n) == 0) { @@ -469,6 +554,9 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) if (strncmp(key,"WRITEABLE",n) == 0) { return arrayflags_writeable_get(self); } + if (strncmp(key, "OWNMASKNA", n) == 0) { + return arrayflags_ownmaskna_get(self); + } break; case 10: if (strncmp(key,"CONTIGUOUS",n) == 0) { @@ -528,6 +616,12 @@ arrayflags_setitem(PyArrayFlagsObject *self, PyObject *ind, PyObject *item) ((n==1) && (strncmp(key, "U", n) == 0))) { return arrayflags_updateifcopy_set(self, item); } + else if ((n==6) && (strncmp(key, "MASKNA", n) == 0)) { + return arrayflags_maskna_set(self, item); + } + else if ((n==9) && (strncmp(key, "OWNMASKNA", n) == 0)) { + return arrayflags_ownmaskna_set(self, item); + } fail: PyErr_SetString(PyExc_KeyError, "Unknown flag"); @@ -555,6 +649,8 @@ arrayflags_print(PyArrayFlagsObject *self) "C_CONTIGUOUS", _torf_(fl, NPY_ARRAY_C_CONTIGUOUS), "F_CONTIGUOUS", _torf_(fl, NPY_ARRAY_F_CONTIGUOUS), "OWNDATA", _torf_(fl, NPY_ARRAY_OWNDATA), + "MASKNA", _torf_(fl, NPY_ARRAY_MASKNA), + "OWNMASKNA", _torf_(fl, NPY_ARRAY_OWNMASKNA), "WRITEABLE", _torf_(fl, NPY_ARRAY_WRITEABLE), "ALIGNED", _torf_(fl, NPY_ARRAY_ALIGNED), "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY)); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 9491bc631..d18721333 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1542,13 +1542,11 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin) } -#define _ARET(x) PyArray_Return((PyArrayObject *)(x)) - -#define STRIDING_OK(op, order) ((order) == NPY_ANYORDER || \ - ((order) == NPY_CORDER && \ - PyArray_ISCONTIGUOUS(op)) || \ - ((order) == NPY_FORTRANORDER && \ - PyArray_ISFORTRAN(op))) +#define STRIDING_OK(op, order) \ + ((order) == NPY_ANYORDER || \ + (order) == NPY_KEEPORDER || \ + ((order) == NPY_CORDER && PyArray_ISCONTIGUOUS(op)) || \ + ((order) == NPY_FORTRANORDER && PyArray_ISFORTRAN(op))) static PyObject * _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) @@ -1561,37 +1559,49 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) PyArray_Descr *type = NULL; PyArray_Descr *oldtype = NULL; NPY_ORDER order = NPY_ANYORDER; - int flags = 0; + int flags = 0, maskna = 0, ownmaskna = 0; static char *kwd[]= {"object", "dtype", "copy", "order", "subok", - "ndmin", NULL}; + "ndmin", "maskna", NULL}; if (PyTuple_GET_SIZE(args) > 2) { PyErr_SetString(PyExc_ValueError, "only 2 non-keyword arguments accepted"); return NULL; } - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i", kwd, &op, + if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&iii", kwd, + &op, PyArray_DescrConverter2, &type, PyArray_BoolConverter, ©, PyArray_OrderConverter, &order, PyArray_BoolConverter, &subok, - &ndmin)) { + &ndmin, + &maskna, + &ownmaskna)) { goto clean_type; } + /* 'ownmaskna' forces 'maskna' to be True */ + if (ownmaskna) { + maskna = 1; + } + if (ndmin > NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, - "ndmin bigger than allowable number of dimensions "\ + "ndmin bigger than allowable number of dimensions " "NPY_MAXDIMS (=%d)", NPY_MAXDIMS); goto clean_type; } /* fast exit if simple call */ - if ((subok && PyArray_Check(op)) - || (!subok && PyArray_CheckExact(op))) { + if (((subok && PyArray_Check(op)) || + (!subok && PyArray_CheckExact(op))) && + ((maskna && + (PyArray_FLAGS((PyArrayObject *)op)&NPY_ARRAY_MASKNA) != 0) || + (!maskna && + (PyArray_FLAGS((PyArrayObject *)op)&NPY_ARRAY_MASKNA) == 0))) { oparr = (PyArrayObject *)op; if (type == NULL) { - if (!copy && STRIDING_OK(oparr, order)) { + if (!copy && !ownmaskna && STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; @@ -1604,7 +1614,7 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) /* One more chance */ oldtype = PyArray_DESCR(oparr); if (PyArray_EquivTypes(oldtype, type)) { - if (!copy && STRIDING_OK(oparr, order)) { + if (!copy && !ownmaskna && STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; @@ -1637,6 +1647,12 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) if (!subok) { flags |= NPY_ARRAY_ENSUREARRAY; } + if (maskna) { + flags |= NPY_ARRAY_MASKNA; + } + if (maskna) { + flags |= NPY_ARRAY_OWNMASKNA; + } flags |= NPY_ARRAY_FORCECAST; Py_XINCREF(type); @@ -1645,10 +1661,12 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) finish: Py_XDECREF(type); - if (!ret) { - return (PyObject *)ret; + if (ret == NULL) { + return NULL; } - else if ((nd=PyArray_NDIM(ret)) >= ndmin) { + + nd = PyArray_NDIM(ret); + if (nd >= ndmin) { return (PyObject *)ret; } /* @@ -2069,7 +2087,7 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args) if (!PyArg_ParseTuple(args, "OO", &a0, &b0)) { return NULL; } - return _ARET(PyArray_InnerProduct(a0, b0)); + return PyArray_Return((PyArrayObject *)PyArray_InnerProduct(a0, b0)); } static PyObject * @@ -2089,7 +2107,7 @@ array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) "'out' must be an array"); return NULL; } - return _ARET(PyArray_MatrixProduct2(a, v, (PyArrayObject *)o)); + return PyArray_Return((PyArrayObject *)PyArray_MatrixProduct2(a, v, (PyArrayObject *)o)); } static int @@ -2430,7 +2448,7 @@ array_einsum(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) /* If no output was supplied, possibly convert to a scalar */ if (ret != NULL && out == NULL) { - ret = _ARET(ret); + ret = PyArray_Return((PyArrayObject *)ret); } finish: @@ -2453,7 +2471,7 @@ array_fastCopyAndTranspose(PyObject *NPY_UNUSED(dummy), PyObject *args) if (!PyArg_ParseTuple(args, "O", &a0)) { return NULL; } - return _ARET(PyArray_CopyAndTranspose(a0)); + return PyArray_Return((PyArrayObject *)PyArray_CopyAndTranspose(a0)); } static PyObject * @@ -2711,11 +2729,9 @@ array_lexsort(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i", kwlist, &obj, &axis)) { return NULL; } - return _ARET(PyArray_LexSort(obj, axis)); + return PyArray_Return((PyArrayObject *)PyArray_LexSort(obj, axis)); } -#undef _ARET - static PyObject * array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) diff --git a/numpy/core/src/multiarray/na_mask.c b/numpy/core/src/multiarray/na_mask.c index ce8dd1e4b..a7eab49c9 100644 --- a/numpy/core/src/multiarray/na_mask.c +++ b/numpy/core/src/multiarray/na_mask.c @@ -17,6 +17,9 @@ #include "npy_config.h" #include "numpy/npy_3kcompat.h" +#include "shape.h" +#include "lowlevel_strided_loops.h" + /*NUMPY_API * * Returns true if the array has an NA mask. When @@ -47,3 +50,135 @@ PyArray_ContainsNA(PyArrayObject *arr) return 0; } + +/*NUMPY_API + * + * If the array does not have an NA mask already, allocates one for it. + * + * If 'ownmaskna' is True, it also allocates one for it if the array does + * not already own its own mask, then copies the data from the old mask + * to the new mask. + * + * If 'multina' is True, the mask is allocated with an NPY_MASK dtype + * instead of NPY_BOOL. + * + * Returns -1 on failure, 0 on success. + */ +NPY_NO_EXPORT int +PyArray_AllocateMaskNA(PyArrayObject *arr, npy_bool ownmaskna, npy_bool multina) +{ + PyArrayObject_fieldaccess *fa = (PyArrayObject_fieldaccess *)arr; + PyArray_Descr *maskna_dtype = NULL; + char *maskna_data = NULL; + npy_intp size; + + /* If the array already owns a mask, done */ + if (fa->flags & NPY_ARRAY_OWNMASKNA) { + return 0; + } + + /* If ownership wasn't requested, and there's already a mask, done */ + if (!ownmaskna && (fa->flags & NPY_ARRAY_MASKNA)) { + return 0; + } + + size = PyArray_SIZE(arr); + + /* Create the mask dtype */ + if (PyArray_HASFIELDS(arr)) { + PyErr_SetString(PyExc_RuntimeError, + "NumPy field-NA isn't supported yet"); + return -1; + } + else { + maskna_dtype = PyArray_DescrFromType(multina ? NPY_MASK + : NPY_BOOL); + if (maskna_dtype == NULL) { + return -1; + } + } + + /* Allocate the mask memory */ + maskna_data = PyArray_malloc(size * maskna_dtype->elsize); + if (maskna_data == NULL) { + Py_DECREF(maskna_dtype); + PyErr_NoMemory(); + return -1; + } + + /* Copy the data and fill in the strides */ + if (fa->nd == 1) { + /* If there already was a mask copy it, otherwise set it to all ones */ + if (fa->flags & NPY_ARRAY_MASKNA) { + if (fa->maskna_strides[0] == 1) { + memcpy(maskna_data, fa->maskna_data, + size * maskna_dtype->elsize); + } + else { + if (PyArray_CastRawArrays(fa->dimensions[0], + (char *)fa->maskna_data, maskna_data, + fa->maskna_strides[0], maskna_dtype->elsize, + fa->maskna_dtype, maskna_dtype, 0) < 0) { + Py_DECREF(maskna_dtype); + PyArray_free(maskna_data); + return -1; + } + } + } + else { + memset(maskna_data, 1, size * maskna_dtype->elsize); + } + + fa->maskna_strides[0] = maskna_dtype->elsize; + } + else if (fa->nd > 1) { + _npy_stride_sort_item strideperm[NPY_MAXDIMS]; + npy_intp stride, maskna_strides[NPY_MAXDIMS], *shape; + int i; + + shape = fa->dimensions; + + /* This causes the NA mask and data memory orderings to match */ + PyArray_CreateSortedStridePerm(fa->nd, fa->strides, strideperm); + stride = maskna_dtype->elsize; + for (i = fa->nd-1; i >= 0; --i) { + npy_intp i_perm = strideperm[i].perm; + maskna_strides[i_perm] = stride; + stride *= shape[i_perm]; + } + + /* If there already was a mask copy it, otherwise set it to all ones */ + if (fa->flags & NPY_ARRAY_MASKNA) { + if (PyArray_CastRawNDimArrays(fa->nd, fa->dimensions, + (char *)fa->maskna_data, maskna_data, + fa->maskna_strides, maskna_strides, + fa->maskna_dtype, maskna_dtype, 0) < 0) { + Py_DECREF(maskna_dtype); + PyArray_free(maskna_data); + return -1; + } + } + else { + memset(maskna_data, 1, size * maskna_dtype->elsize); + } + + memcpy(fa->maskna_strides, maskna_strides, fa->nd * sizeof(npy_intp)); + } + else { + /* If there already was a mask copy it, otherwise set it to all ones */ + if (fa->flags & NPY_ARRAY_MASKNA) { + maskna_data[0] = fa->maskna_data[0]; + } + else { + maskna_data[0] = 1; + } + } + + /* Set the NA mask data in the array */ + Py_XDECREF(maskna_dtype); + fa->maskna_dtype = maskna_dtype; + fa->maskna_data = (npy_mask *)maskna_data; + fa->flags |= (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA); + + return 0; +} diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index 4875c1e34..ba3f0b677 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -1599,7 +1599,6 @@ npyiter_coalesce_axes(NpyIter *iter) } /* - * * If errmsg is non-NULL, it should point to a variable which will * receive the error message, and no Python exception will be set. * This is so that the function can be called from code not holding diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 3754e6a1e..532e8a2ba 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -807,21 +807,21 @@ int _npy_stride_sort_item_comparator(const void *a, const void *b) } /* - * This function populates the first PyArray_NDIM(arr) elements + * This function populates the first ndim elements * of strideperm with sorted descending by their absolute values. * For example, the stride array (4, -2, 12) becomes * [(2, 12), (0, 4), (1, -2)]. */ NPY_NO_EXPORT void -PyArray_CreateSortedStridePerm(PyArrayObject *arr, +PyArray_CreateSortedStridePerm(int ndim, npy_intp * strides, _npy_stride_sort_item *strideperm) { - int i, ndim = PyArray_NDIM(arr); + int i; /* Set up the strideperm values */ for (i = 0; i < ndim; ++i) { strideperm[i].perm = i; - strideperm[i].stride = PyArray_STRIDE(arr, i); + strideperm[i].stride = strides[i]; } /* Sort them */ @@ -865,7 +865,8 @@ PyArray_Ravel(PyArrayObject *a, NPY_ORDER order) npy_intp stride; int i, ndim = PyArray_NDIM(a); - PyArray_CreateSortedStridePerm(a, strideperm); + PyArray_CreateSortedStridePerm(PyArray_NDIM(a), PyArray_STRIDES(a), + strideperm); stride = PyArray_DESCR(a)->elsize; for (i = ndim-1; i >= 0; --i) { diff --git a/numpy/core/src/multiarray/shape.h b/numpy/core/src/multiarray/shape.h index 8038a9f25..71dbd1562 100644 --- a/numpy/core/src/multiarray/shape.h +++ b/numpy/core/src/multiarray/shape.h @@ -12,7 +12,8 @@ typedef struct { * [(2, 12), (0, 4), (1, -2)]. */ NPY_NO_EXPORT void -PyArray_CreateSortedStridePerm(PyArrayObject *arr, +PyArray_CreateSortedStridePerm(int ndim, npy_intp * strides, _npy_stride_sort_item *strideperm); + #endif diff --git a/numpy/core/src/private/lowlevel_strided_loops.h b/numpy/core/src/private/lowlevel_strided_loops.h index b6b53ba45..c606e58f3 100644 --- a/numpy/core/src/private/lowlevel_strided_loops.h +++ b/numpy/core/src/private/lowlevel_strided_loops.h @@ -220,7 +220,7 @@ PyArray_GetMaskedDTypeTransferFunction(int aligned, * 'src_dtype' to 'dst' with 'dst_dtype'. See * PyArray_GetDTypeTransferFunction for more details. * - * returns NPY_SUCCEED or NPY_FAIL. + * Returns NPY_SUCCEED or NPY_FAIL. */ NPY_NO_EXPORT int PyArray_CastRawArrays(npy_intp count, @@ -230,6 +230,20 @@ PyArray_CastRawArrays(npy_intp count, int move_references); /* + * Casts the elements from one n-dimensional array to another n-dimensional + * array with identical shape but possibly different strides and dtypes. + * Does not account for overlap. + * + * Returns NPY_SUCCEED or NPY_FAIL. + */ +NPY_NO_EXPORT int +PyArray_CastRawNDimArrays(int ndim, npy_intp *shape, + char *src, char *dst, + npy_intp *src_strides, npy_intp *dst_strides, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references); + +/* * These two functions copy or convert the data of an n-dimensional array * to/from a 1-dimensional strided buffer. These functions will only call * 'stransfer' with the provided dst_stride/src_stride and |