From c626f19a54624b3b66a1a7d77ef1ec90c1ff27cf Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 29 Jul 2011 17:57:46 -0500 Subject: ENH: missingdata: Fill in buffered NAMASK nditer, add maskna= to zeros, empty, and empty_like --- numpy/add_newdocs.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 1a3f9e461..65c9646eb 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -736,6 +736,8 @@ add_newdoc('numpy.core.multiarray', 'empty', order : {'C', 'F'}, optional Whether to store multi-dimensional data in C (row-major) or Fortran (column-major) order in memory. + maskna : boolean + If this is true, the returned array will have an NA mask. See Also -------- -- cgit v1.2.1 From e73dd6a826ad1b0ccb0e1cbe4aacfeee644f390f Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Sun, 31 Jul 2011 09:24:03 -0500 Subject: ENH: missingdata: Progress towards supporting ufunc reduce with NA masks --- numpy/add_newdocs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 65c9646eb..536cde9d1 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -1278,7 +1278,7 @@ add_newdoc('numpy.core.multiarray','correlate', add_newdoc('numpy.core.multiarray', 'arange', """ - arange([start,] stop[, step,], dtype=None) + arange([start,] stop[, step,], dtype=None, maskna=False) Return evenly spaced values within a given interval. @@ -1307,6 +1307,8 @@ add_newdoc('numpy.core.multiarray', 'arange', dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + maskna : boolean + If this is true, the returned array will have an NA mask. Returns ------- -- cgit v1.2.1 From ea5dcd7947615678bda053ed65ab921fec6f1421 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 2 Aug 2011 17:25:11 -0500 Subject: DOC: ufunc: Document the 'axis=' improvments to reduction functions --- numpy/add_newdocs.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 536cde9d1..95379d611 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -5299,8 +5299,22 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', ---------- a : array_like The array to act on. - axis : int, optional - The axis along which to apply the reduction. + axis : None or int or tuple of ints, optional + Axis or axes along which a reduction is performed. + The default (`axis` = 0) is perform a reduction over the first + dimension of the input array. `axis` may be negative, in + which case it counts from the last to the first axis. + + .. versionadded:: 1.7.0 + + If this is `None`, a reduction is performed over all the axes. + If this is a tuple of ints, a reduction is performed on multiple + axes, instead of a single axis or all the axes as before. + + For operations which are either not commutative or not associative, + doing a reduction over multiple axes is not well-defined. The + ufuncs do not currently raise an exception in this case, but will + likely do so in the future. dtype : data-type code, optional The type used to represent the intermediate results. Defaults to the data-type of the output array if this is provided, or -- cgit v1.2.1 From 99a21efff4b1f2292dc370c7c9c7c58f10385f2a Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 16 Aug 2011 09:07:16 -0700 Subject: ENH: missingdata: Add NA support to np.diagonal, change np.diagonal to always return a view --- numpy/add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 95379d611..d992a7122 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -3252,7 +3252,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('diagonal', Return specified diagonals. - Refer to `numpy.diagonal` for full documentation. + Refer to :func:`numpy.diagonal` for full documentation. See Also -------- -- cgit v1.2.1 From 9194b3af704df71aa9b1ff2f53f169848d0f9dc7 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 16 Aug 2011 16:54:16 -0700 Subject: ENH: missingdata: Rewrite PyArray_Concatenate to work with NA masks It should also have less memory usage for heterogeneous inputs, because it no longer makes extra copies in that case. --- numpy/add_newdocs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index d992a7122..82f59c6b0 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -3702,7 +3702,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('put', add_newdoc('numpy.core.multiarray', 'copyto', """ - copyto(dst, src, casting='same_kind', where=None) + copyto(dst, src, casting='same_kind', where=None, preservena=False) Copies values from `src` into `dst`, broadcasting as necessary. Raises a TypeError if the casting rule is violated, and if @@ -3725,10 +3725,13 @@ add_newdoc('numpy.core.multiarray', 'copyto', * 'same_kind' means only safe casts or casts within a kind, like float64 to float32, are allowed. * 'unsafe' means any data conversions may be done. - where : array_like of bool + where : array_like of bool, optional A boolean array which is broadcasted to match the dimensions of `dst`, and selects elements to copy from `src` to `dst` wherever it contains the value True. + preservena : bool, optional + If set to True, leaves any NA values in `dst` untouched. This + is similar to the "hard mask" feature in numpy.ma. """) -- cgit v1.2.1 From bfda229ec93d37b1ee2cdd8b9443ec4e34536bbf Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 17 Aug 2011 19:18:56 -0700 Subject: ENH: missingdata: Create count_reduce_items function This function either cheaply returns the product of the sizes of all the reduction axes, or counts the number of items which will be used in a reduction operation when skipna is True. Its purpose is to make it easy to do functions like np.mean and np.std. --- numpy/add_newdocs.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 82f59c6b0..8596b9c9c 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -915,6 +915,56 @@ add_newdoc('numpy.core.multiarray', 'count_nonzero', 5 """) +add_newdoc('numpy.core.multiarray', 'count_reduce_items', + """ + count_reduce_items(arr, axis=None, skipna=False) + + Counts the number of items a reduction with the same `axis` + and `skipna` parameter values would use. The purpose of this + function is for the creation of reduction operations + which use the item count, such as :func:`mean`. + + When `skipna` is False or `arr` doesn't have an NA mask, + the result is simply the product of the reduction axis + sizes, returned as a single scalar. + + Parameters + ---------- + arr : array_like + The array for which to count the reduce items. + axis : None or int or tuple of ints, optional + Axis or axes along which a reduction is performed. + The default (`axis` = None) is perform a reduction over all + the dimensions of the input array. + skipna : bool, optional + If this is set to True, any NA elements in the array are not + counted. The only time this function does any actual counting + instead of a cheap multiply of a few sizes is when `skipna` is + true and `arr` has an NA mask. + + Returns + ------- + count : intp or array of intp + Number of items that would be used in a reduction with the + same `axis` and `skipna` parameter values. + + Examples + -------- + >>> a = np.array([[1,np.NA,1], [1,1,np.NA]]) + + >>> np.count_reduce_items(a) + 6 + >>> np.count_reduce_items(a, skipna=True) + 4 + >>> np.sum(a, skipna=True) + 4 + + >>> np.count_reduce_items(a, axis=0, skipna=True) + array([2, 1, 1]) + >>> np.sum(a, axis=0, skipna=True) + array([2, 1, 1]) + """) + add_newdoc('numpy.core.multiarray','set_typeDict', """set_typeDict(dict) @@ -5280,7 +5330,7 @@ add_newdoc('numpy.core', 'ufunc', ('types', add_newdoc('numpy.core', 'ufunc', ('reduce', """ - reduce(a, axis=0, dtype=None, out=None) + reduce(a, axis=0, dtype=None, out=None, skipna=False) Reduces `a`'s dimension by one, by applying ufunc along one axis. @@ -5325,6 +5375,11 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', out : ndarray, optional A location into which the result is stored. If not provided, a freshly-allocated array is returned. + skipna : bool, optional + If this is set to True, the reduction is done as if any NA elements + were not counted in the array. The default, False, causes the + NA values to propagate, so if any element in a set of elements + being reduced is NA, the result will be NA. Returns ------- -- cgit v1.2.1 From 0fa4f22fec4b19e2a8c1d93e5a1f955167c9addd Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 17 Aug 2011 22:03:06 -0700 Subject: ENH: missingdata: Support 'skipna=' parameter in np.mean Also add 'keepdims=' parameter to reductions, to support writing of the np.std function. --- numpy/add_newdocs.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 8596b9c9c..b402d2150 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -896,10 +896,21 @@ add_newdoc('numpy.core.multiarray', 'count_nonzero', ---------- a : array_like The array for which to count non-zeros. + axis : None or int or tuple of ints, optional + Axis or axes along which a reduction is performed. + The default (`axis` = None) is perform a reduction over all + the dimensions of the input array. + skipna : bool, optional + If this is set to True, any NA elements in the array are skipped + instead of propagating. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the original `arr`. Returns ------- - count : int + count : int or array of int Number of non-zero values in the array. See Also @@ -910,14 +921,18 @@ add_newdoc('numpy.core.multiarray', 'count_nonzero', -------- >>> np.count_nonzero(np.eye(4)) 4 - >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]]) 5 + >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=1) + array([2, 3]) + >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=1, keepdims=True) + array([[2], + [3]]) """) add_newdoc('numpy.core.multiarray', 'count_reduce_items', """ - count_reduce_items(arr, axis=None, skipna=False) + count_reduce_items(arr, axis=None, skipna=False, keepdims=False) Counts the number of items a reduction with the same `axis` and `skipna` parameter values would use. The purpose of this @@ -941,6 +956,10 @@ add_newdoc('numpy.core.multiarray', 'count_reduce_items', counted. The only time this function does any actual counting instead of a cheap multiply of a few sizes is when `skipna` is true and `arr` has an NA mask. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the original `arr`. Returns ------- @@ -5330,7 +5349,7 @@ add_newdoc('numpy.core', 'ufunc', ('types', add_newdoc('numpy.core', 'ufunc', ('reduce', """ - reduce(a, axis=0, dtype=None, out=None, skipna=False) + reduce(a, axis=0, dtype=None, out=None, skipna=False, keepdims=False) Reduces `a`'s dimension by one, by applying ufunc along one axis. @@ -5380,6 +5399,10 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', were not counted in the array. The default, False, causes the NA values to propagate, so if any element in a set of elements being reduced is NA, the result will be NA. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the original `arr`. Returns ------- -- cgit v1.2.1 From a112fc4a6b28fbb85e1b0c6d423095d13cf7b226 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 17 Aug 2011 23:07:58 -0700 Subject: ENH: missingdata: Implement skipna= support for np.std and np.var --- numpy/add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index b402d2150..711b7de52 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -4194,7 +4194,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('sort', add_newdoc('numpy.core.multiarray', 'ndarray', ('squeeze', """ - a.squeeze() + a.squeeze(axis=None) Remove single-dimensional entries from the shape of `a`. -- cgit v1.2.1 From 0e1a4e9525b2c1e4abae97a6927cf59b5b2d534b Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Mon, 22 Aug 2011 14:01:55 -0700 Subject: ENH: missingdata: Add maskna= parameter to np.copy and ndarray.copy --- numpy/add_newdocs.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 711b7de52..7bd24b45e 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -3247,17 +3247,21 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('conjugate', add_newdoc('numpy.core.multiarray', 'ndarray', ('copy', """ - a.copy(order='C') + a.copy(order='C', maskna=None) Return a copy of the array. Parameters ---------- - order : {'C', 'F', 'A'}, optional - By default, the result is stored in C-contiguous (row-major) order in - memory. If `order` is `F`, the result has 'Fortran' (column-major) - order. If order is 'A' ('Any'), then the result has the same order - as the input. + order : {'C', 'F', 'A', 'K'}, optional + Controls the memory layout of the copy. 'C' means C-order, + 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, + 'C' otherwise. 'K' means match the layout of `a` as closely + as possible. + maskna : bool, optional + If specifies, forces the copy to have or to not have an + NA mask. This is a way to remove an NA mask from an array + while making a copy. See also -------- -- cgit v1.2.1 From 6c0ad59d384606ccf2a0afa20fb9d8a15ddd7255 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 23 Aug 2011 21:31:09 -0700 Subject: ENH: missingdata: Add maskna= and ownmaskna= parameters to np.asarray and friends Also fix some array() NA mask construction issues and make sure the base object doesn't collapse past the owner of the NA mask being viewed in addition to the data. --- numpy/add_newdocs.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 7bd24b45e..d553e3ea2 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -631,7 +631,7 @@ add_newdoc('numpy.core', 'broadcast', ('reset', add_newdoc('numpy.core.multiarray', 'array', """ - array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0) + array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0, maskna=None, ownmaskna=False) Create an array. @@ -667,6 +667,19 @@ add_newdoc('numpy.core.multiarray', 'array', Specifies the minimum number of dimensions that the resulting array should have. Ones will be pre-pended to the shape as needed to meet this requirement. + maskna : bool or None, optional + If this is set to True, it forces the array to have an NA mask. + If the input is an array without a mask, this means a view with + an NA mask is created. If the input is an array with a mask, the + mask is preserved as-is. + + If this is set to False, it forces the array to not have an NA + mask. If the input is an array with a mask, and has no NA values, + it will create a copy of the input without an NA mask. + ownmaskna : bool, optional + If this is set to True, forces the array to have a mask which + it owns. It may still return a view of the data from the input, + but the result will always own its own mask. Returns ------- -- cgit v1.2.1 From b64ace52c232fca7c1532666d202b756389a6b33 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 25 Aug 2011 10:21:32 -0700 Subject: DOC: missingdata: Add introductory documentation for NA-masked arrays --- numpy/add_newdocs.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'numpy/add_newdocs.py') diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index d553e3ea2..ce61c5f0e 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -197,6 +197,15 @@ add_newdoc('numpy.core', 'nditer', * "allocate" causes the array to be allocated if it is None in the `op` parameter. * "no_subtype" prevents an "allocate" operand from using a subtype. + * "arraymask" indicates that this operand is the mask to use + for selecting elements when writing to operands with the + 'writemasked' flag set. The iterator does not enforce this, + but when writing from a buffer back to the array, it only + copies those elements indicated by this mask. + * 'writemasked' indicates that only elements where the chosen + 'arraymask' operand is True will be written to. + * 'use_maskna' indicates that this operand should be treated + like an NA-masked array. op_dtypes : dtype or tuple of dtype(s), optional The required data type(s) of the operands. If copying or buffering is enabled, the data will be converted to/from their original types. @@ -899,6 +908,35 @@ add_newdoc('numpy.core.multiarray', 'zeros', """) +add_newdoc('numpy.core.multiarray', 'isna', + """ + isna(a) + + Returns an array with True for each element of *a* that is NA. + + Parameters + ---------- + a : array_like + The array for which to check for NA. + + Returns + ------- + result : bool or array of bool + Number of non-zero values in the array. + + Examples + -------- + >>> np.isna(np.NA) + True + >>> np.isna(1.5) + False + >>> np.isna(np.nan) + False + >>> a = np.array([0, np.NA, 3.5, np.NA]) + >>> np.isna(a) + array([False, True, False, True], dtype=bool) + """) + add_newdoc('numpy.core.multiarray', 'count_nonzero', """ count_nonzero(a) -- cgit v1.2.1