diff options
Diffstat (limited to 'numpy/lib')
42 files changed, 3129 insertions, 2471 deletions
diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index c1757150e..2db12d9a4 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -1,14 +1,31 @@ +""" +**Note:** almost all functions in the ``numpy.lib`` namespace +are also present in the main ``numpy`` namespace. Please use the +functions as ``np.<funcname>`` where possible. + +``numpy.lib`` is mostly a space for implementing functions that don't +belong in core or in another NumPy submodule with a clear purpose +(e.g. ``random``, ``fft``, ``linalg``, ``ma``). + +Most contains basic functions that are used by several submodules and are +useful to have in the main name-space. + +""" from __future__ import division, absolute_import, print_function import math -from .info import __doc__ from numpy.version import version as __version__ +# Public submodules +# Note: recfunctions and (maybe) format are public too, but not imported +from . import mixins +from . import scimath as emath + +# Private submodules from .type_check import * from .index_tricks import * from .function_base import * -from .mixins import * from .nanfunctions import * from .shape_base import * from .stride_tricks import * @@ -16,9 +33,7 @@ from .twodim_base import * from .ufunclike import * from .histograms import * -from . import scimath as emath from .polynomial import * -#import convertcode from .utils import * from .arraysetops import * from .npyio import * @@ -28,11 +43,10 @@ from .arraypad import * from ._version import * from numpy.core._multiarray_umath import tracemalloc_domain -__all__ = ['emath', 'math', 'tracemalloc_domain'] +__all__ = ['emath', 'math', 'tracemalloc_domain', 'Arrayterator'] __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ -__all__ += mixins.__all__ __all__ += shape_base.__all__ __all__ += stride_tricks.__all__ __all__ += twodim_base.__all__ diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index 3a0e67f60..0d71375c2 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -41,6 +41,7 @@ import sys import warnings import shutil import io +from contextlib import closing from numpy.core.overrides import set_module @@ -414,13 +415,9 @@ class DataSource(object): # TODO: Doesn't handle compressed files! if self._isurl(path): try: - openedurl = urlopen(path) - f = _open(upath, 'wb') - try: - shutil.copyfileobj(openedurl, f) - finally: - f.close() - openedurl.close() + with closing(urlopen(path)) as openedurl: + with _open(upath, 'wb') as f: + shutil.copyfileobj(openedurl, f) except URLError: raise URLError("URL not found: %s" % path) else: @@ -547,6 +544,11 @@ class DataSource(object): is accessible if it exists in either location. """ + + # First test for local path + if os.path.exists(path): + return True + # We import this here because importing urllib2 is slow and # a significant fraction of numpy's total import time. if sys.version_info[0] >= 3: @@ -556,10 +558,6 @@ class DataSource(object): from urllib2 import urlopen from urllib2 import URLError - # Test local path - if os.path.exists(path): - return True - # Test cached url upath = self.abspath(path) if os.path.exists(upath): diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 0ebd39b8c..c392929fd 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -121,7 +121,7 @@ def has_nested_fields(ndtype): """ for name in ndtype.names or (): - if ndtype[name].names: + if ndtype[name].names is not None: return True return False @@ -931,28 +931,27 @@ def easy_dtype(ndtype, names=None, defaultfmt="f%i", **validationargs): names = validate(names, nbfields=nbfields, defaultfmt=defaultfmt) ndtype = np.dtype(dict(formats=ndtype, names=names)) else: - nbtypes = len(ndtype) # Explicit names if names is not None: validate = NameValidator(**validationargs) if isinstance(names, basestring): names = names.split(",") # Simple dtype: repeat to match the nb of names - if nbtypes == 0: + if ndtype.names is None: formats = tuple([ndtype.type] * len(names)) names = validate(names, defaultfmt=defaultfmt) ndtype = np.dtype(list(zip(names, formats))) # Structured dtype: just validate the names as needed else: - ndtype.names = validate(names, nbfields=nbtypes, + ndtype.names = validate(names, nbfields=len(ndtype.names), defaultfmt=defaultfmt) # No implicit names - elif (nbtypes > 0): + elif ndtype.names is not None: validate = NameValidator(**validationargs) # Default initial names : should we change the format ? - if ((ndtype.names == tuple("f%i" % i for i in range(nbtypes))) and + if ((ndtype.names == tuple("f%i" % i for i in range(len(ndtype.names)))) and (defaultfmt != "f%i")): - ndtype.names = validate([''] * nbtypes, defaultfmt=defaultfmt) + ndtype.names = validate([''] * len(ndtype.names), defaultfmt=defaultfmt) # Explicit initial names : just validate else: ndtype.names = validate(ndtype.names, defaultfmt=defaultfmt) diff --git a/numpy/lib/arraypad.py b/numpy/lib/arraypad.py index b236cc449..33e64708d 100644 --- a/numpy/lib/arraypad.py +++ b/numpy/lib/arraypad.py @@ -7,6 +7,7 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.core.overrides import array_function_dispatch +from numpy.lib.index_tricks import ndindex __all__ = ['pad'] @@ -16,50 +17,7 @@ __all__ = ['pad'] # Private utility functions. -def _arange_ndarray(arr, shape, axis, reverse=False): - """ - Create an ndarray of `shape` with increments along specified `axis` - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - shape : tuple of ints - Shape of desired array. Should be equivalent to `arr.shape` except - `shape[axis]` which may have any positive value. - axis : int - Axis to increment along. - reverse : bool - If False, increment in a positive fashion from 1 to `shape[axis]`, - inclusive. If True, the bounds are the same but the order reversed. - - Returns - ------- - padarr : ndarray - Output array sized to pad `arr` along `axis`, with linear range from - 1 to `shape[axis]` along specified `axis`. - - Notes - ----- - The range is deliberately 1-indexed for this specific use case. Think of - this algorithm as broadcasting `np.arange` to a single `axis` of an - arbitrarily shaped ndarray. - - """ - initshape = tuple(1 if i != axis else shape[axis] - for (i, x) in enumerate(arr.shape)) - if not reverse: - padarr = np.arange(1, shape[axis] + 1) - else: - padarr = np.arange(shape[axis], 0, -1) - padarr = padarr.reshape(initshape) - for i, dim in enumerate(shape): - if padarr.shape[i] != dim: - padarr = padarr.repeat(dim, axis=i) - return padarr - - -def _round_ifneeded(arr, dtype): +def _round_if_needed(arr, dtype): """ Rounds arr inplace if destination dtype is integer. @@ -69,821 +27,433 @@ def _round_ifneeded(arr, dtype): Input array. dtype : dtype The dtype of the destination array. - """ if np.issubdtype(dtype, np.integer): arr.round(out=arr) -def _slice_at_axis(shape, sl, axis): +def _slice_at_axis(sl, axis): """ - Construct a slice tuple the length of shape, with sl at the specified axis - """ - slice_tup = (slice(None),) - return slice_tup * axis + (sl,) + slice_tup * (len(shape) - axis - 1) - - -def _slice_first(shape, n, axis): - """ Construct a slice tuple to take the first n elements along axis """ - return _slice_at_axis(shape, slice(0, n), axis=axis) - - -def _slice_last(shape, n, axis): - """ Construct a slice tuple to take the last n elements along axis """ - dim = shape[axis] # doing this explicitly makes n=0 work - return _slice_at_axis(shape, slice(dim - n, dim), axis=axis) - - -def _do_prepend(arr, pad_chunk, axis): - return np.concatenate( - (pad_chunk.astype(arr.dtype, copy=False), arr), axis=axis) - - -def _do_append(arr, pad_chunk, axis): - return np.concatenate( - (arr, pad_chunk.astype(arr.dtype, copy=False)), axis=axis) - - -def _prepend_const(arr, pad_amt, val, axis=-1): - """ - Prepend constant `val` along `axis` of `arr`. + Construct tuple of slices to slice an array in the given dimension. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - val : scalar - Constant value to use. For best results should be of type `arr.dtype`; - if not `arr.dtype` will be cast to `arr.dtype`. + sl : slice + The slice for the given dimension. axis : int - Axis along which to pad `arr`. + The axis to which `sl` is applied. All other dimensions are left + "unsliced". Returns ------- - padarr : ndarray - Output array, with `pad_amt` constant `val` prepended along `axis`. - - """ - if pad_amt == 0: - return arr - padshape = tuple(x if i != axis else pad_amt - for (i, x) in enumerate(arr.shape)) - return _do_prepend(arr, np.full(padshape, val, dtype=arr.dtype), axis) - + sl : tuple of slices + A tuple with slices matching `shape` in length. -def _append_const(arr, pad_amt, val, axis=-1): - """ - Append constant `val` along `axis` of `arr`. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - val : scalar - Constant value to use. For best results should be of type `arr.dtype`; - if not `arr.dtype` will be cast to `arr.dtype`. - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` constant `val` appended along `axis`. - - """ - if pad_amt == 0: - return arr - padshape = tuple(x if i != axis else pad_amt - for (i, x) in enumerate(arr.shape)) - return _do_append(arr, np.full(padshape, val, dtype=arr.dtype), axis) - - - -def _prepend_edge(arr, pad_amt, axis=-1): + Examples + -------- + >>> _slice_at_axis(slice(None, 3, -1), 1) + (slice(None, None, None), slice(None, 3, -1), (...,)) """ - Prepend `pad_amt` to `arr` along `axis` by extending edge values. + return (slice(None),) * axis + (sl,) + (...,) - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, extended by `pad_amt` edge values appended along `axis`. +def _view_roi(array, original_area_slice, axis): """ - if pad_amt == 0: - return arr + Get a view of the current region of interest during iterative padding. - edge_slice = _slice_first(arr.shape, 1, axis=axis) - edge_arr = arr[edge_slice] - return _do_prepend(arr, edge_arr.repeat(pad_amt, axis=axis), axis) - - -def _append_edge(arr, pad_amt, axis=-1): - """ - Append `pad_amt` to `arr` along `axis` by extending edge values. + When padding multiple dimensions iteratively corner values are + unnecessarily overwritten multiple times. This function reduces the + working area for the first dimensions so that corners are excluded. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. + array : ndarray + The array with the region of interest. + original_area_slice : tuple of slices + Denotes the area with original values of the unpadded array. axis : int - Axis along which to pad `arr`. + The currently padded dimension assuming that `axis` is padded before + `axis` + 1. Returns ------- - padarr : ndarray - Output array, extended by `pad_amt` edge values prepended along - `axis`. - + roi : ndarray + The region of interest of the original `array`. """ - if pad_amt == 0: - return arr + axis += 1 + sl = (slice(None),) * axis + original_area_slice[axis:] + return array[sl] - edge_slice = _slice_last(arr.shape, 1, axis=axis) - edge_arr = arr[edge_slice] - return _do_append(arr, edge_arr.repeat(pad_amt, axis=axis), axis) - -def _prepend_ramp(arr, pad_amt, end, axis=-1): +def _pad_simple(array, pad_width, fill_value=None): """ - Prepend linear ramp along `axis`. + Pad array on all sides with either a single value or undefined values. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - end : scalar - Constal value to use. For best results should be of type `arr.dtype`; - if not `arr.dtype` will be cast to `arr.dtype`. - axis : int - Axis along which to pad `arr`. + array : ndarray + Array to grow. + pad_width : sequence of tuple[int, int] + Pad width on both sides for each dimension in `arr`. + fill_value : scalar, optional + If provided the padded area is filled with this value, otherwise + the pad area left undefined. Returns ------- - padarr : ndarray - Output array, with `pad_amt` values prepended along `axis`. The - prepended region ramps linearly from the edge value to `end`. - + padded : ndarray + The padded array with the same dtype as`array`. Its order will default + to C-style if `array` is not F-contiguous. + original_area_slice : tuple + A tuple of slices pointing to the area of the original array. """ - if pad_amt == 0: - return arr - - # Generate shape for final concatenated array - padshape = tuple(x if i != axis else pad_amt - for (i, x) in enumerate(arr.shape)) + # Allocate grown array + new_shape = tuple( + left + size + right + for size, (left, right) in zip(array.shape, pad_width) + ) + order = 'F' if array.flags.fnc else 'C' # Fortran and not also C-order + padded = np.empty(new_shape, dtype=array.dtype, order=order) - # Generate an n-dimensional array incrementing along `axis` - ramp_arr = _arange_ndarray(arr, padshape, axis, - reverse=True).astype(np.float64) + if fill_value is not None: + padded.fill(fill_value) - # Appropriate slicing to extract n-dimensional edge along `axis` - edge_slice = _slice_first(arr.shape, 1, axis=axis) + # Copy old array into correct space + original_area_slice = tuple( + slice(left, left + size) + for size, (left, right) in zip(array.shape, pad_width) + ) + padded[original_area_slice] = array - # Extract edge, and extend along `axis` - edge_pad = arr[edge_slice].repeat(pad_amt, axis) + return padded, original_area_slice - # Linear ramp - slope = (end - edge_pad) / float(pad_amt) - ramp_arr = ramp_arr * slope - ramp_arr += edge_pad - _round_ifneeded(ramp_arr, arr.dtype) - # Ramp values will most likely be float, cast them to the same type as arr - return _do_prepend(arr, ramp_arr, axis) - - -def _append_ramp(arr, pad_amt, end, axis=-1): +def _set_pad_area(padded, axis, width_pair, value_pair): """ - Append linear ramp along `axis`. + Set empty-padded area in given dimension. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - end : scalar - Constal value to use. For best results should be of type `arr.dtype`; - if not `arr.dtype` will be cast to `arr.dtype`. + padded : ndarray + Array with the pad area which is modified inplace. axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - appended region ramps linearly from the edge value to `end`. - + Dimension with the pad area to set. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. + value_pair : tuple of scalars or ndarrays + Values inserted into the pad area on each side. It must match or be + broadcastable to the shape of `arr`. """ - if pad_amt == 0: - return arr - - # Generate shape for final concatenated array - padshape = tuple(x if i != axis else pad_amt - for (i, x) in enumerate(arr.shape)) - - # Generate an n-dimensional array incrementing along `axis` - ramp_arr = _arange_ndarray(arr, padshape, axis, - reverse=False).astype(np.float64) - - # Slice a chunk from the edge to calculate stats on - edge_slice = _slice_last(arr.shape, 1, axis=axis) - - # Extract edge, and extend along `axis` - edge_pad = arr[edge_slice].repeat(pad_amt, axis) - - # Linear ramp - slope = (end - edge_pad) / float(pad_amt) - ramp_arr = ramp_arr * slope - ramp_arr += edge_pad - _round_ifneeded(ramp_arr, arr.dtype) + left_slice = _slice_at_axis(slice(None, width_pair[0]), axis) + padded[left_slice] = value_pair[0] - # Ramp values will most likely be float, cast them to the same type as arr - return _do_append(arr, ramp_arr, axis) + right_slice = _slice_at_axis( + slice(padded.shape[axis] - width_pair[1], None), axis) + padded[right_slice] = value_pair[1] -def _prepend_max(arr, pad_amt, num, axis=-1): +def _get_edges(padded, axis, width_pair): """ - Prepend `pad_amt` maximum values along `axis`. + Retrieve edge values from empty-padded array in given dimension. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - num : int - Depth into `arr` along `axis` to calculate maximum. - Range: [1, `arr.shape[axis]`] or None (entire axis) + padded : ndarray + Empty-padded array. axis : int - Axis along which to pad `arr`. + Dimension in which the edges are considered. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. Returns ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - prepended region is the maximum of the first `num` values along - `axis`. - + left_edge, right_edge : ndarray + Edge values of the valid area in `padded` in the given dimension. Its + shape will always match `padded` except for the dimension given by + `axis` which will have a length of 1. """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _prepend_edge(arr, pad_amt, axis) + left_index = width_pair[0] + left_slice = _slice_at_axis(slice(left_index, left_index + 1), axis) + left_edge = padded[left_slice] - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None + right_index = padded.shape[axis] - width_pair[1] + right_slice = _slice_at_axis(slice(right_index - 1, right_index), axis) + right_edge = padded[right_slice] - # Slice a chunk from the edge to calculate stats on - max_slice = _slice_first(arr.shape, num, axis=axis) + return left_edge, right_edge - # Extract slice, calculate max - max_chunk = arr[max_slice].max(axis=axis, keepdims=True) - # Concatenate `arr` with `max_chunk`, extended along `axis` by `pad_amt` - return _do_prepend(arr, max_chunk.repeat(pad_amt, axis=axis), axis) - - -def _append_max(arr, pad_amt, num, axis=-1): +def _get_linear_ramps(padded, axis, width_pair, end_value_pair): """ - Pad one `axis` of `arr` with the maximum of the last `num` elements. + Construct linear ramps for empty-padded array in given dimension. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - num : int - Depth into `arr` along `axis` to calculate maximum. - Range: [1, `arr.shape[axis]`] or None (entire axis) + padded : ndarray + Empty-padded array. axis : int - Axis along which to pad `arr`. + Dimension in which the ramps are constructed. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. + end_value_pair : (scalar, scalar) + End values for the linear ramps which form the edge of the fully padded + array. These values are included in the linear ramps. Returns ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - appended region is the maximum of the final `num` values along `axis`. - + left_ramp, right_ramp : ndarray + Linear ramps to set on both sides of `padded`. """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _append_edge(arr, pad_amt, axis) + edge_pair = _get_edges(padded, axis, width_pair) - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - if num is not None: - max_slice = _slice_last(arr.shape, num, axis=axis) - else: - max_slice = tuple(slice(None) for x in arr.shape) + left_ramp = np.linspace( + start=end_value_pair[0], + stop=edge_pair[0].squeeze(axis), # Dimensions is replaced by linspace + num=width_pair[0], + endpoint=False, + dtype=padded.dtype, + axis=axis, + ) - # Extract slice, calculate max - max_chunk = arr[max_slice].max(axis=axis, keepdims=True) + right_ramp = np.linspace( + start=end_value_pair[1], + stop=edge_pair[1].squeeze(axis), # Dimension is replaced by linspace + num=width_pair[1], + endpoint=False, + dtype=padded.dtype, + axis=axis, + ) + # Reverse linear space in appropriate dimension + right_ramp = right_ramp[_slice_at_axis(slice(None, None, -1), axis)] - # Concatenate `arr` with `max_chunk`, extended along `axis` by `pad_amt` - return _do_append(arr, max_chunk.repeat(pad_amt, axis=axis), axis) + return left_ramp, right_ramp -def _prepend_mean(arr, pad_amt, num, axis=-1): +def _get_stats(padded, axis, width_pair, length_pair, stat_func): """ - Prepend `pad_amt` mean values along `axis`. + Calculate statistic for the empty-padded array in given dimnsion. Parameters ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - num : int - Depth into `arr` along `axis` to calculate mean. - Range: [1, `arr.shape[axis]`] or None (entire axis) + padded : ndarray + Empty-padded array. axis : int - Axis along which to pad `arr`. + Dimension in which the statistic is calculated. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. + length_pair : 2-element sequence of None or int + Gives the number of values in valid area from each side that is + taken into account when calculating the statistic. If None the entire + valid area in `padded` is considered. + stat_func : function + Function to compute statistic. The expected signature is + ``stat_func(x: ndarray, axis: int, keepdims: bool) -> ndarray``. Returns ------- - padarr : ndarray - Output array, with `pad_amt` values prepended along `axis`. The - prepended region is the mean of the first `num` values along `axis`. - + left_stat, right_stat : ndarray + Calculated statistic for both sides of `padded`. """ - if pad_amt == 0: - return arr + # Calculate indices of the edges of the area with original values + left_index = width_pair[0] + right_index = padded.shape[axis] - width_pair[1] + # as well as its length + max_length = right_index - left_index - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _prepend_edge(arr, pad_amt, axis) + # Limit stat_lengths to max_length + left_length, right_length = length_pair + if left_length is None or max_length < left_length: + left_length = max_length + if right_length is None or max_length < right_length: + right_length = max_length - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None + if (left_length == 0 or right_length == 0) \ + and stat_func in {np.amax, np.amin}: + # amax and amin can't operate on an emtpy array, + # raise a more descriptive warning here instead of the default one + raise ValueError("stat_length of 0 yields no value for padding") - # Slice a chunk from the edge to calculate stats on - mean_slice = _slice_first(arr.shape, num, axis=axis) + # Calculate statistic for the left side + left_slice = _slice_at_axis( + slice(left_index, left_index + left_length), axis) + left_chunk = padded[left_slice] + left_stat = stat_func(left_chunk, axis=axis, keepdims=True) + _round_if_needed(left_stat, padded.dtype) - # Extract slice, calculate mean - mean_chunk = arr[mean_slice].mean(axis, keepdims=True) - _round_ifneeded(mean_chunk, arr.dtype) + if left_length == right_length == max_length: + # return early as right_stat must be identical to left_stat + return left_stat, left_stat - # Concatenate `arr` with `mean_chunk`, extended along `axis` by `pad_amt` - return _do_prepend(arr, mean_chunk.repeat(pad_amt, axis), axis=axis) + # Calculate statistic for the right side + right_slice = _slice_at_axis( + slice(right_index - right_length, right_index), axis) + right_chunk = padded[right_slice] + right_stat = stat_func(right_chunk, axis=axis, keepdims=True) + _round_if_needed(right_stat, padded.dtype) + return left_stat, right_stat -def _append_mean(arr, pad_amt, num, axis=-1): - """ - Append `pad_amt` mean values along `axis`. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - num : int - Depth into `arr` along `axis` to calculate mean. - Range: [1, `arr.shape[axis]`] or None (entire axis) - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - appended region is the maximum of the final `num` values along `axis`. - - """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _append_edge(arr, pad_amt, axis) - - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - if num is not None: - mean_slice = _slice_last(arr.shape, num, axis=axis) - else: - mean_slice = tuple(slice(None) for x in arr.shape) - - # Extract slice, calculate mean - mean_chunk = arr[mean_slice].mean(axis=axis, keepdims=True) - _round_ifneeded(mean_chunk, arr.dtype) - # Concatenate `arr` with `mean_chunk`, extended along `axis` by `pad_amt` - return _do_append(arr, mean_chunk.repeat(pad_amt, axis), axis=axis) - - -def _prepend_med(arr, pad_amt, num, axis=-1): +def _set_reflect_both(padded, axis, width_pair, method, include_edge=False): """ - Prepend `pad_amt` median values along `axis`. + Pad `axis` of `arr` with reflection. Parameters ---------- - arr : ndarray + padded : ndarray Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - num : int - Depth into `arr` along `axis` to calculate median. - Range: [1, `arr.shape[axis]`] or None (entire axis) axis : int Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values prepended along `axis`. The - prepended region is the median of the first `num` values along `axis`. - - """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _prepend_edge(arr, pad_amt, axis) - - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - med_slice = _slice_first(arr.shape, num, axis=axis) - - # Extract slice, calculate median - med_chunk = np.median(arr[med_slice], axis=axis, keepdims=True) - _round_ifneeded(med_chunk, arr.dtype) - - # Concatenate `arr` with `med_chunk`, extended along `axis` by `pad_amt` - return _do_prepend(arr, med_chunk.repeat(pad_amt, axis), axis=axis) - - -def _append_med(arr, pad_amt, num, axis=-1): - """ - Append `pad_amt` median values along `axis`. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - num : int - Depth into `arr` along `axis` to calculate median. - Range: [1, `arr.shape[axis]`] or None (entire axis) - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - appended region is the median of the final `num` values along `axis`. - - """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _append_edge(arr, pad_amt, axis) - - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - if num is not None: - med_slice = _slice_last(arr.shape, num, axis=axis) - else: - med_slice = tuple(slice(None) for x in arr.shape) - - # Extract slice, calculate median - med_chunk = np.median(arr[med_slice], axis=axis, keepdims=True) - _round_ifneeded(med_chunk, arr.dtype) - - # Concatenate `arr` with `med_chunk`, extended along `axis` by `pad_amt` - return _do_append(arr, med_chunk.repeat(pad_amt, axis), axis=axis) - - -def _prepend_min(arr, pad_amt, num, axis=-1): - """ - Prepend `pad_amt` minimum values along `axis`. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to prepend. - num : int - Depth into `arr` along `axis` to calculate minimum. - Range: [1, `arr.shape[axis]`] or None (entire axis) - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values prepended along `axis`. The - prepended region is the minimum of the first `num` values along - `axis`. - - """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _prepend_edge(arr, pad_amt, axis) - - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - min_slice = _slice_first(arr.shape, num, axis=axis) - - # Extract slice, calculate min - min_chunk = arr[min_slice].min(axis=axis, keepdims=True) - - # Concatenate `arr` with `min_chunk`, extended along `axis` by `pad_amt` - return _do_prepend(arr, min_chunk.repeat(pad_amt, axis), axis=axis) - - -def _append_min(arr, pad_amt, num, axis=-1): - """ - Append `pad_amt` median values along `axis`. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : int - Amount of padding to append. - num : int - Depth into `arr` along `axis` to calculate minimum. - Range: [1, `arr.shape[axis]`] or None (entire axis) - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt` values appended along `axis`. The - appended region is the minimum of the final `num` values along `axis`. - - """ - if pad_amt == 0: - return arr - - # Equivalent to edge padding for single value, so do that instead - if num == 1: - return _append_edge(arr, pad_amt, axis) - - # Use entire array if `num` is too large - if num is not None: - if num >= arr.shape[axis]: - num = None - - # Slice a chunk from the edge to calculate stats on - if num is not None: - min_slice = _slice_last(arr.shape, num, axis=axis) - else: - min_slice = tuple(slice(None) for x in arr.shape) - - # Extract slice, calculate min - min_chunk = arr[min_slice].min(axis=axis, keepdims=True) - - # Concatenate `arr` with `min_chunk`, extended along `axis` by `pad_amt` - return _do_append(arr, min_chunk.repeat(pad_amt, axis), axis=axis) - - -def _pad_ref(arr, pad_amt, method, axis=-1): - """ - Pad `axis` of `arr` by reflection. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. - pad_amt : tuple of ints, length 2 - Padding to (prepend, append) along `axis`. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. method : str Controls method of reflection; options are 'even' or 'odd'. - axis : int - Axis along which to pad `arr`. + include_edge : bool + If true, edge value is included in reflection, otherwise the edge + value forms the symmetric axis to the reflection. Returns ------- - padarr : ndarray - Output array, with `pad_amt[0]` values prepended and `pad_amt[1]` - values appended along `axis`. Both regions are padded with reflected - values from the original array. - - Notes - ----- - This algorithm does not pad with repetition, i.e. the edges are not - repeated in the reflection. For that behavior, use `mode='symmetric'`. - - The modes 'reflect', 'symmetric', and 'wrap' must be padded with a - single function, lest the indexing tricks in non-integer multiples of the - original shape would violate repetition in the final iteration. - - """ - # Implicit booleanness to test for zero (or None) in any scalar type - if pad_amt[0] == 0 and pad_amt[1] == 0: - return arr - - ########################################################################## - # Prepended region - - # Slice off a reverse indexed chunk from near edge to pad `arr` before - ref_slice = _slice_at_axis(arr.shape, slice(pad_amt[0], 0, -1), axis=axis) - - ref_chunk1 = arr[ref_slice] - - # Memory/computationally more expensive, only do this if `method='odd'` - if 'odd' in method and pad_amt[0] > 0: - edge_slice1 = _slice_first(arr.shape, 1, axis=axis) - edge_chunk = arr[edge_slice1] - ref_chunk1 = 2 * edge_chunk - ref_chunk1 - del edge_chunk - - ########################################################################## - # Appended region - - # Slice off a reverse indexed chunk from far edge to pad `arr` after - start = arr.shape[axis] - pad_amt[1] - 1 - end = arr.shape[axis] - 1 - ref_slice = _slice_at_axis(arr.shape, slice(start, end), axis=axis) - rev_idx = _slice_at_axis(arr.shape, slice(None, None, -1), axis=axis) - ref_chunk2 = arr[ref_slice][rev_idx] - - if 'odd' in method: - edge_slice2 = _slice_last(arr.shape, 1, axis=axis) - edge_chunk = arr[edge_slice2] - ref_chunk2 = 2 * edge_chunk - ref_chunk2 - del edge_chunk - - # Concatenate `arr` with both chunks, extending along `axis` - return np.concatenate((ref_chunk1, arr, ref_chunk2), axis=axis) - - -def _pad_sym(arr, pad_amt, method, axis=-1): - """ - Pad `axis` of `arr` by symmetry. - - Parameters - ---------- - arr : ndarray - Input array of arbitrary shape. pad_amt : tuple of ints, length 2 - Padding to (prepend, append) along `axis`. - method : str - Controls method of symmetry; options are 'even' or 'odd'. - axis : int - Axis along which to pad `arr`. - - Returns - ------- - padarr : ndarray - Output array, with `pad_amt[0]` values prepended and `pad_amt[1]` - values appended along `axis`. Both regions are padded with symmetric - values from the original array. - - Notes - ----- - This algorithm DOES pad with repetition, i.e. the edges are repeated. - For padding without repeated edges, use `mode='reflect'`. - - The modes 'reflect', 'symmetric', and 'wrap' must be padded with a - single function, lest the indexing tricks in non-integer multiples of the - original shape would violate repetition in the final iteration. - + New index positions of padding to do along the `axis`. If these are + both 0, padding is done in this dimension. """ - # Implicit booleanness to test for zero (or None) in any scalar type - if pad_amt[0] == 0 and pad_amt[1] == 0: - return arr - - ########################################################################## - # Prepended region - - # Slice off a reverse indexed chunk from near edge to pad `arr` before - sym_slice = _slice_first(arr.shape, pad_amt[0], axis=axis) - rev_idx = _slice_at_axis(arr.shape, slice(None, None, -1), axis=axis) - sym_chunk1 = arr[sym_slice][rev_idx] - - # Memory/computationally more expensive, only do this if `method='odd'` - if 'odd' in method and pad_amt[0] > 0: - edge_slice1 = _slice_first(arr.shape, 1, axis=axis) - edge_chunk = arr[edge_slice1] - sym_chunk1 = 2 * edge_chunk - sym_chunk1 - del edge_chunk - - ########################################################################## - # Appended region - - # Slice off a reverse indexed chunk from far edge to pad `arr` after - sym_slice = _slice_last(arr.shape, pad_amt[1], axis=axis) - sym_chunk2 = arr[sym_slice][rev_idx] - - if 'odd' in method: - edge_slice2 = _slice_last(arr.shape, 1, axis=axis) - edge_chunk = arr[edge_slice2] - sym_chunk2 = 2 * edge_chunk - sym_chunk2 - del edge_chunk - - # Concatenate `arr` with both chunks, extending along `axis` - return np.concatenate((sym_chunk1, arr, sym_chunk2), axis=axis) - + left_pad, right_pad = width_pair + old_length = padded.shape[axis] - right_pad - left_pad -def _pad_wrap(arr, pad_amt, axis=-1): - """ - Pad `axis` of `arr` via wrapping. + if include_edge: + # Edge is included, we need to offset the pad amount by 1 + edge_offset = 1 + else: + edge_offset = 0 # Edge is not included, no need to offset pad amount + old_length -= 1 # but must be omitted from the chunk + + if left_pad > 0: + # Pad with reflected values on left side: + # First limit chunk size which can't be larger than pad area + chunk_length = min(old_length, left_pad) + # Slice right to left, stop on or next to edge, start relative to stop + stop = left_pad - edge_offset + start = stop + chunk_length + left_slice = _slice_at_axis(slice(start, stop, -1), axis) + left_chunk = padded[left_slice] + + if method == "odd": + # Negate chunk and align with edge + edge_slice = _slice_at_axis(slice(left_pad, left_pad + 1), axis) + left_chunk = 2 * padded[edge_slice] - left_chunk + + # Insert chunk into padded area + start = left_pad - chunk_length + stop = left_pad + pad_area = _slice_at_axis(slice(start, stop), axis) + padded[pad_area] = left_chunk + # Adjust pointer to left edge for next iteration + left_pad -= chunk_length + + if right_pad > 0: + # Pad with reflected values on right side: + # First limit chunk size which can't be larger than pad area + chunk_length = min(old_length, right_pad) + # Slice right to left, start on or next to edge, stop relative to start + start = -right_pad + edge_offset - 2 + stop = start - chunk_length + right_slice = _slice_at_axis(slice(start, stop, -1), axis) + right_chunk = padded[right_slice] + + if method == "odd": + # Negate chunk and align with edge + edge_slice = _slice_at_axis( + slice(-right_pad - 1, -right_pad), axis) + right_chunk = 2 * padded[edge_slice] - right_chunk + + # Insert chunk into padded area + start = padded.shape[axis] - right_pad + stop = start + chunk_length + pad_area = _slice_at_axis(slice(start, stop), axis) + padded[pad_area] = right_chunk + # Adjust pointer to right edge for next iteration + right_pad -= chunk_length + + return left_pad, right_pad + + +def _set_wrap_both(padded, axis, width_pair): + """ + Pad `axis` of `arr` with wrapped values. Parameters ---------- - arr : ndarray + padded : ndarray Input array of arbitrary shape. - pad_amt : tuple of ints, length 2 - Padding to (prepend, append) along `axis`. axis : int Axis along which to pad `arr`. + width_pair : (int, int) + Pair of widths that mark the pad area on both sides in the given + dimension. Returns ------- - padarr : ndarray - Output array, with `pad_amt[0]` values prepended and `pad_amt[1]` - values appended along `axis`. Both regions are padded wrapped values - from the opposite end of `axis`. - - Notes - ----- - This method of padding is also known as 'tile' or 'tiling'. - - The modes 'reflect', 'symmetric', and 'wrap' must be padded with a - single function, lest the indexing tricks in non-integer multiples of the - original shape would violate repetition in the final iteration. - - """ - # Implicit booleanness to test for zero (or None) in any scalar type - if pad_amt[0] == 0 and pad_amt[1] == 0: - return arr - - ########################################################################## - # Prepended region - - # Slice off a reverse indexed chunk from near edge to pad `arr` before - wrap_slice = _slice_last(arr.shape, pad_amt[0], axis=axis) - wrap_chunk1 = arr[wrap_slice] - - ########################################################################## - # Appended region - - # Slice off a reverse indexed chunk from far edge to pad `arr` after - wrap_slice = _slice_first(arr.shape, pad_amt[1], axis=axis) - wrap_chunk2 = arr[wrap_slice] - - # Concatenate `arr` with both chunks, extending along `axis` - return np.concatenate((wrap_chunk1, arr, wrap_chunk2), axis=axis) + pad_amt : tuple of ints, length 2 + New index positions of padding to do along the `axis`. If these are + both 0, padding is done in this dimension. + """ + left_pad, right_pad = width_pair + period = padded.shape[axis] - right_pad - left_pad + + # If the current dimension of `arr` doesn't contain enough valid values + # (not part of the undefined pad area) we need to pad multiple times. + # Each time the pad area shrinks on both sides which is communicated with + # these variables. + new_left_pad = 0 + new_right_pad = 0 + + if left_pad > 0: + # Pad with wrapped values on left side + # First slice chunk from right side of the non-pad area. + # Use min(period, left_pad) to ensure that chunk is not larger than + # pad area + right_slice = _slice_at_axis( + slice(-right_pad - min(period, left_pad), + -right_pad if right_pad != 0 else None), + axis + ) + right_chunk = padded[right_slice] + + if left_pad > period: + # Chunk is smaller than pad area + pad_area = _slice_at_axis(slice(left_pad - period, left_pad), axis) + new_left_pad = left_pad - period + else: + # Chunk matches pad area + pad_area = _slice_at_axis(slice(None, left_pad), axis) + padded[pad_area] = right_chunk + + if right_pad > 0: + # Pad with wrapped values on right side + # First slice chunk from left side of the non-pad area. + # Use min(period, right_pad) to ensure that chunk is not larger than + # pad area + left_slice = _slice_at_axis( + slice(left_pad, left_pad + min(period, right_pad),), axis) + left_chunk = padded[left_slice] + + if right_pad > period: + # Chunk is smaller than pad area + pad_area = _slice_at_axis( + slice(-right_pad, -right_pad + period), axis) + new_right_pad = right_pad - period + else: + # Chunk matches pad area + pad_area = _slice_at_axis(slice(-right_pad, None), axis) + padded[pad_area] = left_chunk + + return new_left_pad, new_right_pad def _as_pairs(x, ndim, as_index=False): @@ -953,23 +523,23 @@ def _as_pairs(x, ndim, as_index=False): return np.broadcast_to(x, (ndim, 2)).tolist() -############################################################################### -# Public functions +def _pad_dispatcher(array, pad_width, mode=None, **kwargs): + return (array,) -def _pad_dispatcher(array, pad_width, mode, **kwargs): - return (array,) +############################################################################### +# Public functions @array_function_dispatch(_pad_dispatcher, module='numpy') -def pad(array, pad_width, mode, **kwargs): +def pad(array, pad_width, mode='constant', **kwargs): """ - Pads an array. + Pad an array. Parameters ---------- array : array_like of rank N - Input array + The array to pad. pad_width : {sequence, array_like, int} Number of values padded to the edges of each axis. ((before_1, after_1), ... (before_N, after_N)) unique pad widths @@ -977,10 +547,10 @@ def pad(array, pad_width, mode, **kwargs): ((before, after),) yields same before and after pad for each axis. (pad,) or int is a shortcut for before = after = pad width for all axes. - mode : str or function + mode : str or function, optional One of the following string values or a user supplied function. - 'constant' + 'constant' (default) Pads with a constant value. 'edge' Pads with the edge values of array. @@ -1010,6 +580,11 @@ def pad(array, pad_width, mode, **kwargs): Pads with the wrap of the vector along the axis. The first values are used to pad the end and the end values are used to pad the beginning. + 'empty' + Pads with undefined values. + + .. versionadded:: 1.17 + <function> Padding function, see Notes. stat_length : sequence or int, optional @@ -1026,31 +601,31 @@ def pad(array, pad_width, mode, **kwargs): length for all axes. Default is ``None``, to use the entire axis. - constant_values : sequence or int, optional + constant_values : sequence or scalar, optional Used in 'constant'. The values to set the padded values for each axis. - ((before_1, after_1), ... (before_N, after_N)) unique pad constants + ``((before_1, after_1), ... (before_N, after_N))`` unique pad constants for each axis. - ((before, after),) yields same before and after constants for each + ``((before, after),)`` yields same before and after constants for each axis. - (constant,) or int is a shortcut for before = after = constant for + ``(constant,)`` or ``constant`` is a shortcut for ``before = after = constant`` for all axes. Default is 0. - end_values : sequence or int, optional + end_values : sequence or scalar, optional Used in 'linear_ramp'. The values used for the ending value of the linear_ramp and that will form the edge of the padded array. - ((before_1, after_1), ... (before_N, after_N)) unique end values + ``((before_1, after_1), ... (before_N, after_N))`` unique end values for each axis. - ((before, after),) yields same before and after end values for each + ``((before, after),)`` yields same before and after end values for each axis. - (constant,) or int is a shortcut for before = after = end value for + ``(constant,)`` or ``constant`` is a shortcut for ``before = after = constant`` for all axes. Default is 0. @@ -1075,9 +650,8 @@ def pad(array, pad_width, mode, **kwargs): think about with a rank 2 array where the corners of the padded array are calculated by using padded values from the first axis. - The padding function, if used, should return a rank 1 array equal in - length to the vector argument with padded values replaced. It has the - following signature:: + The padding function, if used, should modify a rank 1 array in-place. It + has the following signature:: padding_func(vector, iaxis_pad_width, iaxis, kwargs) @@ -1085,7 +659,7 @@ def pad(array, pad_width, mode, **kwargs): vector : ndarray A rank 1 array already padded with zeros. Padded values are - vector[:pad_tuple[0]] and vector[-pad_tuple[1]:]. + vector[:iaxis_pad_width[0]] and vector[-iaxis_pad_width[1]:]. iaxis_pad_width : tuple A 2-tuple of ints, iaxis_pad_width[0] represents the number of values padded at the beginning of vector where @@ -1099,7 +673,7 @@ def pad(array, pad_width, mode, **kwargs): Examples -------- >>> a = [1, 2, 3, 4, 5] - >>> np.pad(a, (2,3), 'constant', constant_values=(4, 6)) + >>> np.pad(a, (2, 3), 'constant', constant_values=(4, 6)) array([4, 4, 1, ..., 6, 6, 6]) >>> np.pad(a, (2, 3), 'edge') @@ -1147,7 +721,6 @@ def pad(array, pad_width, mode, **kwargs): ... pad_value = kwargs.get('padder', 10) ... vector[:pad_width[0]] = pad_value ... vector[-pad_width[1]:] = pad_value - ... return vector >>> a = np.arange(6) >>> a = a.reshape((2, 3)) >>> np.pad(a, 2, pad_with) @@ -1165,15 +738,42 @@ def pad(array, pad_width, mode, **kwargs): [100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100]]) """ - if not np.asarray(pad_width).dtype.kind == 'i': + array = np.asarray(array) + pad_width = np.asarray(pad_width) + + if not pad_width.dtype.kind == 'i': raise TypeError('`pad_width` must be of integral type.') - narray = np.array(array) - pad_width = _as_pairs(pad_width, narray.ndim, as_index=True) + # Broadcast to shape (array.ndim, 2) + pad_width = _as_pairs(pad_width, array.ndim, as_index=True) + + if callable(mode): + # Old behavior: Use user-supplied function with np.apply_along_axis + function = mode + # Create a new zero padded array + padded, _ = _pad_simple(array, pad_width, fill_value=0) + # And apply along each axis + + for axis in range(padded.ndim): + # Iterate using ndindex as in apply_along_axis, but assuming that + # function operates inplace on the padded array. + + # view with the iteration axis at the end + view = np.moveaxis(padded, axis, -1) + + # compute indices for the iteration axes, and append a trailing + # ellipsis to prevent 0d arrays decaying to scalars (gh-8642) + inds = ndindex(view.shape[:-1]) + inds = (ind + (Ellipsis,) for ind in inds) + for ind in inds: + function(view[ind], pad_width[axis], axis, kwargs) - allowedkwargs = { + return padded + + # Make sure that no unsupported keywords were passed for the current mode + allowed_kwargs = { + 'empty': [], 'edge': [], 'wrap': [], 'constant': ['constant_values'], - 'edge': [], 'linear_ramp': ['end_values'], 'maximum': ['stat_length'], 'mean': ['stat_length'], @@ -1181,175 +781,101 @@ def pad(array, pad_width, mode, **kwargs): 'minimum': ['stat_length'], 'reflect': ['reflect_type'], 'symmetric': ['reflect_type'], - 'wrap': [], - } - - kwdefaults = { - 'stat_length': None, - 'constant_values': 0, - 'end_values': 0, - 'reflect_type': 'even', - } - - if isinstance(mode, np.compat.basestring): - # Make sure have allowed kwargs appropriate for mode - for key in kwargs: - if key not in allowedkwargs[mode]: - raise ValueError('%s keyword not in allowed keywords %s' % - (key, allowedkwargs[mode])) - - # Set kwarg defaults - for kw in allowedkwargs[mode]: - kwargs.setdefault(kw, kwdefaults[kw]) - - # Need to only normalize particular keywords. - for i in kwargs: - if i == 'stat_length': - kwargs[i] = _as_pairs(kwargs[i], narray.ndim, as_index=True) - if i in ['end_values', 'constant_values']: - kwargs[i] = _as_pairs(kwargs[i], narray.ndim) - else: - # Drop back to old, slower np.apply_along_axis mode for user-supplied - # vector function - function = mode - - # Create a new padded array - rank = list(range(narray.ndim)) - total_dim_increase = [np.sum(pad_width[i]) for i in rank] - offset_slices = tuple( - slice(pad_width[i][0], pad_width[i][0] + narray.shape[i]) - for i in rank) - new_shape = np.array(narray.shape) + total_dim_increase - newmat = np.zeros(new_shape, narray.dtype) - - # Insert the original array into the padded array - newmat[offset_slices] = narray - - # This is the core of pad ... - for iaxis in rank: - np.apply_along_axis(function, - iaxis, - newmat, - pad_width[iaxis], - iaxis, - kwargs) - return newmat - - # If we get here, use new padding method - newmat = narray.copy() - - # API preserved, but completely new algorithm which pads by building the - # entire block to pad before/after `arr` with in one step, for each axis. - if mode == 'constant': - for axis, ((pad_before, pad_after), (before_val, after_val)) \ - in enumerate(zip(pad_width, kwargs['constant_values'])): - newmat = _prepend_const(newmat, pad_before, before_val, axis) - newmat = _append_const(newmat, pad_after, after_val, axis) - - elif mode == 'edge': - for axis, (pad_before, pad_after) in enumerate(pad_width): - newmat = _prepend_edge(newmat, pad_before, axis) - newmat = _append_edge(newmat, pad_after, axis) - - elif mode == 'linear_ramp': - for axis, ((pad_before, pad_after), (before_val, after_val)) \ - in enumerate(zip(pad_width, kwargs['end_values'])): - newmat = _prepend_ramp(newmat, pad_before, before_val, axis) - newmat = _append_ramp(newmat, pad_after, after_val, axis) - - elif mode == 'maximum': - for axis, ((pad_before, pad_after), (chunk_before, chunk_after)) \ - in enumerate(zip(pad_width, kwargs['stat_length'])): - newmat = _prepend_max(newmat, pad_before, chunk_before, axis) - newmat = _append_max(newmat, pad_after, chunk_after, axis) - - elif mode == 'mean': - for axis, ((pad_before, pad_after), (chunk_before, chunk_after)) \ - in enumerate(zip(pad_width, kwargs['stat_length'])): - newmat = _prepend_mean(newmat, pad_before, chunk_before, axis) - newmat = _append_mean(newmat, pad_after, chunk_after, axis) - - elif mode == 'median': - for axis, ((pad_before, pad_after), (chunk_before, chunk_after)) \ - in enumerate(zip(pad_width, kwargs['stat_length'])): - newmat = _prepend_med(newmat, pad_before, chunk_before, axis) - newmat = _append_med(newmat, pad_after, chunk_after, axis) - - elif mode == 'minimum': - for axis, ((pad_before, pad_after), (chunk_before, chunk_after)) \ - in enumerate(zip(pad_width, kwargs['stat_length'])): - newmat = _prepend_min(newmat, pad_before, chunk_before, axis) - newmat = _append_min(newmat, pad_after, chunk_after, axis) - - elif mode == 'reflect': - for axis, (pad_before, pad_after) in enumerate(pad_width): - if narray.shape[axis] == 0: - # Axes with non-zero padding cannot be empty. - if pad_before > 0 or pad_after > 0: - raise ValueError("There aren't any elements to reflect" - " in axis {} of `array`".format(axis)) - # Skip zero padding on empty axes. - continue - - # Recursive padding along any axis where `pad_amt` is too large - # for indexing tricks. We can only safely pad the original axis - # length, to keep the period of the reflections consistent. - if ((pad_before > 0) or - (pad_after > 0)) and newmat.shape[axis] == 1: + } + try: + unsupported_kwargs = set(kwargs) - set(allowed_kwargs[mode]) + except KeyError: + raise ValueError("mode '{}' is not supported".format(mode)) + if unsupported_kwargs: + raise ValueError("unsupported keyword arguments for mode '{}': {}" + .format(mode, unsupported_kwargs)) + + stat_functions = {"maximum": np.amax, "minimum": np.amin, + "mean": np.mean, "median": np.median} + + # Create array with final shape and original values + # (padded area is undefined) + padded, original_area_slice = _pad_simple(array, pad_width) + # And prepare iteration over all dimensions + # (zipping may be more readable than using enumerate) + axes = range(padded.ndim) + + if mode == "constant": + values = kwargs.get("constant_values", 0) + values = _as_pairs(values, padded.ndim) + for axis, width_pair, value_pair in zip(axes, pad_width, values): + roi = _view_roi(padded, original_area_slice, axis) + _set_pad_area(roi, axis, width_pair, value_pair) + + elif mode == "empty": + pass # Do nothing as _pad_simple already returned the correct result + + elif array.size == 0: + # Only modes "constant" and "empty" can extend empty axes, all other + # modes depend on `array` not being empty + # -> ensure every empty axis is only "padded with 0" + for axis, width_pair in zip(axes, pad_width): + if array.shape[axis] == 0 and any(width_pair): + raise ValueError( + "can't extend empty axis {} using modes other than " + "'constant' or 'empty'".format(axis) + ) + # passed, don't need to do anything more as _pad_simple already + # returned the correct result + + elif mode == "edge": + for axis, width_pair in zip(axes, pad_width): + roi = _view_roi(padded, original_area_slice, axis) + edge_pair = _get_edges(roi, axis, width_pair) + _set_pad_area(roi, axis, width_pair, edge_pair) + + elif mode == "linear_ramp": + end_values = kwargs.get("end_values", 0) + end_values = _as_pairs(end_values, padded.ndim) + for axis, width_pair, value_pair in zip(axes, pad_width, end_values): + roi = _view_roi(padded, original_area_slice, axis) + ramp_pair = _get_linear_ramps(roi, axis, width_pair, value_pair) + _set_pad_area(roi, axis, width_pair, ramp_pair) + + elif mode in stat_functions: + func = stat_functions[mode] + length = kwargs.get("stat_length", None) + length = _as_pairs(length, padded.ndim, as_index=True) + for axis, width_pair, length_pair in zip(axes, pad_width, length): + roi = _view_roi(padded, original_area_slice, axis) + stat_pair = _get_stats(roi, axis, width_pair, length_pair, func) + _set_pad_area(roi, axis, width_pair, stat_pair) + + elif mode in {"reflect", "symmetric"}: + method = kwargs.get("reflect_type", "even") + include_edge = True if mode == "symmetric" else False + for axis, (left_index, right_index) in zip(axes, pad_width): + if array.shape[axis] == 1 and (left_index > 0 or right_index > 0): # Extending singleton dimension for 'reflect' is legacy # behavior; it really should raise an error. - newmat = _prepend_edge(newmat, pad_before, axis) - newmat = _append_edge(newmat, pad_after, axis) + edge_pair = _get_edges(padded, axis, (left_index, right_index)) + _set_pad_area( + padded, axis, (left_index, right_index), edge_pair) continue - method = kwargs['reflect_type'] - safe_pad = newmat.shape[axis] - 1 - while ((pad_before > safe_pad) or (pad_after > safe_pad)): - pad_iter_b = min(safe_pad, - safe_pad * (pad_before // safe_pad)) - pad_iter_a = min(safe_pad, safe_pad * (pad_after // safe_pad)) - newmat = _pad_ref(newmat, (pad_iter_b, - pad_iter_a), method, axis) - pad_before -= pad_iter_b - pad_after -= pad_iter_a - safe_pad += pad_iter_b + pad_iter_a - newmat = _pad_ref(newmat, (pad_before, pad_after), method, axis) - - elif mode == 'symmetric': - for axis, (pad_before, pad_after) in enumerate(pad_width): - # Recursive padding along any axis where `pad_amt` is too large - # for indexing tricks. We can only safely pad the original axis - # length, to keep the period of the reflections consistent. - method = kwargs['reflect_type'] - safe_pad = newmat.shape[axis] - while ((pad_before > safe_pad) or - (pad_after > safe_pad)): - pad_iter_b = min(safe_pad, - safe_pad * (pad_before // safe_pad)) - pad_iter_a = min(safe_pad, safe_pad * (pad_after // safe_pad)) - newmat = _pad_sym(newmat, (pad_iter_b, - pad_iter_a), method, axis) - pad_before -= pad_iter_b - pad_after -= pad_iter_a - safe_pad += pad_iter_b + pad_iter_a - newmat = _pad_sym(newmat, (pad_before, pad_after), method, axis) - - elif mode == 'wrap': - for axis, (pad_before, pad_after) in enumerate(pad_width): - # Recursive padding along any axis where `pad_amt` is too large - # for indexing tricks. We can only safely pad the original axis - # length, to keep the period of the reflections consistent. - safe_pad = newmat.shape[axis] - while ((pad_before > safe_pad) or - (pad_after > safe_pad)): - pad_iter_b = min(safe_pad, - safe_pad * (pad_before // safe_pad)) - pad_iter_a = min(safe_pad, safe_pad * (pad_after // safe_pad)) - newmat = _pad_wrap(newmat, (pad_iter_b, pad_iter_a), axis) - - pad_before -= pad_iter_b - pad_after -= pad_iter_a - safe_pad += pad_iter_b + pad_iter_a - newmat = _pad_wrap(newmat, (pad_before, pad_after), axis) - - return newmat + roi = _view_roi(padded, original_area_slice, axis) + while left_index > 0 or right_index > 0: + # Iteratively pad until dimension is filled with reflected + # values. This is necessary if the pad area is larger than + # the length of the original values in the current dimension. + left_index, right_index = _set_reflect_both( + roi, axis, (left_index, right_index), + method, include_edge + ) + + elif mode == "wrap": + for axis, (left_index, right_index) in zip(axes, pad_width): + roi = _view_roi(padded, original_area_slice, axis) + while left_index > 0 or right_index > 0: + # Iteratively pad until dimension is filled with wrapped + # values. This is necessary if the pad area is larger than + # the length of the original values in the current dimension. + left_index, right_index = _set_wrap_both( + roi, axis, (left_index, right_index)) + + return padded diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 558150e48..2309f7e42 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -94,8 +94,7 @@ def ediff1d(ary, to_end=None, to_begin=None): # force a 1d array ary = np.asanyarray(ary).ravel() - # we have unit tests enforcing - # propagation of the dtype of input + # enforce propagation of the dtype of input # ary to returned result dtype_req = ary.dtype @@ -106,23 +105,22 @@ def ediff1d(ary, to_end=None, to_begin=None): if to_begin is None: l_begin = 0 else: - to_begin = np.asanyarray(to_begin) - if not np.can_cast(to_begin, dtype_req): - raise TypeError("dtype of to_begin must be compatible " - "with input ary") - - to_begin = to_begin.ravel() + _to_begin = np.asanyarray(to_begin, dtype=dtype_req) + if not np.all(_to_begin == to_begin): + raise ValueError("cannot convert 'to_begin' to array with dtype " + "'%r' as required for input ary" % dtype_req) + to_begin = _to_begin.ravel() l_begin = len(to_begin) if to_end is None: l_end = 0 else: - to_end = np.asanyarray(to_end) - if not np.can_cast(to_end, dtype_req): - raise TypeError("dtype of to_end must be compatible " - "with input ary") - - to_end = to_end.ravel() + _to_end = np.asanyarray(to_end, dtype=dtype_req) + # check that casting has not overflowed + if not np.all(_to_end == to_end): + raise ValueError("cannot convert 'to_end' to array with dtype " + "'%r' as required for input ary" % dtype_req) + to_end = _to_end.ravel() l_end = len(to_end) # do the calculation in place and copy to_begin and to_end @@ -215,6 +213,7 @@ def unique(ar, return_index=False, return_inverse=False, ----- When an axis is specified the subarrays indexed by the axis are sorted. This is done by making the specified axis the first dimension of the array + (move the axis to the first dimension to keep the order of the other axes) and then flattening the subarrays in C order. The flattened subarrays are then viewed as a structured type with each element given a label, with the effect that we end up with a 1-D array of structured types that can be @@ -266,7 +265,7 @@ def unique(ar, return_index=False, return_inverse=False, # axis was specified and not None try: - ar = np.swapaxes(ar, axis, 0) + ar = np.moveaxis(ar, axis, 0) except np.AxisError: # this removes the "axis1" or "axis2" prefix from the error message raise np.AxisError(axis, ar.ndim) @@ -287,7 +286,7 @@ def unique(ar, return_index=False, return_inverse=False, def reshape_uniq(uniq): uniq = uniq.view(orig_dtype) uniq = uniq.reshape(-1, *orig_shape[1:]) - uniq = np.swapaxes(uniq, 0, axis) + uniq = np.moveaxis(uniq, 0, axis) return uniq output = _unique1d(consolidated, return_index, @@ -385,6 +384,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): To return the indices of the values common to the input arrays along with the intersected values: + >>> x = np.array([1, 1, 2, 3, 4]) >>> y = np.array([2, 1, 4, 6]) >>> xy, x_ind, y_ind = np.intersect1d(x, y, return_indices=True) diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py index 216687475..a011e52a9 100644 --- a/numpy/lib/financial.py +++ b/numpy/lib/financial.py @@ -12,6 +12,7 @@ otherwise stated. """ from __future__ import division, absolute_import, print_function +import warnings from decimal import Decimal import functools @@ -19,6 +20,10 @@ import numpy as np from numpy.core import overrides +_depmsg = ("numpy.{name} is deprecated and will be removed from NumPy 1.20. " + "Use numpy_financial.{name} instead " + "(https://pypi.org/project/numpy-financial/).") + array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') @@ -45,6 +50,8 @@ def _convert_when(when): def _fv_dispatcher(rate, nper, pmt, pv, when=None): + warnings.warn(_depmsg.format(name='fv'), + DeprecationWarning, stacklevel=3) return (rate, nper, pmt, pv) @@ -53,6 +60,12 @@ def fv(rate, nper, pmt, pv, when='end'): """ Compute the future value. + .. deprecated:: 1.18 + + `fv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a present value, `pv` * an interest `rate` compounded once per period, of which @@ -100,7 +113,9 @@ def fv(rate, nper, pmt, pv, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -109,6 +124,7 @@ def fv(rate, nper, pmt, pv, when='end'): http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula OpenDocument-formula-20090508.odt + Examples -------- What is the future value after 10 years of saving $100 now, with @@ -139,6 +155,8 @@ def fv(rate, nper, pmt, pv, when='end'): def _pmt_dispatcher(rate, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='pmt'), + DeprecationWarning, stacklevel=3) return (rate, nper, pv, fv) @@ -147,6 +165,12 @@ def pmt(rate, nper, pv, fv=0, when='end'): """ Compute the payment against loan principal plus interest. + .. deprecated:: 1.18 + + `pmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a present value, `pv` (e.g., an amount borrowed) * a future value, `fv` (e.g., 0) @@ -204,7 +228,9 @@ def pmt(rate, nper, pv, fv=0, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -237,6 +263,8 @@ def pmt(rate, nper, pv, fv=0, when='end'): def _nper_dispatcher(rate, pmt, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='nper'), + DeprecationWarning, stacklevel=3) return (rate, pmt, pv, fv) @@ -245,6 +273,12 @@ def nper(rate, pmt, pv, fv=0, when='end'): """ Compute the number of periodic payments. + .. deprecated:: 1.18 + + `nper` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + :class:`decimal.Decimal` type is not supported. Parameters @@ -270,6 +304,11 @@ def nper(rate, pmt, pv, fv=0, when='end'): fv + pv + pmt*nper = 0 + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + Examples -------- If you only had $150/month to pay towards the loan, how long would it take @@ -311,6 +350,8 @@ def nper(rate, pmt, pv, fv=0, when='end'): def _ipmt_dispatcher(rate, per, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='ipmt'), + DeprecationWarning, stacklevel=3) return (rate, per, nper, pv, fv) @@ -319,6 +360,12 @@ def ipmt(rate, per, nper, pv, fv=0, when='end'): """ Compute the interest portion of a payment. + .. deprecated:: 1.18 + + `ipmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : scalar or array_like of shape(M, ) @@ -354,6 +401,11 @@ def ipmt(rate, per, nper, pv, fv=0, when='end'): ``pmt = ppmt + ipmt`` + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + Examples -------- What is the amortization schedule for a 1 year loan of $2500 at @@ -422,6 +474,8 @@ def _rbl(rate, per, pmt, pv, when): def _ppmt_dispatcher(rate, per, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='ppmt'), + DeprecationWarning, stacklevel=3) return (rate, per, nper, pv, fv) @@ -430,6 +484,12 @@ def ppmt(rate, per, nper, pv, fv=0, when='end'): """ Compute the payment against loan principal. + .. deprecated:: 1.18 + + `ppmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : array_like @@ -450,12 +510,19 @@ def ppmt(rate, per, nper, pv, fv=0, when='end'): -------- pmt, pv, ipmt + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + """ total = pmt(rate, nper, pv, fv, when) return total - ipmt(rate, per, nper, pv, fv, when) def _pv_dispatcher(rate, nper, pmt, fv=None, when=None): + warnings.warn(_depmsg.format(name='pv'), + DeprecationWarning, stacklevel=3) return (rate, nper, nper, pv, fv) @@ -464,6 +531,12 @@ def pv(rate, nper, pmt, fv=0, when='end'): """ Compute the present value. + .. deprecated:: 1.18 + + `pv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a future value, `fv` * an interest `rate` compounded once per period, of which @@ -510,7 +583,9 @@ def pv(rate, nper, pmt, fv=0, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -567,6 +642,8 @@ def _g_div_gp(r, n, p, x, y, w): def _rate_dispatcher(nper, pmt, pv, fv, when=None, guess=None, tol=None, maxiter=None): + warnings.warn(_depmsg.format(name='rate'), + DeprecationWarning, stacklevel=3) return (nper, pmt, pv, fv) @@ -582,6 +659,12 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): """ Compute the rate of interest per period. + .. deprecated:: 1.18 + + `rate` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- nper : array_like @@ -612,13 +695,16 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): References ---------- - Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document - Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated - Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. - Organization for the Advancement of Structured Information Standards - (OASIS). Billerica, MA, USA. [ODT Document]. Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + Open Document Format for Office Applications (OpenDocument)v1.2, + Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, + Pre-Draft 12. Organization for the Advancement of Structured Information + Standards (OASIS). Billerica, MA, USA. [ODT Document]. + Available: + http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula + OpenDocument-formula-20090508.odt """ when = _convert_when(when) @@ -651,6 +737,8 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): def _irr_dispatcher(values): + warnings.warn(_depmsg.format(name='irr'), + DeprecationWarning, stacklevel=3) return (values,) @@ -659,6 +747,12 @@ def irr(values): """ Return the Internal Rate of Return (IRR). + .. deprecated:: 1.18 + + `irr` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + This is the "average" periodically compounded rate of return that gives a net present value of 0.0; for a more complete explanation, see Notes below. @@ -693,13 +787,15 @@ def irr(values): + \\frac{55}{(1+r)^3} + \\frac{20}{(1+r)^4} = 0 In general, for `values` :math:`= [v_0, v_1, ... v_M]`, - irr is the solution of the equation: [G]_ + irr is the solution of the equation: [2]_ .. math:: \\sum_{t=0}^M{\\frac{v_t}{(1+irr)^{t}}} = 0 References ---------- - .. [G] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., Addison-Wesley, 2003, pg. 348. Examples @@ -715,8 +811,6 @@ def irr(values): >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) 0.0886 - (Compare with the Example given for numpy.lib.financial.npv) - """ # `np.roots` call is why this function does not support Decimal type. # @@ -736,6 +830,8 @@ def irr(values): def _npv_dispatcher(rate, values): + warnings.warn(_depmsg.format(name='npv'), + DeprecationWarning, stacklevel=3) return (values,) @@ -744,6 +840,12 @@ def npv(rate, values): """ Returns the NPV (Net Present Value) of a cash flow series. + .. deprecated:: 1.18 + + `npv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : scalar @@ -763,23 +865,48 @@ def npv(rate, values): The NPV of the input cash flow series `values` at the discount `rate`. + Warnings + -------- + ``npv`` considers a series of cashflows starting in the present (t = 0). + NPV can also be defined with a series of future cashflows, paid at the + end, rather than the start, of each period. If future cashflows are used, + the first cashflow `values[0]` must be zeroed and added to the net + present value of the future cashflows. This is demonstrated in the + examples. + Notes ----- - Returns the result of: [G]_ + Returns the result of: [2]_ .. math :: \\sum_{t=0}^{M-1}{\\frac{values_t}{(1+rate)^{t}}} References ---------- - .. [G] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., Addison-Wesley, 2003, pg. 346. Examples -------- - >>> np.npv(0.281,[-100, 39, 59, 55, 20]) - -0.0084785916384548798 # may vary + Consider a potential project with an initial investment of $40 000 and + projected cashflows of $5 000, $8 000, $12 000 and $30 000 at the end of + each period discounted at a rate of 8% per period. To find the project's + net present value: + + >>> rate, cashflows = 0.08, [-40_000, 5_000, 8_000, 12_000, 30_000] + >>> np.npv(rate, cashflows).round(5) + 3065.22267 - (Compare with the Example given for numpy.lib.financial.irr) + It may be preferable to split the projected cashflow into an initial + investment and expected future cashflows. In this case, the value of + the initial cashflow is zero and the initial investment is later added + to the future cashflows net present value: + + >>> initial_cashflow = cashflows[0] + >>> cashflows[0] = 0 + >>> np.round(np.npv(rate, cashflows) + initial_cashflow, 5) + 3065.22267 """ values = np.asarray(values) @@ -787,6 +914,8 @@ def npv(rate, values): def _mirr_dispatcher(values, finance_rate, reinvest_rate): + warnings.warn(_depmsg.format(name='mirr'), + DeprecationWarning, stacklevel=3) return (values,) @@ -795,6 +924,12 @@ def mirr(values, finance_rate, reinvest_rate): """ Modified internal rate of return. + .. deprecated:: 1.18 + + `mirr` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- values : array_like @@ -811,6 +946,10 @@ def mirr(values, finance_rate, reinvest_rate): out : float Modified internal rate of return + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html """ values = np.asarray(values) n = values.size diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 10945e5e8..20e2e9c72 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -146,10 +146,17 @@ The description of the fourth element of the header therefore has become: "The next 4 bytes form a little-endian unsigned int: the length of the header data HEADER_LEN." +Format Version 3.0 +------------------ + +This version replaces the ASCII string (which in practice was latin1) with +a utf8-encoded string, so supports structured types with any unicode field +names. + Notes ----- The ``.npy`` format, including motivation for creating it and a comparison of -alternatives, is described in the `"npy-format" NEP +alternatives, is described in the `"npy-format" NEP <https://www.numpy.org/neps/nep-0001-npy-format.html>`_, however details have evolved with time and this document is more current. @@ -162,9 +169,11 @@ import io import warnings from numpy.lib.utils import safe_eval from numpy.compat import ( - asbytes, asstr, isfileobj, long, os_fspath + isfileobj, long, os_fspath, pickle ) -from numpy.core.numeric import pickle + + +__all__ = [] MAGIC_PREFIX = b'\x93NUMPY' @@ -174,10 +183,16 @@ BUFFER_SIZE = 2**18 # size of buffer for reading npz files in bytes # difference between version 1.0 and 2.0 is a 4 byte (I) header length # instead of 2 bytes (H) allowing storage of large structured arrays +_header_size_info = { + (1, 0): ('<H', 'latin1'), + (2, 0): ('<I', 'latin1'), + (3, 0): ('<I', 'utf8'), +} + def _check_version(version): - if version not in [(1, 0), (2, 0), None]: - msg = "we only support format version (1,0) and (2, 0), not %s" + if version not in [(1, 0), (2, 0), (3, 0), None]: + msg = "we only support format version (1,0), (2,0), and (3,0), not %s" raise ValueError(msg % (version,)) def magic(major, minor): @@ -227,6 +242,16 @@ def read_magic(fp): major, minor = magic_str[-2:] return major, minor +def _has_metadata(dt): + if dt.metadata is not None: + return True + elif dt.names is not None: + return any(_has_metadata(dt[k]) for k in dt.names) + elif dt.subdtype is not None: + return _has_metadata(dt.base) + else: + return False + def dtype_to_descr(dtype): """ Get a serializable descriptor from the dtype. @@ -250,6 +275,10 @@ def dtype_to_descr(dtype): replicate the input dtype. """ + if _has_metadata(dtype): + warnings.warn("metadata on a dtype may be saved or ignored, but will " + "raise if saved when read. Use another form of storage.", + UserWarning, stacklevel=2) if dtype.names is not None: # This is a record array. The .descr is fine. XXX: parts of the # record array with an empty name, like padding bytes, still get @@ -262,15 +291,19 @@ def dtype_to_descr(dtype): def descr_to_dtype(descr): ''' descr may be stored as dtype.descr, which is a list of - (name, format, [shape]) tuples. Offsets are not explicitly saved, rather - empty fields with name,format == '', '|Vn' are added as padding. + (name, format, [shape]) tuples where format may be a str or a tuple. + Offsets are not explicitly saved, rather empty fields with + name, format == '', '|Vn' are added as padding. This function reverses the process, eliminating the empty padding fields. ''' - if isinstance(descr, (str, dict)): + if isinstance(descr, str): # No padding removal needed return numpy.dtype(descr) - + elif isinstance(descr, tuple): + # subtype, will always have a shape descr[1] + dt = descr_to_dtype(descr[0]) + return numpy.dtype((dt, descr[1])) fields = [] offset = 0 for field in descr: @@ -323,6 +356,56 @@ def header_data_from_array_1_0(array): d['descr'] = dtype_to_descr(array.dtype) return d + +def _wrap_header(header, version): + """ + Takes a stringified header, and attaches the prefix and padding to it + """ + import struct + assert version is not None + fmt, encoding = _header_size_info[version] + if not isinstance(header, bytes): # always true on python 3 + header = header.encode(encoding) + hlen = len(header) + 1 + padlen = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize(fmt) + hlen) % ARRAY_ALIGN) + try: + header_prefix = magic(*version) + struct.pack(fmt, hlen + padlen) + except struct.error: + msg = "Header length {} too big for version={}".format(hlen, version) + raise ValueError(msg) + + # Pad the header with spaces and a final newline such that the magic + # string, the header-length short and the header are aligned on a + # ARRAY_ALIGN byte boundary. This supports memory mapping of dtypes + # aligned up to ARRAY_ALIGN on systems like Linux where mmap() + # offset must be page-aligned (i.e. the beginning of the file). + return header_prefix + header + b' '*padlen + b'\n' + + +def _wrap_header_guess_version(header): + """ + Like `_wrap_header`, but chooses an appropriate version given the contents + """ + try: + return _wrap_header(header, (1, 0)) + except ValueError: + pass + + try: + ret = _wrap_header(header, (2, 0)) + except UnicodeEncodeError: + pass + else: + warnings.warn("Stored array in format 2.0. It can only be" + "read by NumPy >= 1.9", UserWarning, stacklevel=2) + return ret + + header = _wrap_header(header, (3, 0)) + warnings.warn("Stored array in format 3.0. It can only be " + "read by NumPy >= 1.17", UserWarning, stacklevel=2) + return header + + def _write_array_header(fp, d, version=None): """ Write the header for an array and returns the version used @@ -336,48 +419,19 @@ def _write_array_header(fp, d, version=None): None means use oldest that works explicit version will raise a ValueError if the format does not allow saving this data. Default: None - Returns - ------- - version : tuple of int - the file version which needs to be used to store the data """ - import struct header = ["{"] for key, value in sorted(d.items()): # Need to use repr here, since we eval these when reading header.append("'%s': %s, " % (key, repr(value))) header.append("}") header = "".join(header) - header = asbytes(_filter_header(header)) - - hlen = len(header) + 1 # 1 for newline - padlen_v1 = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize('<H') + hlen) % ARRAY_ALIGN) - padlen_v2 = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize('<I') + hlen) % ARRAY_ALIGN) - - # Which version(s) we write depends on the total header size; v1 has a max of 65535 - if hlen + padlen_v1 < 2**16 and version in (None, (1, 0)): - version = (1, 0) - header_prefix = magic(1, 0) + struct.pack('<H', hlen + padlen_v1) - topad = padlen_v1 - elif hlen + padlen_v2 < 2**32 and version in (None, (2, 0)): - version = (2, 0) - header_prefix = magic(2, 0) + struct.pack('<I', hlen + padlen_v2) - topad = padlen_v2 + header = _filter_header(header) + if version is None: + header = _wrap_header_guess_version(header) else: - msg = "Header length %s too big for version=%s" - msg %= (hlen, version) - raise ValueError(msg) - - # Pad the header with spaces and a final newline such that the magic - # string, the header-length short and the header are aligned on a - # ARRAY_ALIGN byte boundary. This supports memory mapping of dtypes - # aligned up to ARRAY_ALIGN on systems like Linux where mmap() - # offset must be page-aligned (i.e. the beginning of the file). - header = header + b' '*topad + b'\n' - - fp.write(header_prefix) + header = _wrap_header(header, version) fp.write(header) - return version def write_array_header_1_0(fp, d): """ Write the header for an array using the 1.0 format. @@ -480,7 +534,7 @@ def _filter_header(s): Parameters ---------- - s : byte string + s : string Npy file header. Returns @@ -498,7 +552,7 @@ def _filter_header(s): tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround - string = asstr(s) + "\n" + string = s + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] @@ -520,16 +574,15 @@ def _read_array_header(fp, version): # Read an unsigned, little-endian short int which has the length of the # header. import struct - if version == (1, 0): - hlength_type = '<H' - elif version == (2, 0): - hlength_type = '<I' - else: - raise ValueError("Invalid version %r" % version) + hinfo = _header_size_info.get(version) + if hinfo is None: + raise ValueError("Invalid version {!r}".format(version)) + hlength_type, encoding = hinfo hlength_str = _read_bytes(fp, struct.calcsize(hlength_type), "array header length") header_length = struct.unpack(hlength_type, hlength_str)[0] header = _read_bytes(fp, header_length, "array header") + header = header.decode(encoding) # The header is a pretty-printed string representation of a literal # Python dictionary with trailing newlines padded to a ARRAY_ALIGN byte @@ -541,29 +594,29 @@ def _read_array_header(fp, version): try: d = safe_eval(header) except SyntaxError as e: - msg = "Cannot parse header: %r\nException: %r" - raise ValueError(msg % (header, e)) + msg = "Cannot parse header: {!r}\nException: {!r}" + raise ValueError(msg.format(header, e)) if not isinstance(d, dict): - msg = "Header is not a dictionary: %r" - raise ValueError(msg % d) + msg = "Header is not a dictionary: {!r}" + raise ValueError(msg.format(d)) keys = sorted(d.keys()) if keys != ['descr', 'fortran_order', 'shape']: - msg = "Header does not contain the correct keys: %r" - raise ValueError(msg % (keys,)) + msg = "Header does not contain the correct keys: {!r}" + raise ValueError(msg.format(keys)) # Sanity-check the values. if (not isinstance(d['shape'], tuple) or not numpy.all([isinstance(x, (int, long)) for x in d['shape']])): - msg = "shape is not valid: %r" - raise ValueError(msg % (d['shape'],)) + msg = "shape is not valid: {!r}" + raise ValueError(msg.format(d['shape'])) if not isinstance(d['fortran_order'], bool): - msg = "fortran_order is not a valid bool: %r" - raise ValueError(msg % (d['fortran_order'],)) + msg = "fortran_order is not a valid bool: {!r}" + raise ValueError(msg.format(d['fortran_order'])) try: dtype = descr_to_dtype(d['descr']) except TypeError as e: - msg = "descr is not a valid dtype descriptor: %r" - raise ValueError(msg % (d['descr'],)) + msg = "descr is not a valid dtype descriptor: {!r}" + raise ValueError(msg.format(d['descr'])) return d['shape'], d['fortran_order'], dtype @@ -604,12 +657,7 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): """ _check_version(version) - used_ver = _write_array_header(fp, header_data_from_array_1_0(array), - version) - # this warning can be removed when 1.9 has aged enough - if version != (2, 0) and used_ver == (2, 0): - warnings.warn("Stored array in format 2.0. It can only be" - "read by NumPy >= 1.9", UserWarning, stacklevel=2) + _write_array_header(fp, header_data_from_array_1_0(array), version) if array.itemsize == 0: buffersize = 0 @@ -619,14 +667,13 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): if array.dtype.hasobject: # We contain Python objects so we cannot write out the data - # directly. Instead, we will pickle it out with version 2 of the - # pickle protocol. + # directly. Instead, we will pickle it out if not allow_pickle: raise ValueError("Object arrays cannot be saved when " "allow_pickle=False") if pickle_kwargs is None: pickle_kwargs = {} - pickle.dump(array, fp, protocol=2, **pickle_kwargs) + pickle.dump(array, fp, protocol=3, **pickle_kwargs) elif array.flags.f_contiguous and not array.flags.c_contiguous: if isfileobj(fp): array.T.tofile(fp) @@ -645,7 +692,7 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): fp.write(chunk.tobytes('C')) -def read_array(fp, allow_pickle=True, pickle_kwargs=None): +def read_array(fp, allow_pickle=False, pickle_kwargs=None): """ Read an array from an NPY file. @@ -655,7 +702,11 @@ def read_array(fp, allow_pickle=True, pickle_kwargs=None): If this is not a real file object, then this may take extra memory and time. allow_pickle : bool, optional - Whether to allow reading pickled data. Default: True + Whether to allow writing pickled data. Default: False + + .. versionchanged:: 1.16.3 + Made default False in response to CVE-2019-6446. + pickle_kwargs : dict Additional keyword arguments to pass to pickle.load. These are only useful when loading object arrays saved on Python 2 when using @@ -806,20 +857,12 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, shape=shape, ) # If we got here, then it should be safe to create the file. - fp = open(os_fspath(filename), mode+'b') - try: - used_ver = _write_array_header(fp, d, version) - # this warning can be removed when 1.9 has aged enough - if version != (2, 0) and used_ver == (2, 0): - warnings.warn("Stored array in format 2.0. It can only be" - "read by NumPy >= 1.9", UserWarning, stacklevel=2) + with open(os_fspath(filename), mode+'b') as fp: + _write_array_header(fp, d, version) offset = fp.tell() - finally: - fp.close() else: # Read the header of the file first. - fp = open(os_fspath(filename), 'rb') - try: + with open(os_fspath(filename), 'rb') as fp: version = read_magic(fp) _check_version(version) @@ -828,8 +871,6 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, msg = "Array can't be memory-mapped: Python objects in dtype." raise ValueError(msg) offset = fp.tell() - finally: - fp.close() if fortran_order: order = 'F' diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index cee7b3a62..499120630 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1,7 +1,7 @@ from __future__ import division, absolute_import, print_function try: - # Accessing collections abstact classes from collections + # Accessing collections abstract classes from collections # has been deprecated since Python 3.3 import collections.abc as collections_abc except ImportError: @@ -31,7 +31,6 @@ from numpy.core.overrides import set_module from numpy.core import overrides from numpy.core.function_base import add_newdoc from numpy.lib.twodim_base import diag -from .utils import deprecate from numpy.core.multiarray import ( _insert, add_docstring, bincount, normalize_axis_index, _monotonicity, interp as compiled_interp, interp_complex as compiled_interp_complex @@ -317,14 +316,17 @@ def average(a, axis=None, weights=None, returned=False): The weights array can either be 1-D (in which case its length must be the size of `a` along the given axis) or of the same shape as `a`. If `weights=None`, then all data in `a` are assumed to have a - weight equal to one. + weight equal to one. The 1-D calculation is:: + + avg = sum(a * weights) / sum(weights) + + The only constraint on `weights` is that `sum(weights)` must not be 0. returned : bool, optional Default is `False`. If `True`, the tuple (`average`, `sum_of_weights`) is returned, otherwise only the average is returned. If `weights=None`, `sum_of_weights` is equivalent to the number of elements over which the average is taken. - Returns ------- retval, [sum_of_weights] : array_type or double @@ -359,12 +361,12 @@ def average(a, axis=None, weights=None, returned=False): Examples -------- - >>> data = list(range(1,5)) + >>> data = np.arange(1, 5) >>> data - [1, 2, 3, 4] + array([1, 2, 3, 4]) >>> np.average(data) 2.5 - >>> np.average(range(1,11), weights=range(10,0,-1)) + >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1)) 4.0 >>> data = np.arange(6).reshape((3,2)) @@ -378,7 +380,7 @@ def average(a, axis=None, weights=None, returned=False): Traceback (most recent call last): ... TypeError: Axis must be specified when shapes of a and weights differ. - + >>> a = np.ones(5, dtype=np.float128) >>> w = np.ones(5, dtype=np.complex64) >>> avg = np.average(a, weights=w) @@ -680,11 +682,7 @@ def select(condlist, choicelist, default=0): # Now that the dtype is known, handle the deprecated select([], []) case if len(condlist) == 0: - # 2014-02-24, 1.9 - warnings.warn("select with an empty condition list is not possible" - "and will be deprecated", - DeprecationWarning, stacklevel=2) - return np.asarray(default)[()] + raise ValueError("select with an empty condition list is not possible") choicelist = [np.asarray(choice) for choice in choicelist] choicelist.append(np.asarray(default)) @@ -700,25 +698,11 @@ def select(condlist, choicelist, default=0): choicelist = np.broadcast_arrays(*choicelist) # If cond array is not an ndarray in boolean format or scalar bool, abort. - deprecated_ints = False for i in range(len(condlist)): cond = condlist[i] if cond.dtype.type is not np.bool_: - if np.issubdtype(cond.dtype, np.integer): - # A previous implementation accepted int ndarrays accidentally. - # Supported here deliberately, but deprecated. - condlist[i] = condlist[i].astype(bool) - deprecated_ints = True - else: - raise ValueError( - 'invalid entry {} in condlist: should be boolean ndarray'.format(i)) - - if deprecated_ints: - # 2014-02-24, 1.9 - msg = "select condlists containing integer ndarrays is deprecated " \ - "and will be removed in the future. Use `.astype(bool)` to " \ - "convert to bools." - warnings.warn(msg, DeprecationWarning, stacklevel=2) + raise TypeError( + 'invalid entry {} in condlist: should be boolean ndarray'.format(i)) if choicelist[0].ndim == 0: # This may be common, so avoid the call. @@ -1150,7 +1134,7 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): """ Calculate the n-th discrete difference along the given axis. - The first difference is given by ``out[n] = a[n+1] - a[n]`` along + The first difference is given by ``out[i] = a[i+1] - a[i]`` along the given axis, higher differences are calculated by using `diff` recursively. @@ -1165,11 +1149,13 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): The axis along which the difference is taken, default is the last axis. prepend, append : array_like, optional - Values to prepend or append to "a" along axis prior to + Values to prepend or append to `a` along axis prior to performing the difference. Scalar values are expanded to arrays with length 1 in the direction of axis and the shape of the input array in along all other axes. Otherwise the - dimension and shape must match "a" except along axis. + dimension and shape must match `a` except along axis. + + .. versionadded:: 1.16.0 Returns ------- @@ -1236,6 +1222,8 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): a = asanyarray(a) nd = a.ndim + if nd == 0: + raise ValueError("diff requires input that is at least one dimensional") axis = normalize_axis_index(axis, nd) combined = [] @@ -1326,9 +1314,13 @@ def interp(x, xp, fp, left=None, right=None, period=None): Notes ----- - Does not check that the x-coordinate sequence `xp` is increasing. - If `xp` is not increasing, the results are nonsense. - A simple check for increasing is:: + The x-coordinate sequence is expected to be increasing, but this is not + explicitly enforced. However, if the sequence `xp` is non-increasing, + interpolation results are meaningless. + + Note that, since NaN is unsortable, `xp` also cannot contain NaNs. + + A simple check for `xp` being strictly increasing is:: np.all(np.diff(xp) > 0) @@ -1430,9 +1422,9 @@ def angle(z, deg=False): Returns ------- angle : ndarray or scalar - The counterclockwise angle from the positive real axis on - the complex plane, with dtype as numpy.float64. - + The counterclockwise angle from the positive real axis on the complex + plane in the range ``(-pi, pi]``, with dtype as numpy.float64. + ..versionchanged:: 1.16.0 This function works on subclasses of ndarray like `ma.array`. @@ -1901,7 +1893,7 @@ class vectorize(object): typecode characters or a list of data type specifiers. There should be one data type specifier for each output. doc : str, optional - The docstring for the function. If `None`, the docstring will be the + The docstring for the function. If None, the docstring will be the ``pyfunc.__doc__``. excluded : set, optional Set of strings or integers representing the positional or keyword @@ -1930,6 +1922,30 @@ class vectorize(object): vectorized : callable Vectorized function. + See Also + -------- + frompyfunc : Takes an arbitrary Python function and returns a ufunc + + Notes + ----- + The `vectorize` function is provided primarily for convenience, not for + performance. The implementation is essentially a for loop. + + If `otypes` is not specified, then a call to the function with the + first argument will be used to determine the number of outputs. The + results of this call will be cached if `cache` is `True` to prevent + calling the function twice. However, to implement the cache, the + original function must be wrapped which will slow down subsequent + calls, so only do this if your function is expensive. + + The new keyword argument interface and `excluded` argument support + further degrades performance. + + References + ---------- + .. [1] NumPy Reference, section `Generalized Universal Function API + <https://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html>`_. + Examples -------- >>> def myfunc(a, b): @@ -1989,8 +2005,8 @@ class vectorize(object): >>> import scipy.stats >>> pearsonr = np.vectorize(scipy.stats.pearsonr, - ... signature='(n),(n)->(),()') - >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) + ... signature='(n),(n)->(),()') + >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) (array([ 1., -1.]), array([ 0., 0.])) Or for a vectorized convolution: @@ -2002,31 +2018,7 @@ class vectorize(object): [0., 0., 1., 2., 1., 0.], [0., 0., 0., 1., 2., 1.]]) - See Also - -------- - frompyfunc : Takes an arbitrary Python function and returns a ufunc - - Notes - ----- - The `vectorize` function is provided primarily for convenience, not for - performance. The implementation is essentially a for loop. - - If `otypes` is not specified, then a call to the function with the - first argument will be used to determine the number of outputs. The - results of this call will be cached if `cache` is `True` to prevent - calling the function twice. However, to implement the cache, the - original function must be wrapped which will slow down subsequent - calls, so only do this if your function is expensive. - - The new keyword argument interface and `excluded` argument support - further degrades performance. - - References - ---------- - .. [1] NumPy Reference, section `Generalized Universal Function API - <https://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html>`_. """ - def __init__(self, pyfunc, otypes=None, doc=None, excluded=None, cache=False, signature=None): self.pyfunc = pyfunc @@ -2313,7 +2305,7 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, >>> m = np.arange(10, dtype=np.float64) >>> f = np.arange(10) * 2 >>> a = np.arange(10) ** 2. - >>> ddof = 9 # N - 1 + >>> ddof = 1 >>> w = f * a >>> v1 = np.sum(w) >>> v2 = np.sum(w * a) @@ -2442,7 +2434,7 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, if fact <= 0: warnings.warn("Degrees of freedom <= 0 for slice", - RuntimeWarning, stacklevel=2) + RuntimeWarning, stacklevel=3) fact = 0.0 X -= avg[:, None] @@ -2521,7 +2513,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): if bias is not np._NoValue or ddof is not np._NoValue: # 2015-03-15, 1.10 warnings.warn('bias and ddof have no effect and are deprecated', - DeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=3) c = cov(x, y, rowvar) try: d = diag(c) @@ -2619,7 +2611,9 @@ def blackman(M): >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) - >>> response = 20 * np.log10(mag) + >>> with np.errstate(divide='ignore', invalid='ignore'): + ... response = 20 * np.log10(mag) + ... >>> response = np.clip(response, -100, 100) >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] @@ -2726,7 +2720,9 @@ def bartlett(M): >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) - >>> response = 20 * np.log10(mag) + >>> with np.errstate(divide='ignore', invalid='ignore'): + ... response = 20 * np.log10(mag) + ... >>> response = np.clip(response, -100, 100) >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] @@ -2827,7 +2823,9 @@ def hanning(M): >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) - >>> response = 20 * np.log10(mag) + >>> with np.errstate(divide='ignore', invalid='ignore'): + ... response = 20 * np.log10(mag) + ... >>> response = np.clip(response, -100, 100) >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] @@ -3063,10 +3061,13 @@ def i0(x): See Also -------- - scipy.special.iv, scipy.special.ive + scipy.special.i0, scipy.special.iv, scipy.special.ive Notes ----- + The scipy implementation is recommended over this function: it is a + proper ufunc written in C, and more than an order of magnitude faster. + We use the algorithm published by Clenshaw [1]_ and referenced by Abramowitz and Stegun [2]_, for which the function domain is partitioned into the two intervals [0,8] and (8,inf), and Chebyshev @@ -3086,21 +3087,15 @@ def i0(x): Examples -------- - >>> np.i0([0.]) - array(1.0) # may vary + >>> np.i0(0.) + array(1.0) # may vary >>> np.i0([0., 1. + 2j]) - array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary + array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary """ - x = atleast_1d(x).copy() - y = empty_like(x) - ind = (x < 0) - x[ind] = -x[ind] - ind = (x <= 8.0) - y[ind] = _i0_1(x[ind]) - ind2 = ~ind - y[ind2] = _i0_2(x[ind2]) - return y.squeeze() + x = np.asanyarray(x) + x = np.abs(x) + return piecewise(x, [x <= 8.0], [_i0_1, _i0_2]) ## End of cephes code for i0 @@ -3306,13 +3301,6 @@ def sinc(x): Text(0.5, 0, 'X') >>> plt.show() - It works in 2-D as well: - - >>> x = np.linspace(-4, 4, 401) - >>> xx = np.outer(x, x) - >>> plt.imshow(np.sinc(xx)) - <matplotlib.image.AxesImage object at 0x...> - """ x = np.asanyarray(x) y = pi * where(x == 0, 1.0e-20, x) @@ -3701,7 +3689,8 @@ def percentile(a, q, axis=None, out=None, plt.show() """ - q = np.true_divide(q, 100.0) # handles the asarray for us too + q = np.true_divide(q, 100) + q = asanyarray(q) # undo any decay that the ufunc performed (see gh-13105) if not _quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") return _quantile_unchecked( @@ -3718,7 +3707,8 @@ def quantile(a, q, axis=None, out=None, overwrite_input=False, interpolation='linear', keepdims=False): """ Compute the q-th quantile of the data along the specified axis. - ..versionadded:: 1.15.0 + + .. versionadded:: 1.15.0 Parameters ---------- @@ -3919,7 +3909,7 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, indices_above = concatenate((indices_above, [-1])) weights_above = indices - indices_below - weights_below = 1.0 - weights_above + weights_below = 1 - weights_above weights_shape = [1, ] * ap.ndim weights_shape[axis] = len(indices) @@ -3956,8 +3946,6 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, r = add(x1, x2) if np.any(n): - warnings.warn("Invalid value encountered in percentile", - RuntimeWarning, stacklevel=3) if zerod: if ap.ndim == 1: if out is not None: @@ -4298,7 +4286,7 @@ def delete(arr, obj, axis=None): # 2013-09-24, 1.9 warnings.warn( "in the future the special handling of scalars will be removed " - "from delete and raise an error", DeprecationWarning, stacklevel=2) + "from delete and raise an error", DeprecationWarning, stacklevel=3) if wrap: return wrap(arr) else: @@ -4335,7 +4323,7 @@ def delete(arr, obj, axis=None): else: slobj[axis] = slice(None, start) new[tuple(slobj)] = arr[tuple(slobj)] - # copy end chunck + # copy end chunk if stop == N: pass else: @@ -4367,7 +4355,7 @@ def delete(arr, obj, axis=None): if obj.dtype == bool: warnings.warn("in the future insert will treat boolean arrays and " "array-likes as boolean index instead of casting it " - "to integer", FutureWarning, stacklevel=2) + "to integer", FutureWarning, stacklevel=3) obj = obj.astype(intp) if isinstance(_obj, (int, long, integer)): # optimization for a single value @@ -4395,7 +4383,7 @@ def delete(arr, obj, axis=None): # 2013-09-24, 1.9 warnings.warn( "using a non-integer array as obj in delete will result in an " - "error in the future", DeprecationWarning, stacklevel=2) + "error in the future", DeprecationWarning, stacklevel=3) obj = obj.astype(intp) keep = ones(N, dtype=bool) @@ -4406,13 +4394,13 @@ def delete(arr, obj, axis=None): warnings.warn( "in the future out of bounds indices will raise an error " "instead of being ignored by `numpy.delete`.", - DeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=3) obj = obj[inside_bounds] positive_indices = obj >= 0 if not positive_indices.all(): warnings.warn( "in the future negative indices will not be ignored by " - "`numpy.delete`.", FutureWarning, stacklevel=2) + "`numpy.delete`.", FutureWarning, stacklevel=3) obj = obj[positive_indices] keep[obj, ] = False @@ -4537,7 +4525,7 @@ def insert(arr, obj, values, axis=None): # 2013-09-24, 1.9 warnings.warn( "in the future the special handling of scalars will be removed " - "from insert and raise an error", DeprecationWarning, stacklevel=2) + "from insert and raise an error", DeprecationWarning, stacklevel=3) arr = arr.copy(order=arrorder) arr[...] = values if wrap: @@ -4561,7 +4549,7 @@ def insert(arr, obj, values, axis=None): warnings.warn( "in the future insert will treat boolean arrays and " "array-likes as a boolean index instead of casting it to " - "integer", FutureWarning, stacklevel=2) + "integer", FutureWarning, stacklevel=3) indices = indices.astype(intp) # Code after warning period: #if obj.ndim != 1: @@ -4611,7 +4599,7 @@ def insert(arr, obj, values, axis=None): # 2013-09-24, 1.9 warnings.warn( "using a non-integer array as obj in insert will result in an " - "error in the future", DeprecationWarning, stacklevel=2) + "error in the future", DeprecationWarning, stacklevel=3) indices = indices.astype(intp) indices[indices < 0] += N diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 7b229cc89..03c365ab6 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -3,6 +3,7 @@ Histogram-related functions """ from __future__ import division, absolute_import, print_function +import contextlib import functools import operator import warnings @@ -21,6 +22,16 @@ array_function_dispatch = functools.partial( _range = range +def _ptp(x): + """Peak-to-peak value of x. + + This implementation avoids the problem of signed integer arrays having a + peak-to-peak value that cannot be represented with the array's data type. + This function returns an unsigned value for signed integer arrays. + """ + return _unsigned_subtract(x.max(), x.min()) + + def _hist_bin_sqrt(x, range): """ Square root histogram bin estimator. @@ -39,7 +50,7 @@ def _hist_bin_sqrt(x, range): h : An estimate of the optimal bin width for the given data. """ del range # unused - return x.ptp() / np.sqrt(x.size) + return _ptp(x) / np.sqrt(x.size) def _hist_bin_sturges(x, range): @@ -62,7 +73,7 @@ def _hist_bin_sturges(x, range): h : An estimate of the optimal bin width for the given data. """ del range # unused - return x.ptp() / (np.log2(x.size) + 1.0) + return _ptp(x) / (np.log2(x.size) + 1.0) def _hist_bin_rice(x, range): @@ -86,7 +97,7 @@ def _hist_bin_rice(x, range): h : An estimate of the optimal bin width for the given data. """ del range # unused - return x.ptp() / (2.0 * x.size ** (1.0 / 3)) + return _ptp(x) / (2.0 * x.size ** (1.0 / 3)) def _hist_bin_scott(x, range): @@ -136,7 +147,7 @@ def _hist_bin_stone(x, range): """ n = x.size - ptp_x = np.ptp(x) + ptp_x = _ptp(x) if n <= 1 or ptp_x == 0: return 0 @@ -148,7 +159,8 @@ def _hist_bin_stone(x, range): nbins_upper_bound = max(100, int(np.sqrt(n))) nbins = min(_range(1, nbins_upper_bound + 1), key=jhat) if nbins == nbins_upper_bound: - warnings.warn("The number of bins estimated may be suboptimal.", RuntimeWarning, stacklevel=2) + warnings.warn("The number of bins estimated may be suboptimal.", + RuntimeWarning, stacklevel=3) return ptp_x / nbins @@ -182,7 +194,7 @@ def _hist_bin_doane(x, range): np.true_divide(temp, sigma, temp) np.power(temp, 3, temp) g1 = np.mean(temp) - return x.ptp() / (1.0 + np.log2(x.size) + + return _ptp(x) / (1.0 + np.log2(x.size) + np.log2(1.0 + np.absolute(g1) / sg1)) return 0.0 @@ -279,7 +291,7 @@ def _ravel_and_check_weights(a, weights): if a.dtype == np.bool_: warnings.warn("Converting input from {} to {} for compatibility." .format(a.dtype, np.uint8), - RuntimeWarning, stacklevel=2) + RuntimeWarning, stacklevel=3) a = a.astype(np.uint8) if weights is not None: @@ -461,7 +473,8 @@ def _histogram_bin_edges_dispatcher(a, bins=None, range=None, weights=None): @array_function_dispatch(_histogram_bin_edges_dispatcher) def histogram_bin_edges(a, bins=10, range=None, weights=None): r""" - Function to calculate only the edges of the bins used by the `histogram` function. + Function to calculate only the edges of the bins used by the `histogram` + function. Parameters ---------- @@ -554,14 +567,14 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): using the `ptp` of the data. The final bin count is obtained from ``np.round(np.ceil(range / h))``. - 'Auto' (maximum of the 'Sturges' and 'FD' estimators) + 'auto' (maximum of the 'sturges' and 'fd' estimators) A compromise to get a good value. For small datasets the Sturges value will usually be chosen, while larger datasets will usually default to FD. Avoids the overly conservative behaviour of FD and Sturges for small and large datasets respectively. Switchover point is usually :math:`a.size \approx 1000`. - 'FD' (Freedman Diaconis Estimator) + 'fd' (Freedman Diaconis Estimator) .. math:: h = 2 \frac{IQR}{n^{1/3}} The binwidth is proportional to the interquartile range (IQR) @@ -569,7 +582,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): conservative for small datasets, but is quite good for large datasets. The IQR is very robust to outliers. - 'Scott' + 'scott' .. math:: h = \sigma \sqrt[3]{\frac{24 * \sqrt{\pi}}{n}} The binwidth is proportional to the standard deviation of the @@ -579,14 +592,14 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): outliers. Values are very similar to the Freedman-Diaconis estimator in the absence of outliers. - 'Rice' + 'rice' .. math:: n_h = 2n^{1/3} The number of bins is only proportional to cube root of ``a.size``. It tends to overestimate the number of bins and it does not take into account data variability. - 'Sturges' + 'sturges' .. math:: n_h = \log _{2}n+1 The number of bins is the base 2 log of ``a.size``. This @@ -594,7 +607,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): larger, non-normal datasets. This is the default method in R's ``hist`` method. - 'Doane' + 'doane' .. math:: n_h = 1 + \log_{2}(n) + \log_{2}(1 + \frac{|g_1|}{\sigma_{g_1}}) @@ -606,8 +619,9 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): estimates for non-normal datasets. This estimator attempts to account for the skew of the data. - 'Sqrt' + 'sqrt' .. math:: n_h = \sqrt n + The simplest and fastest estimator. Only takes into account the data size. @@ -886,7 +900,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, warnings.warn( "The normed argument is ignored when density is provided. " "In future passing both will result in an error.", - DeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=3) normed = None if density: @@ -902,7 +916,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, "density=True will produce the same result anyway. " "The argument will be removed in a future version of " "numpy.", - np.VisibleDeprecationWarning, stacklevel=2) + np.VisibleDeprecationWarning, stacklevel=3) # this normalization is incorrect, but db = np.array(np.diff(bin_edges), float) @@ -913,13 +927,19 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, warnings.warn( "Passing normed=False is deprecated, and has no effect. " "Consider passing the density argument instead.", - DeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=3) return n, bin_edges def _histogramdd_dispatcher(sample, bins=None, range=None, normed=None, weights=None, density=None): - return (sample, bins, weights) + if hasattr(sample, 'shape'): # same condition as used in histogramdd + yield sample + else: + yield from sample + with contextlib.suppress(TypeError): + yield from bins + yield weights @array_function_dispatch(_histogramdd_dispatcher) diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index 64c491cfa..04384854c 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -94,12 +94,13 @@ def ix_(*args): out = [] nd = len(args) for k, new in enumerate(args): - new = asarray(new) + if not isinstance(new, _nx.ndarray): + new = asarray(new) + if new.size == 0: + # Explicitly type empty arrays to avoid float default + new = new.astype(_nx.intp) if new.ndim != 1: raise ValueError("Cross index must be 1 dimensional") - if new.size == 0: - # Explicitly type empty arrays to avoid float default - new = new.astype(_nx.intp) if issubdtype(new.dtype, _nx.bool_): new, = new.nonzero() new = new.reshape((1,)*k + (new.size,) + (1,)*(nd-k-1)) @@ -269,8 +270,9 @@ class OGridClass(nd_grid): the stop value **is inclusive**. Returns - ---------- - mesh-grid `ndarrays` with only one dimension :math:`\\neq 1` + ------- + mesh-grid + `ndarrays` with only one dimension not equal to 1 See Also -------- @@ -812,7 +814,7 @@ def fill_diagonal(a, val, wrap=False): The wrap option affects only tall matrices: >>> # tall matrices no wrap - >>> a = np.zeros((5, 3),int) + >>> a = np.zeros((5, 3), int) >>> np.fill_diagonal(a, 4) >>> a array([[4, 0, 0], @@ -822,7 +824,7 @@ def fill_diagonal(a, val, wrap=False): [0, 0, 0]]) >>> # tall matrices wrap - >>> a = np.zeros((5, 3),int) + >>> a = np.zeros((5, 3), int) >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0], @@ -832,13 +834,30 @@ def fill_diagonal(a, val, wrap=False): [4, 0, 0]]) >>> # wide matrices - >>> a = np.zeros((3, 5),int) + >>> a = np.zeros((3, 5), int) >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0, 0, 0], [0, 4, 0, 0, 0], [0, 0, 4, 0, 0]]) + The anti-diagonal can be filled by reversing the order of elements + using either `numpy.flipud` or `numpy.fliplr`. + + >>> a = np.zeros((3, 3), int); + >>> np.fill_diagonal(np.fliplr(a), [1,2,3]) # Horizontal flip + >>> a + array([[0, 0, 1], + [0, 2, 0], + [3, 0, 0]]) + >>> np.fill_diagonal(np.flipud(a), [1,2,3]) # Vertical flip + >>> a + array([[0, 0, 3], + [0, 2, 0], + [1, 0, 0]]) + + Note that the order in which the diagonal is filled varies depending + on the flip function. """ if a.ndim < 2: raise ValueError("array must be at least 2-d") diff --git a/numpy/lib/info.py b/numpy/lib/info.py deleted file mode 100644 index 8815a52f0..000000000 --- a/numpy/lib/info.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Basic functions used by several sub-packages and -useful to have in the main name-space. - -Type Handling -------------- -================ =================== -iscomplexobj Test for complex object, scalar result -isrealobj Test for real object, scalar result -iscomplex Test for complex elements, array result -isreal Test for real elements, array result -imag Imaginary part -real Real part -real_if_close Turns complex number with tiny imaginary part to real -isneginf Tests for negative infinity, array result -isposinf Tests for positive infinity, array result -isnan Tests for nans, array result -isinf Tests for infinity, array result -isfinite Tests for finite numbers, array result -isscalar True if argument is a scalar -nan_to_num Replaces NaN's with 0 and infinities with large numbers -cast Dictionary of functions to force cast to each type -common_type Determine the minimum common type code for a group - of arrays -mintypecode Return minimal allowed common typecode. -================ =================== - -Index Tricks ------------- -================ =================== -mgrid Method which allows easy construction of N-d - 'mesh-grids' -``r_`` Append and construct arrays: turns slice objects into - ranges and concatenates them, for 2d arrays appends rows. -index_exp Konrad Hinsen's index_expression class instance which - can be useful for building complicated slicing syntax. -================ =================== - -Useful Functions ----------------- -================ =================== -select Extension of where to multiple conditions and choices -extract Extract 1d array from flattened array according to mask -insert Insert 1d array of values into Nd array according to mask -linspace Evenly spaced samples in linear space -logspace Evenly spaced samples in logarithmic space -fix Round x to nearest integer towards zero -mod Modulo mod(x,y) = x % y except keeps sign of y -amax Array maximum along axis -amin Array minimum along axis -ptp Array max-min along axis -cumsum Cumulative sum along axis -prod Product of elements along axis -cumprod Cumluative product along axis -diff Discrete differences along axis -angle Returns angle of complex argument -unwrap Unwrap phase along given axis (1-d algorithm) -sort_complex Sort a complex-array (based on real, then imaginary) -trim_zeros Trim the leading and trailing zeros from 1D array. -vectorize A class that wraps a Python function taking scalar - arguments into a generalized function which can handle - arrays of arguments using the broadcast rules of - numerix Python. -================ =================== - -Shape Manipulation ------------------- -================ =================== -squeeze Return a with length-one dimensions removed. -atleast_1d Force arrays to be >= 1D -atleast_2d Force arrays to be >= 2D -atleast_3d Force arrays to be >= 3D -vstack Stack arrays vertically (row on row) -hstack Stack arrays horizontally (column on column) -column_stack Stack 1D arrays as columns into 2D array -dstack Stack arrays depthwise (along third dimension) -stack Stack arrays along a new axis -split Divide array into a list of sub-arrays -hsplit Split into columns -vsplit Split into rows -dsplit Split along third dimension -================ =================== - -Matrix (2D Array) Manipulations -------------------------------- -================ =================== -fliplr 2D array with columns flipped -flipud 2D array with rows flipped -rot90 Rotate a 2D array a multiple of 90 degrees -eye Return a 2D array with ones down a given diagonal -diag Construct a 2D array from a vector, or return a given - diagonal from a 2D array. -mat Construct a Matrix -bmat Build a Matrix from blocks -================ =================== - -Polynomials ------------ -================ =================== -poly1d A one-dimensional polynomial class -poly Return polynomial coefficients from roots -roots Find roots of polynomial given coefficients -polyint Integrate polynomial -polyder Differentiate polynomial -polyadd Add polynomials -polysub Subtract polynomials -polymul Multiply polynomials -polydiv Divide polynomials -polyval Evaluate polynomial at given argument -================ =================== - -Iterators ---------- -================ =================== -Arrayterator A buffered iterator for big arrays. -================ =================== - -Import Tricks -------------- -================ =================== -ppimport Postpone module import until trying to use it -ppimport_attr Postpone module import until trying to use its attribute -ppresolve Import postponed module and return it. -================ =================== - -Machine Arithmetics -------------------- -================ =================== -machar_single Single precision floating point arithmetic parameters -machar_double Double precision floating point arithmetic parameters -================ =================== - -Threading Tricks ----------------- -================ =================== -ParallelExec Execute commands in parallel thread. -================ =================== - -Array Set Operations ------------------------ -Set operations for numeric arrays based on sort() function. - -================ =================== -unique Unique elements of an array. -isin Test whether each element of an ND array is present - anywhere within a second array. -ediff1d Array difference (auxiliary function). -intersect1d Intersection of 1D arrays with unique elements. -setxor1d Set exclusive-or of 1D arrays with unique elements. -in1d Test whether elements in a 1D array are also present in - another array. -union1d Union of 1D arrays with unique elements. -setdiff1d Set difference of 1D arrays with unique elements. -================ =================== - -""" -from __future__ import division, absolute_import, print_function - -depends = ['core', 'testing'] -global_symbols = ['*'] diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 52ad45b68..f974a7724 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -5,8 +5,8 @@ import sys from numpy.core import umath as um -# Nothing should be exposed in the top-level NumPy module. -__all__ = [] + +__all__ = ['NDArrayOperatorsMixin'] def _disables_array_ufunc(obj): diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index b3bf1880b..8e2a34e70 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -40,6 +40,33 @@ __all__ = [ ] +def _nan_mask(a, out=None): + """ + Parameters + ---------- + a : array-like + Input array with at least 1 dimension. + out : ndarray, optional + Alternate output array in which to place the result. The default + is ``None``; if provided, it must have the same shape as the + expected output and will prevent the allocation of a new array. + + Returns + ------- + y : bool ndarray or True + A bool array where ``np.nan`` positions are marked with ``False`` + and other positions are marked with ``True``. If the type of ``a`` + is such that it can't possibly contain ``np.nan``, returns ``True``. + """ + # we assume that a is an array for this private function + + if a.dtype.kind not in 'fc': + return True + + y = np.isnan(a, out=out) + y = np.invert(y, out=y) + return y + def _replace_nan(a, val): """ If `a` is of inexact type, make a copy of `a`, replace NaNs with @@ -68,17 +95,18 @@ def _replace_nan(a, val): NaNs, otherwise return None. """ - a = np.array(a, subok=True, copy=True) + a = np.asanyarray(a) if a.dtype == np.object_: # object arrays do not support `isnan` (gh-9009), so make a guess - mask = a != a + mask = np.not_equal(a, a, dtype=bool) elif issubclass(a.dtype.type, np.inexact): mask = np.isnan(a) else: mask = None if mask is not None: + a = np.array(a, subok=True, copy=True) np.copyto(a, val, where=mask) return a, mask @@ -138,7 +166,8 @@ def _remove_nan_1d(arr1d, overwrite_input=False): c = np.isnan(arr1d) s = np.nonzero(c)[0] if s.size == arr1d.size: - warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=4) + warnings.warn("All-NaN slice encountered", RuntimeWarning, + stacklevel=5) return arr1d[:0], True elif s.size == 0: return arr1d, overwrite_input @@ -216,8 +245,8 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -291,7 +320,8 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): # which do not implement isnan (gh-9009), or fmin correctly (gh-8975) res = np.fmin.reduce(a, axis=axis, out=out, **kwargs) if np.isnan(res).any(): - warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=2) + warnings.warn("All-NaN slice encountered", RuntimeWarning, + stacklevel=3) else: # Slow, but safe for subclasses of ndarray a, mask = _replace_nan(a, +np.inf) @@ -303,7 +333,8 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): mask = np.all(mask, axis=axis, **kwargs) if np.any(mask): res = _copyto(res, np.nan, mask) - warnings.warn("All-NaN axis encountered", RuntimeWarning, stacklevel=2) + warnings.warn("All-NaN axis encountered", RuntimeWarning, + stacklevel=3) return res @@ -329,8 +360,8 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -404,7 +435,8 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): # which do not implement isnan (gh-9009), or fmax correctly (gh-8975) res = np.fmax.reduce(a, axis=axis, out=out, **kwargs) if np.isnan(res).any(): - warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=2) + warnings.warn("All-NaN slice encountered", RuntimeWarning, + stacklevel=3) else: # Slow, but safe for subclasses of ndarray a, mask = _replace_nan(a, -np.inf) @@ -416,7 +448,8 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): mask = np.all(mask, axis=axis, **kwargs) if np.any(mask): res = _copyto(res, np.nan, mask) - warnings.warn("All-NaN axis encountered", RuntimeWarning, stacklevel=2) + warnings.warn("All-NaN axis encountered", RuntimeWarning, + stacklevel=3) return res @@ -553,8 +586,8 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. The casting of NaN to integer can yield - unexpected results. + `ufuncs-output-type` for more details. The casting of NaN to integer + can yield unexpected results. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -649,9 +682,9 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. The casting of NaN to integer can yield - unexpected results. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. The casting of NaN to integer + can yield unexpected results. keepdims : bool, optional If True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will @@ -718,8 +751,8 @@ def nancumsum(a, axis=None, dtype=None, out=None): out : ndarray, optional Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output - but the type will be cast if necessary. See `doc.ufuncs` - (Section "Output arguments") for more details. + but the type will be cast if necessary. See `ufuncs-output-type` for + more details. Returns ------- @@ -856,8 +889,8 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -920,7 +953,7 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): isbad = (cnt == 0) if isbad.any(): - warnings.warn("Mean of empty slice", RuntimeWarning, stacklevel=2) + warnings.warn("Mean of empty slice", RuntimeWarning, stacklevel=3) # NaN is the only possible bad value, so no further # action is needed to handle bad results. return avg @@ -932,7 +965,7 @@ def _nanmedian1d(arr1d, overwrite_input=False): See nanmedian for parameter usage """ arr1d, overwrite_input = _remove_nan_1d(arr1d, - overwrite_input=overwrite_input) + overwrite_input=overwrite_input) if arr1d.size == 0: return np.nan @@ -975,7 +1008,8 @@ def _nanmedian_small(a, axis=None, out=None, overwrite_input=False): a = np.ma.masked_array(a, np.isnan(a)) m = np.ma.median(a, axis=axis, overwrite_input=overwrite_input) for i in range(np.count_nonzero(m.mask.ravel())): - warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=3) + warnings.warn("All-NaN slice encountered", RuntimeWarning, + stacklevel=4) if out is not None: out[...] = m.filled(np.nan) return out @@ -1224,6 +1258,7 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, Compute the qth quantile of the data along the specified axis, while ignoring nan values. Returns the qth quantile(s) of the array elements. + .. versionadded:: 1.15.0 Parameters @@ -1409,7 +1444,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): the variance of the flattened array. dtype : data-type, optional Type to use in computing the variance. For arrays of integer type - the default is `float32`; for arrays of float types it is the same as + the default is `float64`; for arrays of float types it is the same as the array type. out : ndarray, optional Alternate output array in which to place the result. It must have @@ -1439,7 +1474,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): mean : Average var : Variance while not ignoring NaNs nanstd, nanmean - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -1520,7 +1555,8 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): isbad = (dof <= 0) if np.any(isbad): - warnings.warn("Degrees of freedom <= 0 for slice.", RuntimeWarning, stacklevel=2) + warnings.warn("Degrees of freedom <= 0 for slice.", RuntimeWarning, + stacklevel=3) # NaN, inf, or negative numbers are all possible bad # values, so explicitly replace them with NaN. var = _copyto(var, np.nan, isbad) @@ -1590,7 +1626,7 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): -------- var, mean, std nanvar, nanmean - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 704fea108..3e54ff10c 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -7,6 +7,7 @@ import functools import itertools import warnings import weakref +import contextlib from operator import itemgetter, index as opindex import numpy as np @@ -23,10 +24,9 @@ from ._iotools import ( ) from numpy.compat import ( - asbytes, asstr, asunicode, asbytes_nested, bytes, basestring, unicode, - os_fspath, os_PathLike + asbytes, asstr, asunicode, bytes, basestring, os_fspath, os_PathLike, + pickle, contextlib_nullcontext ) -from numpy.core.numeric import pickle if sys.version_info[0] >= 3: from collections.abc import Mapping @@ -146,7 +146,11 @@ class NpzFile(Mapping): An object on which attribute can be performed as an alternative to getitem access on the `NpzFile` instance itself. allow_pickle : bool, optional - Allow loading pickled data. Default: True + Allow loading pickled data. Default: False + + .. versionchanged:: 1.16.3 + Made default False in response to CVE-2019-6446. + pickle_kwargs : dict, optional Additional keyword arguments to pass on to pickle.load. These are only useful when loading object arrays saved on @@ -182,7 +186,7 @@ class NpzFile(Mapping): """ - def __init__(self, fid, own_fid=False, allow_pickle=True, + def __init__(self, fid, own_fid=False, allow_pickle=False, pickle_kwargs=None): # Import is postponed to here since zipfile depends on gzip, an # optional component of the so-called standard library. @@ -285,11 +289,17 @@ class NpzFile(Mapping): @set_module('numpy') -def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, +def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, encoding='ASCII'): """ Load arrays or pickled objects from ``.npy``, ``.npz`` or pickled files. + .. warning:: Loading files that contain object arrays uses the ``pickle`` + module, which is not secure against erroneous or maliciously + constructed data. Consider passing ``allow_pickle=False`` to + load data that is known not to contain object arrays for the + safer handling of untrusted sources. + Parameters ---------- file : file-like object, string, or pathlib.Path @@ -307,8 +317,11 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, Allow loading pickled object arrays stored in npy files. Reasons for disallowing pickles include security, as loading pickled data can execute arbitrary code. If pickles are disallowed, loading object - arrays will fail. - Default: True + arrays will fail. Default: False + + .. versionchanged:: 1.16.3 + Made default False in response to CVE-2019-6446. + fix_imports : bool, optional Only useful when loading Python 2 generated pickled files on Python 3, which includes npy/npz files containing object arrays. If `fix_imports` @@ -467,7 +480,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): file : file, str, or pathlib.Path File or filename to which the data is saved. If file is a file-object, then the filename is unchanged. If file is a string or Path, a ``.npy`` - extension will be appended to the file name if it does not already + extension will be appended to the filename if it does not already have one. arr : array_like Array data to be saved. @@ -494,6 +507,8 @@ def save(file, arr, allow_pickle=True, fix_imports=True): ----- For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`. + Any data saved to the file is appended to the end of the file. + Examples -------- >>> from tempfile import TemporaryFile @@ -506,9 +521,18 @@ def save(file, arr, allow_pickle=True, fix_imports=True): >>> np.load(outfile) array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + >>> with open('test.npy', 'wb') as f: + ... np.save(f, np.array([1, 2])) + ... np.save(f, np.array([1, 3])) + >>> with open('test.npy', 'rb') as f: + ... a = np.load(f) + ... b = np.load(f) + >>> print(a, b) + # [1 2] [1 3] """ own_fid = False - if hasattr(file, 'read'): + if hasattr(file, 'write'): fid = file else: file = os_fspath(file) @@ -541,8 +565,7 @@ def _savez_dispatcher(file, *args, **kwds): @array_function_dispatch(_savez_dispatcher) def savez(file, *args, **kwds): - """ - Save several arrays into a single file in uncompressed ``.npz`` format. + """Save several arrays into a single file in uncompressed ``.npz`` format. If arguments are passed in with no keywords, the corresponding variable names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword @@ -552,9 +575,9 @@ def savez(file, *args, **kwds): Parameters ---------- file : str or file - Either the file name (string) or an open file (file-like object) + Either the filename (string) or an open file (file-like object) where the data will be saved. If file is a string or a Path, the - ``.npz`` extension will be appended to the file name if it is not + ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional Arrays to save to the file. Since it is not possible for Python to @@ -587,6 +610,10 @@ def savez(file, *args, **kwds): its list of arrays (with the ``.files`` attribute), and for the arrays themselves. + When saving dictionaries, the dictionary keys become filenames + inside the ZIP archive. Therefore, keys should be valid filenames. + E.g., avoid keys that begin with ``/`` or contain ``.``. + Examples -------- >>> from tempfile import TemporaryFile @@ -614,7 +641,6 @@ def savez(file, *args, **kwds): ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - """ _savez(file, args, kwds, False) @@ -632,15 +658,15 @@ def savez_compressed(file, *args, **kwds): Save several arrays into a single file in compressed ``.npz`` format. If keyword arguments are given, then filenames are taken from the keywords. - If arguments are passed in with no keywords, then stored file names are + If arguments are passed in with no keywords, then stored filenames are arr_0, arr_1, etc. Parameters ---------- file : str or file - Either the file name (string) or an open file (file-like object) + Either the filename (string) or an open file (file-like object) where the data will be saved. If file is a string or a Path, the - ``.npz`` extension will be appended to the file name if it is not + ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional Arrays to save to the file. Since it is not possible for Python to @@ -696,7 +722,7 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None): # component of the so-called standard library. import zipfile - if not hasattr(file, 'read'): + if not hasattr(file, 'write'): file = os_fspath(file) if not file.endswith('.npz'): file = file + '.npz' @@ -721,8 +747,8 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None): for key, val in namedict.items(): fname = key + '.npy' val = np.asanyarray(val) - force_zip64 = val.nbytes >= 2**30 - with zipf.open(fname, 'w', force_zip64=force_zip64) as fid: + # always force zip64, gh-10776 + with zipf.open(fname, 'w', force_zip64=True) as fid: format.write_array(fid, val, allow_pickle=allow_pickle, pickle_kwargs=pickle_kwargs) @@ -807,7 +833,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, fname : file, str, or pathlib.Path File, filename, or generator to read. If the filename extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note that - generators should return byte strings for Python 3k. + generators should return byte strings. dtype : data-type, optional Data-type of the resulting array; default: float. If this is a structured data-type, the resulting array will be 1-dimensional, and @@ -1118,7 +1144,6 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, if type(x) is bytes: return conv(x) return conv(x.encode("latin1")) - import functools converters[i] = functools.partial(tobytes_first, conv=conv) else: converters[i] = conv @@ -1376,7 +1401,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', # Complex dtype -- each field indicates a separate column else: - ncol = len(X.dtype.descr) + ncol = len(X.dtype.names) else: ncol = X.shape[1] @@ -1387,7 +1412,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', if len(fmt) != ncol: raise AttributeError('fmt has wrong shape. %s' % str(fmt)) format = asstr(delimiter).join(map(asstr, fmt)) - elif isinstance(fmt, str): + elif isinstance(fmt, basestring): n_fmt_chars = fmt.count('%') error = ValueError('fmt has wrong number of %% formats: %s' % fmt) if n_fmt_chars == 1: @@ -1446,7 +1471,7 @@ def fromregex(file, regexp, dtype, encoding=None): Parameters ---------- file : str or file - File name or file object to read. + Filename or file object to read. regexp : str or regexp Regular expression used to parse the file. Groups in the regular expression correspond to fields in the dtype. @@ -1504,9 +1529,9 @@ def fromregex(file, regexp, dtype, encoding=None): dtype = np.dtype(dtype) content = file.read() - if isinstance(content, bytes) and isinstance(regexp, np.unicode): + if isinstance(content, bytes) and isinstance(regexp, np.compat.unicode): regexp = asbytes(regexp) - elif isinstance(content, np.unicode) and isinstance(regexp, bytes): + elif isinstance(content, np.compat.unicode) and isinstance(regexp, bytes): regexp = asstr(regexp) if not hasattr(regexp, 'match'): @@ -1537,7 +1562,8 @@ def fromregex(file, regexp, dtype, encoding=None): def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skip_header=0, skip_footer=0, converters=None, missing_values=None, filling_values=None, usecols=None, - names=None, excludelist=None, deletechars=None, + names=None, excludelist=None, + deletechars=''.join(sorted(NameValidator.defaultdeletechars)), replace_space='_', autostrip=False, case_sensitive=True, defaultfmt="f%i", unpack=None, usemask=False, loose=True, invalid_raise=True, max_rows=None, encoding='bytes'): @@ -1552,7 +1578,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, fname : file, str, pathlib.Path, list of str, generator File, filename, list, or generator to read. If the filename extension is `.gz` or `.bz2`, the file is first decompressed. Note - that generators must return byte strings in Python 3k. The strings + that generators must return byte strings. The strings in a list or produced by a generator are treated as lines. dtype : dtype, optional Data type of the resulting array. @@ -1704,6 +1730,16 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, array((1, 1.3, b'abcde'), dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', 'S5')]) + An example to show comments + + >>> f = StringIO(''' + ... text,# of chars + ... hello world,11 + ... numpy,5''') + >>> np.genfromtxt(f, dtype='S12,S12', delimiter=',') + array([(b'text', b''), (b'hello world', b'11'), (b'numpy', b'5')], + dtype=[('f0', 'S12'), ('f1', 'S12')]) + """ if max_rows is not None: if skip_footer: @@ -1729,301 +1765,300 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, byte_converters = False # Initialize the filehandle, the LineSplitter and the NameValidator - own_fhd = False try: if isinstance(fname, os_PathLike): fname = os_fspath(fname) if isinstance(fname, basestring): - fhd = iter(np.lib._datasource.open(fname, 'rt', encoding=encoding)) - own_fhd = True + fid = np.lib._datasource.open(fname, 'rt', encoding=encoding) + fid_ctx = contextlib.closing(fid) else: - fhd = iter(fname) + fid = fname + fid_ctx = contextlib_nullcontext(fid) + fhd = iter(fid) except TypeError: raise TypeError( "fname must be a string, filehandle, list of strings, " "or generator. Got %s instead." % type(fname)) - split_line = LineSplitter(delimiter=delimiter, comments=comments, - autostrip=autostrip, encoding=encoding) - validate_names = NameValidator(excludelist=excludelist, - deletechars=deletechars, - case_sensitive=case_sensitive, - replace_space=replace_space) - - # Skip the first `skip_header` rows - for i in range(skip_header): - next(fhd) + with fid_ctx: + split_line = LineSplitter(delimiter=delimiter, comments=comments, + autostrip=autostrip, encoding=encoding) + validate_names = NameValidator(excludelist=excludelist, + deletechars=deletechars, + case_sensitive=case_sensitive, + replace_space=replace_space) - # Keep on until we find the first valid values - first_values = None - try: - while not first_values: - first_line = _decode_line(next(fhd), encoding) - if (names is True) and (comments is not None): - if comments in first_line: - first_line = ( - ''.join(first_line.split(comments)[1:])) - first_values = split_line(first_line) - except StopIteration: - # return an empty array if the datafile is empty - first_line = '' - first_values = [] - warnings.warn('genfromtxt: Empty input file: "%s"' % fname, stacklevel=2) - - # Should we take the first values as names ? - if names is True: - fval = first_values[0].strip() - if comments is not None: - if fval in comments: - del first_values[0] - - # Check the columns to use: make sure `usecols` is a list - if usecols is not None: + # Skip the first `skip_header` rows try: - usecols = [_.strip() for _ in usecols.split(",")] - except AttributeError: + for i in range(skip_header): + next(fhd) + + # Keep on until we find the first valid values + first_values = None + + while not first_values: + first_line = _decode_line(next(fhd), encoding) + if (names is True) and (comments is not None): + if comments in first_line: + first_line = ( + ''.join(first_line.split(comments)[1:])) + first_values = split_line(first_line) + except StopIteration: + # return an empty array if the datafile is empty + first_line = '' + first_values = [] + warnings.warn('genfromtxt: Empty input file: "%s"' % fname, stacklevel=2) + + # Should we take the first values as names ? + if names is True: + fval = first_values[0].strip() + if comments is not None: + if fval in comments: + del first_values[0] + + # Check the columns to use: make sure `usecols` is a list + if usecols is not None: try: - usecols = list(usecols) - except TypeError: - usecols = [usecols, ] - nbcols = len(usecols or first_values) - - # Check the names and overwrite the dtype.names if needed - if names is True: - names = validate_names([str(_.strip()) for _ in first_values]) - first_line = '' - elif _is_string_like(names): - names = validate_names([_.strip() for _ in names.split(',')]) - elif names: - names = validate_names(names) - # Get the dtype - if dtype is not None: - dtype = easy_dtype(dtype, defaultfmt=defaultfmt, names=names, - excludelist=excludelist, - deletechars=deletechars, - case_sensitive=case_sensitive, - replace_space=replace_space) - # Make sure the names is a list (for 2.5) - if names is not None: - names = list(names) - - if usecols: - for (i, current) in enumerate(usecols): - # if usecols is a list of names, convert to a list of indices - if _is_string_like(current): - usecols[i] = names.index(current) - elif current < 0: - usecols[i] = current + len(first_values) - # If the dtype is not None, make sure we update it - if (dtype is not None) and (len(dtype) > nbcols): - descr = dtype.descr - dtype = np.dtype([descr[_] for _ in usecols]) - names = list(dtype.names) - # If `names` is not None, update the names - elif (names is not None) and (len(names) > nbcols): - names = [names[_] for _ in usecols] - elif (names is not None) and (dtype is not None): - names = list(dtype.names) - - # Process the missing values ............................... - # Rename missing_values for convenience - user_missing_values = missing_values or () - if isinstance(user_missing_values, bytes): - user_missing_values = user_missing_values.decode('latin1') - - # Define the list of missing_values (one column: one list) - missing_values = [list(['']) for _ in range(nbcols)] - - # We have a dictionary: process it field by field - if isinstance(user_missing_values, dict): - # Loop on the items - for (key, val) in user_missing_values.items(): - # Is the key a string ? - if _is_string_like(key): - try: - # Transform it into an integer - key = names.index(key) - except ValueError: - # We couldn't find it: the name must have been dropped - continue - # Redefine the key as needed if it's a column number - if usecols: + usecols = [_.strip() for _ in usecols.split(",")] + except AttributeError: try: - key = usecols.index(key) - except ValueError: - pass - # Transform the value as a list of string - if isinstance(val, (list, tuple)): - val = [str(_) for _ in val] + usecols = list(usecols) + except TypeError: + usecols = [usecols, ] + nbcols = len(usecols or first_values) + + # Check the names and overwrite the dtype.names if needed + if names is True: + names = validate_names([str(_.strip()) for _ in first_values]) + first_line = '' + elif _is_string_like(names): + names = validate_names([_.strip() for _ in names.split(',')]) + elif names: + names = validate_names(names) + # Get the dtype + if dtype is not None: + dtype = easy_dtype(dtype, defaultfmt=defaultfmt, names=names, + excludelist=excludelist, + deletechars=deletechars, + case_sensitive=case_sensitive, + replace_space=replace_space) + # Make sure the names is a list (for 2.5) + if names is not None: + names = list(names) + + if usecols: + for (i, current) in enumerate(usecols): + # if usecols is a list of names, convert to a list of indices + if _is_string_like(current): + usecols[i] = names.index(current) + elif current < 0: + usecols[i] = current + len(first_values) + # If the dtype is not None, make sure we update it + if (dtype is not None) and (len(dtype) > nbcols): + descr = dtype.descr + dtype = np.dtype([descr[_] for _ in usecols]) + names = list(dtype.names) + # If `names` is not None, update the names + elif (names is not None) and (len(names) > nbcols): + names = [names[_] for _ in usecols] + elif (names is not None) and (dtype is not None): + names = list(dtype.names) + + # Process the missing values ............................... + # Rename missing_values for convenience + user_missing_values = missing_values or () + if isinstance(user_missing_values, bytes): + user_missing_values = user_missing_values.decode('latin1') + + # Define the list of missing_values (one column: one list) + missing_values = [list(['']) for _ in range(nbcols)] + + # We have a dictionary: process it field by field + if isinstance(user_missing_values, dict): + # Loop on the items + for (key, val) in user_missing_values.items(): + # Is the key a string ? + if _is_string_like(key): + try: + # Transform it into an integer + key = names.index(key) + except ValueError: + # We couldn't find it: the name must have been dropped + continue + # Redefine the key as needed if it's a column number + if usecols: + try: + key = usecols.index(key) + except ValueError: + pass + # Transform the value as a list of string + if isinstance(val, (list, tuple)): + val = [str(_) for _ in val] + else: + val = [str(val), ] + # Add the value(s) to the current list of missing + if key is None: + # None acts as default + for miss in missing_values: + miss.extend(val) + else: + missing_values[key].extend(val) + # We have a sequence : each item matches a column + elif isinstance(user_missing_values, (list, tuple)): + for (value, entry) in zip(user_missing_values, missing_values): + value = str(value) + if value not in entry: + entry.append(value) + # We have a string : apply it to all entries + elif isinstance(user_missing_values, basestring): + user_value = user_missing_values.split(",") + for entry in missing_values: + entry.extend(user_value) + # We have something else: apply it to all entries + else: + for entry in missing_values: + entry.extend([str(user_missing_values)]) + + # Process the filling_values ............................... + # Rename the input for convenience + user_filling_values = filling_values + if user_filling_values is None: + user_filling_values = [] + # Define the default + filling_values = [None] * nbcols + # We have a dictionary : update each entry individually + if isinstance(user_filling_values, dict): + for (key, val) in user_filling_values.items(): + if _is_string_like(key): + try: + # Transform it into an integer + key = names.index(key) + except ValueError: + # We couldn't find it: the name must have been dropped, + continue + # Redefine the key if it's a column number and usecols is defined + if usecols: + try: + key = usecols.index(key) + except ValueError: + pass + # Add the value to the list + filling_values[key] = val + # We have a sequence : update on a one-to-one basis + elif isinstance(user_filling_values, (list, tuple)): + n = len(user_filling_values) + if (n <= nbcols): + filling_values[:n] = user_filling_values else: - val = [str(val), ] - # Add the value(s) to the current list of missing - if key is None: - # None acts as default - for miss in missing_values: - miss.extend(val) + filling_values = user_filling_values[:nbcols] + # We have something else : use it for all entries + else: + filling_values = [user_filling_values] * nbcols + + # Initialize the converters ................................ + if dtype is None: + # Note: we can't use a [...]*nbcols, as we would have 3 times the same + # ... converter, instead of 3 different converters. + converters = [StringConverter(None, missing_values=miss, default=fill) + for (miss, fill) in zip(missing_values, filling_values)] + else: + dtype_flat = flatten_dtype(dtype, flatten_base=True) + # Initialize the converters + if len(dtype_flat) > 1: + # Flexible type : get a converter from each dtype + zipit = zip(dtype_flat, missing_values, filling_values) + converters = [StringConverter(dt, locked=True, + missing_values=miss, default=fill) + for (dt, miss, fill) in zipit] else: - missing_values[key].extend(val) - # We have a sequence : each item matches a column - elif isinstance(user_missing_values, (list, tuple)): - for (value, entry) in zip(user_missing_values, missing_values): - value = str(value) - if value not in entry: - entry.append(value) - # We have a string : apply it to all entries - elif isinstance(user_missing_values, basestring): - user_value = user_missing_values.split(",") - for entry in missing_values: - entry.extend(user_value) - # We have something else: apply it to all entries - else: - for entry in missing_values: - entry.extend([str(user_missing_values)]) - - # Process the filling_values ............................... - # Rename the input for convenience - user_filling_values = filling_values - if user_filling_values is None: - user_filling_values = [] - # Define the default - filling_values = [None] * nbcols - # We have a dictionary : update each entry individually - if isinstance(user_filling_values, dict): - for (key, val) in user_filling_values.items(): - if _is_string_like(key): + # Set to a default converter (but w/ different missing values) + zipit = zip(missing_values, filling_values) + converters = [StringConverter(dtype, locked=True, + missing_values=miss, default=fill) + for (miss, fill) in zipit] + # Update the converters to use the user-defined ones + uc_update = [] + for (j, conv) in user_converters.items(): + # If the converter is specified by column names, use the index instead + if _is_string_like(j): try: - # Transform it into an integer - key = names.index(key) + j = names.index(j) + i = j except ValueError: - # We couldn't find it: the name must have been dropped, continue - # Redefine the key if it's a column number and usecols is defined - if usecols: + elif usecols: try: - key = usecols.index(key) + i = usecols.index(j) except ValueError: - pass - # Add the value to the list - filling_values[key] = val - # We have a sequence : update on a one-to-one basis - elif isinstance(user_filling_values, (list, tuple)): - n = len(user_filling_values) - if (n <= nbcols): - filling_values[:n] = user_filling_values - else: - filling_values = user_filling_values[:nbcols] - # We have something else : use it for all entries - else: - filling_values = [user_filling_values] * nbcols - - # Initialize the converters ................................ - if dtype is None: - # Note: we can't use a [...]*nbcols, as we would have 3 times the same - # ... converter, instead of 3 different converters. - converters = [StringConverter(None, missing_values=miss, default=fill) - for (miss, fill) in zip(missing_values, filling_values)] - else: - dtype_flat = flatten_dtype(dtype, flatten_base=True) - # Initialize the converters - if len(dtype_flat) > 1: - # Flexible type : get a converter from each dtype - zipit = zip(dtype_flat, missing_values, filling_values) - converters = [StringConverter(dt, locked=True, - missing_values=miss, default=fill) - for (dt, miss, fill) in zipit] - else: - # Set to a default converter (but w/ different missing values) - zipit = zip(missing_values, filling_values) - converters = [StringConverter(dtype, locked=True, - missing_values=miss, default=fill) - for (miss, fill) in zipit] - # Update the converters to use the user-defined ones - uc_update = [] - for (j, conv) in user_converters.items(): - # If the converter is specified by column names, use the index instead - if _is_string_like(j): - try: - j = names.index(j) + # Unused converter specified + continue + else: i = j - except ValueError: - continue - elif usecols: - try: - i = usecols.index(j) - except ValueError: - # Unused converter specified + # Find the value to test - first_line is not filtered by usecols: + if len(first_line): + testing_value = first_values[j] + else: + testing_value = None + if conv is bytes: + user_conv = asbytes + elif byte_converters: + # converters may use decode to workaround numpy's old behaviour, + # so encode the string again before passing to the user converter + def tobytes_first(x, conv): + if type(x) is bytes: + return conv(x) + return conv(x.encode("latin1")) + user_conv = functools.partial(tobytes_first, conv=conv) + else: + user_conv = conv + converters[i].update(user_conv, locked=True, + testing_value=testing_value, + default=filling_values[i], + missing_values=missing_values[i],) + uc_update.append((i, user_conv)) + # Make sure we have the corrected keys in user_converters... + user_converters.update(uc_update) + + # Fixme: possible error as following variable never used. + # miss_chars = [_.missing_values for _ in converters] + + # Initialize the output lists ... + # ... rows + rows = [] + append_to_rows = rows.append + # ... masks + if usemask: + masks = [] + append_to_masks = masks.append + # ... invalid + invalid = [] + append_to_invalid = invalid.append + + # Parse each line + for (i, line) in enumerate(itertools.chain([first_line, ], fhd)): + values = split_line(line) + nbvalues = len(values) + # Skip an empty line + if nbvalues == 0: continue - else: - i = j - # Find the value to test - first_line is not filtered by usecols: - if len(first_line): - testing_value = first_values[j] - else: - testing_value = None - if conv is bytes: - user_conv = asbytes - elif byte_converters: - # converters may use decode to workaround numpy's old behaviour, - # so encode the string again before passing to the user converter - def tobytes_first(x, conv): - if type(x) is bytes: - return conv(x) - return conv(x.encode("latin1")) - import functools - user_conv = functools.partial(tobytes_first, conv=conv) - else: - user_conv = conv - converters[i].update(user_conv, locked=True, - testing_value=testing_value, - default=filling_values[i], - missing_values=missing_values[i],) - uc_update.append((i, user_conv)) - # Make sure we have the corrected keys in user_converters... - user_converters.update(uc_update) - - # Fixme: possible error as following variable never used. - # miss_chars = [_.missing_values for _ in converters] - - # Initialize the output lists ... - # ... rows - rows = [] - append_to_rows = rows.append - # ... masks - if usemask: - masks = [] - append_to_masks = masks.append - # ... invalid - invalid = [] - append_to_invalid = invalid.append - - # Parse each line - for (i, line) in enumerate(itertools.chain([first_line, ], fhd)): - values = split_line(line) - nbvalues = len(values) - # Skip an empty line - if nbvalues == 0: - continue - if usecols: - # Select only the columns we need - try: - values = [values[_] for _ in usecols] - except IndexError: + if usecols: + # Select only the columns we need + try: + values = [values[_] for _ in usecols] + except IndexError: + append_to_invalid((i + skip_header + 1, nbvalues)) + continue + elif nbvalues != nbcols: append_to_invalid((i + skip_header + 1, nbvalues)) continue - elif nbvalues != nbcols: - append_to_invalid((i + skip_header + 1, nbvalues)) - continue - # Store the values - append_to_rows(tuple(values)) - if usemask: - append_to_masks(tuple([v.strip() in m - for (v, m) in zip(values, - missing_values)])) - if len(rows) == max_rows: - break - - if own_fhd: - fhd.close() + # Store the values + append_to_rows(tuple(values)) + if usemask: + append_to_masks(tuple([v.strip() in m + for (v, m) in zip(values, + missing_values)])) + if len(rows) == max_rows: + break # Upgrade the converters (if needed) if dtype is None: @@ -2147,7 +2182,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, outputmask = np.array(masks, dtype=mdtype) else: # Overwrite the initial dtype names if needed - if names and dtype.names: + if names and dtype.names is not None: dtype.names = names # Case 1. We have a structured type if len(dtype_flat) > 1: @@ -2197,7 +2232,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, # output = np.array(data, dtype) if usemask: - if dtype.names: + if dtype.names is not None: mdtype = [(_, bool) for _ in dtype.names] else: mdtype = bool @@ -2223,6 +2258,12 @@ def ndfromtxt(fname, **kwargs): """ Load ASCII data stored in a file and return it as a single array. + .. deprecated:: 1.17 + ndfromtxt` is a deprecated alias of `genfromtxt` which + overwrites the ``usemask`` argument with `False` even when + explicitly called as ``ndfromtxt(..., usemask=True)``. + Use `genfromtxt` instead. + Parameters ---------- fname, kwargs : For a description of input parameters, see `genfromtxt`. @@ -2233,6 +2274,11 @@ def ndfromtxt(fname, **kwargs): """ kwargs['usemask'] = False + # Numpy 1.17 + warnings.warn( + "np.ndfromtxt is a deprecated alias of np.genfromtxt, " + "prefer the latter.", + DeprecationWarning, stacklevel=2) return genfromtxt(fname, **kwargs) @@ -2240,6 +2286,12 @@ def mafromtxt(fname, **kwargs): """ Load ASCII data stored in a text file and return a masked array. + .. deprecated:: 1.17 + np.mafromtxt is a deprecated alias of `genfromtxt` which + overwrites the ``usemask`` argument with `True` even when + explicitly called as ``mafromtxt(..., usemask=False)``. + Use `genfromtxt` instead. + Parameters ---------- fname, kwargs : For a description of input parameters, see `genfromtxt`. @@ -2250,6 +2302,11 @@ def mafromtxt(fname, **kwargs): """ kwargs['usemask'] = True + # Numpy 1.17 + warnings.warn( + "np.mafromtxt is a deprecated alias of np.genfromtxt, " + "prefer the latter.", + DeprecationWarning, stacklevel=2) return genfromtxt(fname, **kwargs) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index b55764b5d..3d07a0de4 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -479,10 +479,10 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): coefficients for `k`-th data set are in ``p[:,k]``. residuals, rank, singular_values, rcond - Present only if `full` = True. Residuals of the least-squares fit, - the effective rank of the scaled Vandermonde coefficient matrix, - its singular values, and the specified value of `rcond`. For more - details, see `linalg.lstsq`. + Present only if `full` = True. Residuals is sum of squared residuals + of the least-squares fit, the effective rank of the scaled Vandermonde + coefficient matrix, its singular values, and the specified value of + `rcond`. For more details, see `linalg.lstsq`. V : ndarray, shape (M,M) or (M,M,K) Present only if `full` = False and `cov`=True. The covariance @@ -548,6 +548,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): Examples -------- + >>> import warnings >>> x = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]) >>> y = np.array([0.0, 0.8, 0.9, 0.1, -0.8, -1.0]) >>> z = np.polyfit(x, y, 3) @@ -566,9 +567,10 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): High-order polynomials may oscillate wildly: - >>> p30 = np.poly1d(np.polyfit(x, y, 30)) - ... - >>> # RankWarning: Polyfit may be poorly conditioned... + >>> with warnings.catch_warnings(): + ... warnings.simplefilter('ignore', np.RankWarning) + ... p30 = np.poly1d(np.polyfit(x, y, 30)) + ... >>> p30(4) -0.80000000000000204 # may vary >>> p30(5) @@ -632,7 +634,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): # warn on rank reduction, which indicates an ill conditioned matrix if rank != order and not full: msg = "Polyfit may be poorly conditioned" - warnings.warn(msg, RankWarning, stacklevel=2) + warnings.warn(msg, RankWarning, stacklevel=4) if full: return c, resids, rank, s, rcond @@ -874,8 +876,7 @@ def polymul(a1, a2): See Also -------- poly1d : A one-dimensional polynomial class. - poly, polyadd, polyder, polydiv, polyfit, polyint, polysub, - polyval + poly, polyadd, polyder, polydiv, polyfit, polyint, polysub, polyval convolve : Array convolution. Same output as polymul, but has parameter for overlap mode. @@ -937,7 +938,7 @@ def polydiv(u, v): See Also -------- - poly, polyadd, polyder, polydiv, polyfit, polyint, polymul, polysub, + poly, polyadd, polyder, polydiv, polyfit, polyint, polymul, polysub polyval Notes @@ -1110,8 +1111,14 @@ class poly1d(object): @property def coeffs(self): - """ A copy of the polynomial coefficients """ - return self._coeffs.copy() + """ The polynomial coefficients """ + return self._coeffs + + @coeffs.setter + def coeffs(self, value): + # allowing this makes p.coeffs *= 2 legal + if value is not self._coeffs: + raise AttributeError("Cannot set attribute") @property def variable(self): diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index 5ff35f0bb..927161ddb 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -26,10 +26,13 @@ _check_fill_value = np.ma.core._check_fill_value __all__ = [ - 'append_fields', 'drop_fields', 'find_duplicates', - 'get_fieldstructure', 'join_by', 'merge_arrays', - 'rec_append_fields', 'rec_drop_fields', 'rec_join', - 'recursive_fill_fields', 'rename_fields', 'stack_arrays', + 'append_fields', 'apply_along_fields', 'assign_fields_by_name', + 'drop_fields', 'find_duplicates', 'flatten_descr', + 'get_fieldstructure', 'get_names', 'get_names_flat', + 'join_by', 'merge_arrays', 'rec_append_fields', + 'rec_drop_fields', 'rec_join', 'recursive_fill_fields', + 'rename_fields', 'repack_fields', 'require_fields', + 'stack_arrays', 'structured_to_unstructured', 'unstructured_to_structured', ] @@ -69,14 +72,14 @@ def recursive_fill_fields(input, output): current = input[field] except ValueError: continue - if current.dtype.names: + if current.dtype.names is not None: recursive_fill_fields(current, output[field]) else: output[field][:len(current)] = current return output -def get_fieldspec(dtype): +def _get_fieldspec(dtype): """ Produce a list of name/dtype pairs corresponding to the dtype fields @@ -91,7 +94,7 @@ def get_fieldspec(dtype): >>> dt = np.dtype([(('a', 'A'), np.int64), ('b', np.double, 3)]) >>> dt.descr [(('a', 'A'), '<i8'), ('b', '<f8', (3,))] - >>> get_fieldspec(dt) + >>> _get_fieldspec(dt) [(('a', 'A'), dtype('int64')), ('b', dtype(('<f8', (3,))))] """ @@ -136,17 +139,17 @@ def get_names(adtype): names = adtype.names for name in names: current = adtype[name] - if current.names: + if current.names is not None: listnames.append((name, tuple(get_names(current)))) else: listnames.append(name) - return tuple(listnames) or None + return tuple(listnames) def get_names_flat(adtype): """ Returns the field names of the input datatype as a tuple. Nested structure - are flattend beforehand. + are flattened beforehand. Parameters ---------- @@ -173,9 +176,9 @@ def get_names_flat(adtype): for name in names: listnames.append(name) current = adtype[name] - if current.names: + if current.names is not None: listnames.extend(get_names_flat(current)) - return tuple(listnames) or None + return tuple(listnames) def flatten_descr(ndtype): @@ -197,19 +200,14 @@ def flatten_descr(ndtype): descr = [] for field in names: (typ, _) = ndtype.fields[field] - if typ.names: + if typ.names is not None: descr.extend(flatten_descr(typ)) else: descr.append((field, typ)) return tuple(descr) -def _zip_dtype_dispatcher(seqarrays, flatten=None): - return seqarrays - - -@array_function_dispatch(_zip_dtype_dispatcher) -def zip_dtype(seqarrays, flatten=False): +def _zip_dtype(seqarrays, flatten=False): newdtype = [] if flatten: for a in seqarrays: @@ -217,16 +215,15 @@ def zip_dtype(seqarrays, flatten=False): else: for a in seqarrays: current = a.dtype - if current.names and len(current.names) <= 1: - # special case - dtypes of 0 or 1 field are flattened - newdtype.extend(get_fieldspec(current)) + if current.names is not None and len(current.names) == 1: + # special case - dtypes of 1 field are flattened + newdtype.extend(_get_fieldspec(current)) else: newdtype.append(('', current)) return np.dtype(newdtype) -@array_function_dispatch(_zip_dtype_dispatcher) -def zip_descr(seqarrays, flatten=False): +def _zip_descr(seqarrays, flatten=False): """ Combine the dtype description of a series of arrays. @@ -237,7 +234,7 @@ def zip_descr(seqarrays, flatten=False): flatten : {boolean}, optional Whether to collapse nested descriptions. """ - return zip_dtype(seqarrays, flatten=flatten).descr + return _zip_dtype(seqarrays, flatten=flatten).descr def get_fieldstructure(adtype, lastname=None, parents=None,): @@ -271,7 +268,7 @@ def get_fieldstructure(adtype, lastname=None, parents=None,): names = adtype.names for name in names: current = adtype[name] - if current.names: + if current.names is not None: if lastname: parents[name] = [lastname, ] else: @@ -284,7 +281,7 @@ def get_fieldstructure(adtype, lastname=None, parents=None,): elif lastname: lastparent = [lastname, ] parents[name] = lastparent or [] - return parents or None + return parents def _izip_fields_flat(iterable): @@ -318,12 +315,7 @@ def _izip_fields(iterable): yield element -def _izip_records_dispatcher(seqarrays, fill_value=None, flatten=None): - return seqarrays - - -@array_function_dispatch(_izip_records_dispatcher) -def izip_records(seqarrays, fill_value=None, flatten=True): +def _izip_records(seqarrays, fill_value=None, flatten=True): """ Returns an iterator of concatenated items from a sequence of arrays. @@ -443,9 +435,9 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, if isinstance(seqarrays, (ndarray, np.void)): seqdtype = seqarrays.dtype # Make sure we have named fields - if not seqdtype.names: + if seqdtype.names is None: seqdtype = np.dtype([('', seqdtype)]) - if not flatten or zip_dtype((seqarrays,), flatten=True) == seqdtype: + if not flatten or _zip_dtype((seqarrays,), flatten=True) == seqdtype: # Minimal processing needed: just make sure everythng's a-ok seqarrays = seqarrays.ravel() # Find what type of array we must return @@ -468,7 +460,7 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, sizes = tuple(a.size for a in seqarrays) maxlength = max(sizes) # Get the dtype of the output (flattening if needed) - newdtype = zip_dtype(seqarrays, flatten=flatten) + newdtype = _zip_dtype(seqarrays, flatten=flatten) # Initialize the sequences for data and mask seqdata = [] seqmask = [] @@ -496,9 +488,9 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, seqdata.append(itertools.chain(data, [fval] * nbmissing)) seqmask.append(itertools.chain(mask, [fmsk] * nbmissing)) # Create an iterator for the data - data = tuple(izip_records(seqdata, flatten=flatten)) + data = tuple(_izip_records(seqdata, flatten=flatten)) output = ma.array(np.fromiter(data, dtype=newdtype, count=maxlength), - mask=list(izip_records(seqmask, flatten=flatten))) + mask=list(_izip_records(seqmask, flatten=flatten))) if asrecarray: output = output.view(MaskedRecords) else: @@ -516,7 +508,7 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, else: fval = None seqdata.append(itertools.chain(data, [fval] * nbmissing)) - output = np.fromiter(tuple(izip_records(seqdata, flatten=flatten)), + output = np.fromiter(tuple(_izip_records(seqdata, flatten=flatten)), dtype=newdtype, count=maxlength) if asrecarray: output = output.view(recarray) @@ -535,6 +527,10 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): Nested fields are supported. + ..versionchanged: 1.18.0 + `drop_fields` returns an array with 0 fields if all fields are dropped, + rather than returning ``None`` as it did previously. + Parameters ---------- base : array @@ -574,7 +570,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): current = ndtype[name] if name in drop_names: continue - if current.names: + if current.names is not None: descr = _drop_descr(current, drop_names) if descr: newdtype.append((name, descr)) @@ -583,8 +579,6 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): return newdtype newdtype = _drop_descr(base.dtype, drop_names) - if not newdtype: - return None output = np.empty(base.shape, dtype=newdtype) output = recursive_fill_fields(base, output) @@ -661,7 +655,7 @@ def rename_fields(base, namemapper): for name in ndtype.names: newname = namemapper.get(name, name) current = ndtype[name] - if current.names: + if current.names is not None: newdtype.append( (newname, _recursive_rename_fields(current, namemapper)) ) @@ -743,7 +737,7 @@ def append_fields(base, names, data, dtypes=None, # output = ma.masked_all( max(len(base), len(data)), - dtype=get_fieldspec(base.dtype) + get_fieldspec(data.dtype)) + dtype=_get_fieldspec(base.dtype) + _get_fieldspec(data.dtype)) output = recursive_fill_fields(base, output) output = recursive_fill_fields(data, output) # @@ -834,17 +828,18 @@ def repack_fields(a, align=False, recurse=False): Examples -------- + >>> from numpy.lib import recfunctions as rfn >>> def print_offsets(d): ... print("offsets:", [d.fields[name][1] for name in d.names]) ... print("itemsize:", d.itemsize) ... - >>> dt = np.dtype('u1,<i4,<f4', align=True) + >>> dt = np.dtype('u1, <i8, <f8', align=True) >>> dt dtype({'names':['f0','f1','f2'], 'formats':['u1','<i8','<f8'], 'offsets':[0,8,16], 'itemsize':24}, align=True) >>> print_offsets(dt) offsets: [0, 8, 16] itemsize: 24 - >>> packed_dt = repack_fields(dt) + >>> packed_dt = rfn.repack_fields(dt) >>> packed_dt dtype([('f0', 'u1'), ('f1', '<i8'), ('f2', '<f8')]) >>> print_offsets(packed_dt) @@ -881,16 +876,35 @@ def _get_fields_and_offsets(dt, offset=0): scalar fields in the dtype "dt", including nested fields, in left to right order. """ + + # counts up elements in subarrays, including nested subarrays, and returns + # base dtype and count + def count_elem(dt): + count = 1 + while dt.shape != (): + for size in dt.shape: + count *= size + dt = dt.base + return dt, count + fields = [] for name in dt.names: field = dt.fields[name] - if field[0].names is None: - count = 1 - for size in field[0].shape: - count *= size - fields.append((field[0], count, field[1] + offset)) + f_dt, f_offset = field[0], field[1] + f_dt, n = count_elem(f_dt) + + if f_dt.names is None: + fields.append((np.dtype((f_dt, (n,))), n, f_offset + offset)) else: - fields.extend(_get_fields_and_offsets(field[0], field[1] + offset)) + subfields = _get_fields_and_offsets(f_dt, f_offset + offset) + size = f_dt.itemsize + + for i in range(n): + if i == 0: + # optimization: avoid list comprehension if no subarray + fields.extend(subfields) + else: + fields.extend([(d, c, o + i*size) for d, c, o in subfields]) return fields @@ -932,12 +946,13 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): Examples -------- + >>> from numpy.lib import recfunctions as rfn >>> a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) >>> a array([(0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.])], dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) - >>> structured_to_unstructured(arr) + >>> rfn.structured_to_unstructured(a) array([[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], @@ -945,7 +960,7 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) - >>> np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + >>> np.mean(rfn.structured_to_unstructured(b[['x', 'z']]), axis=-1) array([ 3. , 5.5, 9. , 11. ]) """ @@ -954,6 +969,12 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): fields = _get_fields_and_offsets(arr.dtype) n_fields = len(fields) + if n_fields == 0 and dtype is None: + raise ValueError("arr has no fields. Unable to guess dtype") + elif n_fields == 0: + # too many bugs elsewhere for this to work now + raise NotImplementedError("arr with no fields is not supported") + dts, counts, offsets = zip(*fields) names = ['f{}'.format(n) for n in range(n_fields)] @@ -976,11 +997,12 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): # next cast to a packed format with all fields converted to new dtype packed_fields = np.dtype({'names': names, - 'formats': [(out_dtype, c) for c in counts]}) + 'formats': [(out_dtype, dt.shape) for dt in dts]}) arr = arr.astype(packed_fields, copy=copy, casting=casting) # finally is it safe to view the packed fields as the unstructured type - return arr.view((out_dtype, sum(counts))) + return arr.view((out_dtype, (sum(counts),))) + def _unstructured_to_structured_dispatcher(arr, dtype=None, names=None, align=None, copy=None, casting=None): @@ -1027,6 +1049,7 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, Examples -------- + >>> from numpy.lib import recfunctions as rfn >>> dt = np.dtype([('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) >>> a = np.arange(20).reshape((4,5)) >>> a @@ -1034,7 +1057,7 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, [ 5, 6, 7, 8, 9], [10, 11, 12, 13, 14], [15, 16, 17, 18, 19]]) - >>> unstructured_to_structured(a, dt) + >>> rfn.unstructured_to_structured(a, dt) array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])], dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) @@ -1043,6 +1066,9 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, if arr.shape == (): raise ValueError('arr must have at least one dimension') n_elem = arr.shape[-1] + if n_elem == 0: + # too many bugs elsewhere for this to work now + raise NotImplementedError("last axis with size 0 is not supported") if dtype is None: if names is None: @@ -1055,7 +1081,11 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, raise ValueError("don't supply both dtype and names") # sanity check of the input dtype fields = _get_fields_and_offsets(dtype) - dts, counts, offsets = zip(*fields) + if len(fields) == 0: + dts, counts, offsets = [], [], [] + else: + dts, counts, offsets = zip(*fields) + if n_elem != sum(counts): raise ValueError('The length of the last dimension of arr must ' 'be equal to the number of fields in dtype') @@ -1069,7 +1099,7 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, # first view as a packed structured array of one dtype packed_fields = np.dtype({'names': names, - 'formats': [(arr.dtype, c) for c in counts]}) + 'formats': [(arr.dtype, dt.shape) for dt in dts]}) arr = np.ascontiguousarray(arr).view(packed_fields) # next cast to an unpacked but flattened format with varied dtypes @@ -1111,11 +1141,12 @@ def apply_along_fields(func, arr): Examples -------- + >>> from numpy.lib import recfunctions as rfn >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) - >>> apply_along_fields(np.mean, b) + >>> rfn.apply_along_fields(np.mean, b) array([ 2.66666667, 5.33333333, 8.66666667, 11. ]) - >>> apply_along_fields(np.mean, b[['x', 'z']]) + >>> rfn.apply_along_fields(np.mean, b[['x', 'z']]) array([ 3. , 5.5, 9. , 11. ]) """ @@ -1200,14 +1231,15 @@ def require_fields(array, required_dtype): Examples -------- + >>> from numpy.lib import recfunctions as rfn >>> a = np.ones(4, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) - >>> require_fields(a, [('b', 'f4'), ('c', 'u1')]) + >>> rfn.require_fields(a, [('b', 'f4'), ('c', 'u1')]) array([(1., 1), (1., 1), (1., 1), (1., 1)], dtype=[('b', '<f4'), ('c', 'u1')]) - >>> require_fields(a, [('b', 'f4'), ('newf', 'u1')]) + >>> rfn.require_fields(a, [('b', 'f4'), ('newf', 'u1')]) array([(1., 0), (1., 0), (1., 0), (1., 0)], dtype=[('b', '<f4'), ('newf', 'u1')]) - + """ out = np.empty(array.shape, dtype=required_dtype) assign_fields_by_name(out, array) @@ -1270,10 +1302,10 @@ def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, fldnames = [d.names for d in ndtype] # dtype_l = ndtype[0] - newdescr = get_fieldspec(dtype_l) + newdescr = _get_fieldspec(dtype_l) names = [n for n, d in newdescr] for dtype_n in ndtype[1:]: - for fname, fdtype in get_fieldspec(dtype_n): + for fname, fdtype in _get_fieldspec(dtype_n): if fname not in names: newdescr.append((fname, fdtype)) names.append(fname) @@ -1496,15 +1528,15 @@ def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', # # Build the new description of the output array ....... # Start with the key fields - ndtype = get_fieldspec(r1k.dtype) + ndtype = _get_fieldspec(r1k.dtype) # Add the fields from r1 - for fname, fdtype in get_fieldspec(r1.dtype): + for fname, fdtype in _get_fieldspec(r1.dtype): if fname not in key: ndtype.append((fname, fdtype)) # Add the fields from r2 - for fname, fdtype in get_fieldspec(r2.dtype): + for fname, fdtype in _get_fieldspec(r2.dtype): # Have we seen the current name already ? # we need to rebuild this list every time names = list(name for name, dtype in ndtype) diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index e088a6c4a..dbb61c225 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -1,18 +1,17 @@ from __future__ import division, absolute_import, print_function import functools -import warnings import numpy.core.numeric as _nx from numpy.core.numeric import ( asarray, zeros, outer, concatenate, array, asanyarray ) -from numpy.core.fromnumeric import product, reshape, transpose +from numpy.core.fromnumeric import reshape, transpose from numpy.core.multiarray import normalize_axis_index from numpy.core import overrides from numpy.core import vstack, atleast_3d -from numpy.core.shape_base import ( - _arrays_for_stack_dispatcher, _warn_for_nonsequence) +from numpy.core.numeric import normalize_axis_tuple +from numpy.core.shape_base import _arrays_for_stack_dispatcher from numpy.lib.index_tricks import ndindex from numpy.matrixlib.defmatrix import matrix # this raises all the right alarm bells @@ -30,7 +29,7 @@ array_function_dispatch = functools.partial( def _make_along_axis_idx(arr_shape, indices, axis): - # compute dimensions to iterate over + # compute dimensions to iterate over if not _nx.issubdtype(indices.dtype, _nx.integer): raise IndexError('`indices` must be an integer array') if len(arr_shape) != indices.ndim: @@ -95,7 +94,7 @@ def take_along_axis(arr, indices, axis): Ni, M, Nk = a.shape[:axis], a.shape[axis], a.shape[axis+1:] J = indices.shape[axis] # Need not equal M - out = np.empty(Nk + (J,) + Nk) + out = np.empty(Ni + (J,) + Nk) for ii in ndindex(Ni): for kk in ndindex(Nk): @@ -518,23 +517,26 @@ def expand_dims(a, axis): Insert a new axis that will appear at the `axis` position in the expanded array shape. - .. note:: Previous to NumPy 1.13.0, neither ``axis < -a.ndim - 1`` nor - ``axis > a.ndim`` raised errors or put the new axis where documented. - Those axis values are now deprecated and will raise an AxisError in the - future. - Parameters ---------- a : array_like Input array. - axis : int - Position in the expanded axes where the new axis is placed. + axis : int or tuple of ints + Position in the expanded axes where the new axis (or axes) is placed. + + .. deprecated:: 1.13.0 + Passing an axis where ``axis > a.ndim`` will be treated as + ``axis == a.ndim``, and passing ``axis < -a.ndim - 1`` will + be treated as ``axis == 0``. This behavior is deprecated. + + .. versionchanged:: 1.18.0 + A tuple of axes is now supported. Out of range axes as + described above are now forbidden and raise an `AxisError`. Returns ------- - res : ndarray - Output array. The number of dimensions is one greater than that of - the input array. + result : ndarray + View of `a` with the number of dimensions increased. See Also -------- @@ -544,11 +546,11 @@ def expand_dims(a, axis): Examples -------- - >>> x = np.array([1,2]) + >>> x = np.array([1, 2]) >>> x.shape (2,) - The following is equivalent to ``x[np.newaxis,:]`` or ``x[np.newaxis]``: + The following is equivalent to ``x[np.newaxis, :]`` or ``x[np.newaxis]``: >>> y = np.expand_dims(x, axis=0) >>> y @@ -556,13 +558,26 @@ def expand_dims(a, axis): >>> y.shape (1, 2) - >>> y = np.expand_dims(x, axis=1) # Equivalent to x[:,np.newaxis] + The following is equivalent to ``x[:, np.newaxis]``: + + >>> y = np.expand_dims(x, axis=1) >>> y array([[1], [2]]) >>> y.shape (2, 1) + ``axis`` may also be a tuple: + + >>> y = np.expand_dims(x, axis=(0, 1)) + >>> y + array([[[1, 2]]]) + + >>> y = np.expand_dims(x, axis=(2, 0)) + >>> y + array([[[1], + [2]]]) + Note that some examples may use ``None`` instead of ``np.newaxis``. These are the same objects: @@ -575,18 +590,16 @@ def expand_dims(a, axis): else: a = asanyarray(a) - shape = a.shape - if axis > a.ndim or axis < -a.ndim - 1: - # 2017-05-17, 1.13.0 - warnings.warn("Both axis > a.ndim and axis < -a.ndim - 1 are " - "deprecated and will raise an AxisError in the future.", - DeprecationWarning, stacklevel=2) - # When the deprecation period expires, delete this if block, - if axis < 0: - axis = axis + a.ndim + 1 - # and uncomment the following line. - # axis = normalize_axis_index(axis, a.ndim + 1) - return a.reshape(shape[:axis] + (1,) + shape[axis:]) + if type(axis) not in (tuple, list): + axis = (axis,) + + out_ndim = len(axis) + a.ndim + axis = normalize_axis_tuple(axis, out_ndim) + + shape_it = iter(a.shape) + shape = [1 if ax in axis else next(shape_it) for ax in range(out_ndim)] + + return a.reshape(shape) row_stack = vstack @@ -630,7 +643,10 @@ def column_stack(tup): [3, 4]]) """ - _warn_for_nonsequence(tup) + if not overrides.ARRAY_FUNCTION_ENABLED: + # raise warning if necessary + _arrays_for_stack_dispatcher(tup, stacklevel=2) + arrays = [] for v in tup: arr = array(v, copy=False, subok=True) @@ -695,8 +711,14 @@ def dstack(tup): [[3, 4]]]) """ - _warn_for_nonsequence(tup) - return _nx.concatenate([atleast_3d(_m) for _m in tup], 2) + if not overrides.ARRAY_FUNCTION_ENABLED: + # raise warning if necessary + _arrays_for_stack_dispatcher(tup, stacklevel=2) + + arrs = atleast_3d(*tup) + if not isinstance(arrs, list): + arrs = [arrs] + return _nx.concatenate(arrs, 2) def _replace_zero_by_x_arrays(sub_arys): @@ -775,7 +797,7 @@ def _split_dispatcher(ary, indices_or_sections, axis=None): @array_function_dispatch(_split_dispatcher) def split(ary, indices_or_sections, axis=0): """ - Split an array into multiple sub-arrays. + Split an array into multiple sub-arrays as views into `ary`. Parameters ---------- @@ -802,7 +824,7 @@ def split(ary, indices_or_sections, axis=0): Returns ------- sub-arrays : list of ndarrays - A list of sub-arrays. + A list of sub-arrays as views into `ary`. Raises ------ @@ -847,8 +869,7 @@ def split(ary, indices_or_sections, axis=0): if N % sections: raise ValueError( 'array split does not result in an equal division') - res = array_split(ary, indices_or_sections, axis) - return res + return array_split(ary, indices_or_sections, axis) def _hvdsplit_dispatcher(ary, indices_or_sections): diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index 0dc36e41c..8aafd094b 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -121,18 +121,18 @@ def _broadcast_to(array, shape, subok, readonly): if any(size < 0 for size in shape): raise ValueError('all elements of broadcast shape must be non-' 'negative') - needs_writeable = not readonly and array.flags.writeable - extras = ['reduce_ok'] if needs_writeable else [] - op_flag = 'readwrite' if needs_writeable else 'readonly' + extras = [] it = np.nditer( (array,), flags=['multi_index', 'refs_ok', 'zerosize_ok'] + extras, - op_flags=[op_flag], itershape=shape, order='C') + op_flags=['readonly'], itershape=shape, order='C') with it: # never really has writebackifcopy semantics broadcast = it.itviews[0] result = _maybe_view_as_subclass(array, broadcast) - if needs_writeable and not result.flags.writeable: + # In a future version this will go away + if not readonly and array.flags._writeable_no_warn: result.flags.writeable = True + result.flags._warn_on_write = True return result @@ -186,8 +186,6 @@ def _broadcast_shape(*args): """Returns the shape of the arrays that would result from broadcasting the supplied arrays against each other. """ - if not args: - return () # use the old-iterator because np.nditer does not handle size 0 arrays # consistently b = np.broadcast(*args[:32]) @@ -224,8 +222,15 @@ def broadcast_arrays(*args, **kwargs): broadcasted : list of arrays These arrays are views on the original arrays. They are typically not contiguous. Furthermore, more than one element of a - broadcasted array may refer to a single memory location. If you - need to write to the arrays, make copies first. + broadcasted array may refer to a single memory location. If you need + to write to the arrays, make copies first. While you can set the + ``writable`` flag True, writing to a single output value may end up + changing more than one location in the output array. + + .. deprecated:: 1.17 + The output is currently marked so that if written to, a deprecation + warning will be emitted. A future version will set the + ``writable`` flag False so writing to it will raise an error. Examples -------- @@ -262,7 +267,5 @@ def broadcast_arrays(*args, **kwargs): # Common case where nothing needs to be broadcasted. return args - # TODO: consider making the results of broadcast_arrays readonly to match - # broadcast_to. This will require a deprecation cycle. return [_broadcast_to(array, shape, subok=subok, readonly=False) for array in args] diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index e04fdc808..15cd3ad9d 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -204,14 +204,18 @@ class TestStringConverter(object): def test_upgrademapper(self): "Tests updatemapper" dateparser = _bytes_to_date - StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) - convert = StringConverter(dateparser, date(2000, 1, 1)) - test = convert('2001-01-01') - assert_equal(test, date(2001, 1, 1)) - test = convert('2009-01-01') - assert_equal(test, date(2009, 1, 1)) - test = convert('') - assert_equal(test, date(2000, 1, 1)) + _original_mapper = StringConverter._mapper[:] + try: + StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) + convert = StringConverter(dateparser, date(2000, 1, 1)) + test = convert('2001-01-01') + assert_equal(test, date(2001, 1, 1)) + test = convert('2009-01-01') + assert_equal(test, date(2009, 1, 1)) + test = convert('') + assert_equal(test, date(2000, 1, 1)) + finally: + StringConverter._mapper = _original_mapper def test_string_to_object(self): "Make sure that string-to-object functions are properly recognized" diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index 20f6e4a1b..1c3507a62 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -6,14 +6,32 @@ from __future__ import division, absolute_import, print_function import pytest import numpy as np -from numpy.testing import (assert_array_equal, assert_raises, assert_allclose, - assert_equal) -from numpy.lib import pad +from numpy.testing import assert_array_equal, assert_allclose, assert_equal from numpy.lib.arraypad import _as_pairs -class TestAsPairs(object): +_numeric_dtypes = ( + np.sctypes["uint"] + + np.sctypes["int"] + + np.sctypes["float"] + + np.sctypes["complex"] +) +_all_modes = { + 'constant': {'constant_values': 0}, + 'edge': {}, + 'linear_ramp': {'end_values': 0}, + 'maximum': {'stat_length': None}, + 'mean': {'stat_length': None}, + 'median': {'stat_length': None}, + 'minimum': {'stat_length': None}, + 'reflect': {'reflect_type': 'even'}, + 'symmetric': {'reflect_type': 'even'}, + 'wrap': {}, + 'empty': {} +} + +class TestAsPairs(object): def test_single_value(self): """Test casting for a single value.""" expected = np.array([[3, 3]] * 10) @@ -97,52 +115,31 @@ class TestAsPairs(object): class TestConditionalShortcuts(object): - def test_zero_padding_shortcuts(self): + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_zero_padding_shortcuts(self, mode): test = np.arange(120).reshape(4, 5, 6) - pad_amt = [(0, 0) for axis in test.shape] - modes = ['constant', - 'edge', - 'linear_ramp', - 'maximum', - 'mean', - 'median', - 'minimum', - 'reflect', - 'symmetric', - 'wrap', - ] - for mode in modes: - assert_array_equal(test, pad(test, pad_amt, mode=mode)) - - def test_shallow_statistic_range(self): + pad_amt = [(0, 0) for _ in test.shape] + assert_array_equal(test, np.pad(test, pad_amt, mode=mode)) + + @pytest.mark.parametrize("mode", ['maximum', 'mean', 'median', 'minimum',]) + def test_shallow_statistic_range(self, mode): test = np.arange(120).reshape(4, 5, 6) - pad_amt = [(1, 1) for axis in test.shape] - modes = ['maximum', - 'mean', - 'median', - 'minimum', - ] - for mode in modes: - assert_array_equal(pad(test, pad_amt, mode='edge'), - pad(test, pad_amt, mode=mode, stat_length=1)) - - def test_clip_statistic_range(self): + pad_amt = [(1, 1) for _ in test.shape] + assert_array_equal(np.pad(test, pad_amt, mode='edge'), + np.pad(test, pad_amt, mode=mode, stat_length=1)) + + @pytest.mark.parametrize("mode", ['maximum', 'mean', 'median', 'minimum',]) + def test_clip_statistic_range(self, mode): test = np.arange(30).reshape(5, 6) - pad_amt = [(3, 3) for axis in test.shape] - modes = ['maximum', - 'mean', - 'median', - 'minimum', - ] - for mode in modes: - assert_array_equal(pad(test, pad_amt, mode=mode), - pad(test, pad_amt, mode=mode, stat_length=30)) + pad_amt = [(3, 3) for _ in test.shape] + assert_array_equal(np.pad(test, pad_amt, mode=mode), + np.pad(test, pad_amt, mode=mode, stat_length=30)) class TestStatistic(object): def test_check_mean_stat_length(self): a = np.arange(100).astype('f') - a = pad(a, ((25, 20), ), 'mean', stat_length=((2, 3), )) + a = np.pad(a, ((25, 20), ), 'mean', stat_length=((2, 3), )) b = np.array( [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, @@ -166,7 +163,7 @@ class TestStatistic(object): def test_check_maximum_1(self): a = np.arange(100) - a = pad(a, (25, 20), 'maximum') + a = np.pad(a, (25, 20), 'maximum') b = np.array( [99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, @@ -190,7 +187,7 @@ class TestStatistic(object): def test_check_maximum_2(self): a = np.arange(100) + 1 - a = pad(a, (25, 20), 'maximum') + a = np.pad(a, (25, 20), 'maximum') b = np.array( [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, @@ -214,7 +211,7 @@ class TestStatistic(object): def test_check_maximum_stat_length(self): a = np.arange(100) + 1 - a = pad(a, (25, 20), 'maximum', stat_length=10) + a = np.pad(a, (25, 20), 'maximum', stat_length=10) b = np.array( [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, @@ -238,7 +235,7 @@ class TestStatistic(object): def test_check_minimum_1(self): a = np.arange(100) - a = pad(a, (25, 20), 'minimum') + a = np.pad(a, (25, 20), 'minimum') b = np.array( [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -262,7 +259,7 @@ class TestStatistic(object): def test_check_minimum_2(self): a = np.arange(100) + 2 - a = pad(a, (25, 20), 'minimum') + a = np.pad(a, (25, 20), 'minimum') b = np.array( [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -286,7 +283,7 @@ class TestStatistic(object): def test_check_minimum_stat_length(self): a = np.arange(100) + 1 - a = pad(a, (25, 20), 'minimum', stat_length=10) + a = np.pad(a, (25, 20), 'minimum', stat_length=10) b = np.array( [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -310,7 +307,7 @@ class TestStatistic(object): def test_check_median(self): a = np.arange(100).astype('f') - a = pad(a, (25, 20), 'median') + a = np.pad(a, (25, 20), 'median') b = np.array( [49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, @@ -334,7 +331,7 @@ class TestStatistic(object): def test_check_median_01(self): a = np.array([[3, 1, 4], [4, 5, 9], [9, 8, 2]]) - a = pad(a, 1, 'median') + a = np.pad(a, 1, 'median') b = np.array( [[4, 4, 5, 4, 4], @@ -348,7 +345,7 @@ class TestStatistic(object): def test_check_median_02(self): a = np.array([[3, 1, 4], [4, 5, 9], [9, 8, 2]]) - a = pad(a.T, 1, 'median').T + a = np.pad(a.T, 1, 'median').T b = np.array( [[5, 4, 5, 4, 5], @@ -364,7 +361,7 @@ class TestStatistic(object): a = np.arange(100).astype('f') a[1] = 2. a[97] = 96. - a = pad(a, (25, 20), 'median', stat_length=(3, 5)) + a = np.pad(a, (25, 20), 'median', stat_length=(3, 5)) b = np.array( [ 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., @@ -388,7 +385,7 @@ class TestStatistic(object): def test_check_mean_shape_one(self): a = [[4, 5, 6]] - a = pad(a, (5, 7), 'mean', stat_length=2) + a = np.pad(a, (5, 7), 'mean', stat_length=2) b = np.array( [[4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6], [4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6], @@ -410,7 +407,7 @@ class TestStatistic(object): def test_check_mean_2(self): a = np.arange(100).astype('f') - a = pad(a, (25, 20), 'mean') + a = np.pad(a, (25, 20), 'mean') b = np.array( [49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, 49.5, @@ -433,7 +430,7 @@ class TestStatistic(object): assert_array_equal(a, b) @pytest.mark.parametrize("mode", [ - pytest.param("mean", marks=pytest.mark.xfail(reason="gh-11216")), + "mean", "median", "minimum", "maximum" @@ -446,11 +443,65 @@ class TestStatistic(object): a = np.pad(a, (1, 1), mode) assert_equal(a[0], a[-1]) + @pytest.mark.parametrize("mode", ["mean", "median", "minimum", "maximum"]) + @pytest.mark.parametrize( + "stat_length", [-2, (-2,), (3, -1), ((5, 2), (-2, 3)), ((-4,), (2,))] + ) + def test_check_negative_stat_length(self, mode, stat_length): + arr = np.arange(30).reshape((6, 5)) + match = "index can't contain negative values" + with pytest.raises(ValueError, match=match): + np.pad(arr, 2, mode, stat_length=stat_length) + + def test_simple_stat_length(self): + a = np.arange(30) + a = np.reshape(a, (6, 5)) + a = np.pad(a, ((2, 3), (3, 2)), mode='mean', stat_length=(3,)) + b = np.array( + [[6, 6, 6, 5, 6, 7, 8, 9, 8, 8], + [6, 6, 6, 5, 6, 7, 8, 9, 8, 8], + + [1, 1, 1, 0, 1, 2, 3, 4, 3, 3], + [6, 6, 6, 5, 6, 7, 8, 9, 8, 8], + [11, 11, 11, 10, 11, 12, 13, 14, 13, 13], + [16, 16, 16, 15, 16, 17, 18, 19, 18, 18], + [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], + [26, 26, 26, 25, 26, 27, 28, 29, 28, 28], + + [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], + [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], + [21, 21, 21, 20, 21, 22, 23, 24, 23, 23]] + ) + assert_array_equal(a, b) + + @pytest.mark.filterwarnings("ignore:Mean of empty slice:RuntimeWarning") + @pytest.mark.filterwarnings( + "ignore:invalid value encountered in (true_divide|double_scalars):" + "RuntimeWarning" + ) + @pytest.mark.parametrize("mode", ["mean", "median"]) + def test_zero_stat_length_valid(self, mode): + arr = np.pad([1., 2.], (1, 2), mode, stat_length=0) + expected = np.array([np.nan, 1., 2., np.nan, np.nan]) + assert_equal(arr, expected) + + @pytest.mark.parametrize("mode", ["minimum", "maximum"]) + def test_zero_stat_length_invalid(self, mode): + match = "stat_length of 0 yields no value for padding" + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 0, mode, stat_length=0) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 0, mode, stat_length=(1, 0)) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 1, mode, stat_length=0) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 1, mode, stat_length=(1, 0)) + class TestConstant(object): def test_check_constant(self): a = np.arange(100) - a = pad(a, (25, 20), 'constant', constant_values=(10, 20)) + a = np.pad(a, (25, 20), 'constant', constant_values=(10, 20)) b = np.array( [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, @@ -474,7 +525,7 @@ class TestConstant(object): def test_check_constant_zeros(self): a = np.arange(100) - a = pad(a, (25, 20), 'constant') + a = np.pad(a, (25, 20), 'constant') b = np.array( [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -500,7 +551,7 @@ class TestConstant(object): # If input array is int, but constant_values are float, the dtype of # the array to be padded is kept arr = np.arange(30).reshape(5, 6) - test = pad(arr, (1, 2), mode='constant', + test = np.pad(arr, (1, 2), mode='constant', constant_values=1.1) expected = np.array( [[ 1, 1, 1, 1, 1, 1, 1, 1, 1], @@ -521,7 +572,7 @@ class TestConstant(object): # the array to be padded is kept - here retaining the float constants arr = np.arange(30).reshape(5, 6) arr_float = arr.astype(np.float64) - test = pad(arr_float, ((1, 2), (1, 2)), mode='constant', + test = np.pad(arr_float, ((1, 2), (1, 2)), mode='constant', constant_values=1.1) expected = np.array( [[ 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], @@ -539,7 +590,7 @@ class TestConstant(object): def test_check_constant_float3(self): a = np.arange(100, dtype=float) - a = pad(a, (25, 20), 'constant', constant_values=(-1.1, -1.2)) + a = np.pad(a, (25, 20), 'constant', constant_values=(-1.1, -1.2)) b = np.array( [-1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, -1.1, @@ -563,7 +614,7 @@ class TestConstant(object): def test_check_constant_odd_pad_amount(self): arr = np.arange(30).reshape(5, 6) - test = pad(arr, ((1,), (2,)), mode='constant', + test = np.pad(arr, ((1,), (2,)), mode='constant', constant_values=3) expected = np.array( [[ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], @@ -620,11 +671,16 @@ class TestConstant(object): assert_array_equal(arr, expected) + def test_pad_empty_dimension(self): + arr = np.zeros((3, 0, 2)) + result = np.pad(arr, [(0,), (2,), (1,)], mode="constant") + assert result.shape == (3, 4, 4) + class TestLinearRamp(object): def test_check_simple(self): a = np.arange(100).astype('f') - a = pad(a, (25, 20), 'linear_ramp', end_values=(4, 5)) + a = np.pad(a, (25, 20), 'linear_ramp', end_values=(4, 5)) b = np.array( [4.00, 3.84, 3.68, 3.52, 3.36, 3.20, 3.04, 2.88, 2.72, 2.56, 2.40, 2.24, 2.08, 1.92, 1.76, 1.60, 1.44, 1.28, 1.12, 0.96, @@ -648,7 +704,7 @@ class TestLinearRamp(object): def test_check_2d(self): arr = np.arange(20).reshape(4, 5).astype(np.float64) - test = pad(arr, (2, 2), mode='linear_ramp', end_values=(0, 0)) + test = np.pad(arr, (2, 2), mode='linear_ramp', end_values=(0, 0)) expected = np.array( [[0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0.5, 1., 1.5, 2., 1., 0.], @@ -679,11 +735,37 @@ class TestLinearRamp(object): ]) assert_equal(actual, expected) + def test_end_values(self): + """Ensure that end values are exact.""" + a = np.pad(np.ones(10).reshape(2, 5), (223, 123), mode="linear_ramp") + assert_equal(a[:, 0], 0.) + assert_equal(a[:, -1], 0.) + assert_equal(a[0, :], 0.) + assert_equal(a[-1, :], 0.) + + @pytest.mark.parametrize("dtype", _numeric_dtypes) + def test_negative_difference(self, dtype): + """ + Check correct behavior of unsigned dtypes if there is a negative + difference between the edge to pad and `end_values`. Check both cases + to be independent of implementation. Test behavior for all other dtypes + in case dtype casting interferes with complex dtypes. See gh-14191. + """ + x = np.array([3], dtype=dtype) + result = np.pad(x, 3, mode="linear_ramp", end_values=0) + expected = np.array([0, 1, 2, 3, 2, 1, 0], dtype=dtype) + assert_equal(result, expected) + + x = np.array([0], dtype=dtype) + result = np.pad(x, 3, mode="linear_ramp", end_values=3) + expected = np.array([3, 2, 1, 0, 1, 2, 3], dtype=dtype) + assert_equal(result, expected) + class TestReflect(object): def test_check_simple(self): a = np.arange(100) - a = pad(a, (25, 20), 'reflect') + a = np.pad(a, (25, 20), 'reflect') b = np.array( [25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, @@ -707,7 +789,7 @@ class TestReflect(object): def test_check_odd_method(self): a = np.arange(100) - a = pad(a, (25, 20), 'reflect', reflect_type='odd') + a = np.pad(a, (25, 20), 'reflect', reflect_type='odd') b = np.array( [-25, -24, -23, -22, -21, -20, -19, -18, -17, -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, @@ -731,7 +813,7 @@ class TestReflect(object): def test_check_large_pad(self): a = [[4, 5, 6], [6, 7, 8]] - a = pad(a, (5, 7), 'reflect') + a = np.pad(a, (5, 7), 'reflect') b = np.array( [[7, 6, 7, 8, 7, 6, 7, 8, 7, 6, 7, 8, 7, 6, 7], [5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5], @@ -754,7 +836,7 @@ class TestReflect(object): def test_check_shape(self): a = [[4, 5, 6]] - a = pad(a, (5, 7), 'reflect') + a = np.pad(a, (5, 7), 'reflect') b = np.array( [[5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5], [5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5, 6, 5, 4, 5], @@ -775,30 +857,49 @@ class TestReflect(object): assert_array_equal(a, b) def test_check_01(self): - a = pad([1, 2, 3], 2, 'reflect') + a = np.pad([1, 2, 3], 2, 'reflect') b = np.array([3, 2, 1, 2, 3, 2, 1]) assert_array_equal(a, b) def test_check_02(self): - a = pad([1, 2, 3], 3, 'reflect') + a = np.pad([1, 2, 3], 3, 'reflect') b = np.array([2, 3, 2, 1, 2, 3, 2, 1, 2]) assert_array_equal(a, b) def test_check_03(self): - a = pad([1, 2, 3], 4, 'reflect') + a = np.pad([1, 2, 3], 4, 'reflect') b = np.array([1, 2, 3, 2, 1, 2, 3, 2, 1, 2, 3]) assert_array_equal(a, b) - def test_check_padding_an_empty_array(self): - a = pad(np.zeros((0, 3)), ((0,), (1,)), mode='reflect') - b = np.zeros((0, 5)) - assert_array_equal(a, b) + +class TestEmptyArray(object): + """Check how padding behaves on arrays with an empty dimension.""" + + @pytest.mark.parametrize( + # Keep parametrization ordered, otherwise pytest-xdist might believe + # that different tests were collected during parallelization + "mode", sorted(_all_modes.keys() - {"constant", "empty"}) + ) + def test_pad_empty_dimension(self, mode): + match = ("can't extend empty axis 0 using modes other than 'constant' " + "or 'empty'") + with pytest.raises(ValueError, match=match): + np.pad([], 4, mode=mode) + with pytest.raises(ValueError, match=match): + np.pad(np.ndarray(0), 4, mode=mode) + with pytest.raises(ValueError, match=match): + np.pad(np.zeros((0, 3)), ((1,), (0,)), mode=mode) + + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_pad_non_empty_dimension(self, mode): + result = np.pad(np.ones((2, 0, 2)), ((3,), (0,), (1,)), mode=mode) + assert result.shape == (8, 0, 4) class TestSymmetric(object): def test_check_simple(self): a = np.arange(100) - a = pad(a, (25, 20), 'symmetric') + a = np.pad(a, (25, 20), 'symmetric') b = np.array( [24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, @@ -822,7 +923,7 @@ class TestSymmetric(object): def test_check_odd_method(self): a = np.arange(100) - a = pad(a, (25, 20), 'symmetric', reflect_type='odd') + a = np.pad(a, (25, 20), 'symmetric', reflect_type='odd') b = np.array( [-24, -23, -22, -21, -20, -19, -18, -17, -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, @@ -846,7 +947,7 @@ class TestSymmetric(object): def test_check_large_pad(self): a = [[4, 5, 6], [6, 7, 8]] - a = pad(a, (5, 7), 'symmetric') + a = np.pad(a, (5, 7), 'symmetric') b = np.array( [[5, 6, 6, 5, 4, 4, 5, 6, 6, 5, 4, 4, 5, 6, 6], [5, 6, 6, 5, 4, 4, 5, 6, 6, 5, 4, 4, 5, 6, 6], @@ -870,7 +971,7 @@ class TestSymmetric(object): def test_check_large_pad_odd(self): a = [[4, 5, 6], [6, 7, 8]] - a = pad(a, (5, 7), 'symmetric', reflect_type='odd') + a = np.pad(a, (5, 7), 'symmetric', reflect_type='odd') b = np.array( [[-3, -2, -2, -1, 0, 0, 1, 2, 2, 3, 4, 4, 5, 6, 6], [-3, -2, -2, -1, 0, 0, 1, 2, 2, 3, 4, 4, 5, 6, 6], @@ -893,7 +994,7 @@ class TestSymmetric(object): def test_check_shape(self): a = [[4, 5, 6]] - a = pad(a, (5, 7), 'symmetric') + a = np.pad(a, (5, 7), 'symmetric') b = np.array( [[5, 6, 6, 5, 4, 4, 5, 6, 6, 5, 4, 4, 5, 6, 6], [5, 6, 6, 5, 4, 4, 5, 6, 6, 5, 4, 4, 5, 6, 6], @@ -914,17 +1015,17 @@ class TestSymmetric(object): assert_array_equal(a, b) def test_check_01(self): - a = pad([1, 2, 3], 2, 'symmetric') + a = np.pad([1, 2, 3], 2, 'symmetric') b = np.array([2, 1, 1, 2, 3, 3, 2]) assert_array_equal(a, b) def test_check_02(self): - a = pad([1, 2, 3], 3, 'symmetric') + a = np.pad([1, 2, 3], 3, 'symmetric') b = np.array([3, 2, 1, 1, 2, 3, 3, 2, 1]) assert_array_equal(a, b) def test_check_03(self): - a = pad([1, 2, 3], 6, 'symmetric') + a = np.pad([1, 2, 3], 6, 'symmetric') b = np.array([1, 2, 3, 3, 2, 1, 1, 2, 3, 3, 2, 1, 1, 2, 3]) assert_array_equal(a, b) @@ -932,7 +1033,7 @@ class TestSymmetric(object): class TestWrap(object): def test_check_simple(self): a = np.arange(100) - a = pad(a, (25, 20), 'wrap') + a = np.pad(a, (25, 20), 'wrap') b = np.array( [75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, @@ -957,7 +1058,7 @@ class TestWrap(object): def test_check_large_pad(self): a = np.arange(12) a = np.reshape(a, (3, 4)) - a = pad(a, (10, 12), 'wrap') + a = np.pad(a, (10, 12), 'wrap') b = np.array( [[10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11], @@ -1015,12 +1116,12 @@ class TestWrap(object): assert_array_equal(a, b) def test_check_01(self): - a = pad([1, 2, 3], 3, 'wrap') + a = np.pad([1, 2, 3], 3, 'wrap') b = np.array([1, 2, 3, 1, 2, 3, 1, 2, 3]) assert_array_equal(a, b) def test_check_02(self): - a = pad([1, 2, 3], 4, 'wrap') + a = np.pad([1, 2, 3], 4, 'wrap') b = np.array([3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1]) assert_array_equal(a, b) @@ -1029,35 +1130,25 @@ class TestWrap(object): b = np.pad(a, (0, 5), mode="wrap") assert_array_equal(a, b[:-5, :-5]) + def test_repeated_wrapping(self): + """ + Check wrapping on each side individually if the wrapped area is longer + than the original array. + """ + a = np.arange(5) + b = np.pad(a, (12, 0), mode="wrap") + assert_array_equal(np.r_[a, a, a, a][3:], b) -class TestStatLen(object): - def test_check_simple(self): - a = np.arange(30) - a = np.reshape(a, (6, 5)) - a = pad(a, ((2, 3), (3, 2)), mode='mean', stat_length=(3,)) - b = np.array( - [[6, 6, 6, 5, 6, 7, 8, 9, 8, 8], - [6, 6, 6, 5, 6, 7, 8, 9, 8, 8], - - [1, 1, 1, 0, 1, 2, 3, 4, 3, 3], - [6, 6, 6, 5, 6, 7, 8, 9, 8, 8], - [11, 11, 11, 10, 11, 12, 13, 14, 13, 13], - [16, 16, 16, 15, 16, 17, 18, 19, 18, 18], - [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], - [26, 26, 26, 25, 26, 27, 28, 29, 28, 28], - - [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], - [21, 21, 21, 20, 21, 22, 23, 24, 23, 23], - [21, 21, 21, 20, 21, 22, 23, 24, 23, 23]] - ) - assert_array_equal(a, b) + a = np.arange(5) + b = np.pad(a, (0, 12), mode="wrap") + assert_array_equal(np.r_[a, a, a, a][:-3], b) class TestEdge(object): def test_check_simple(self): a = np.arange(12) a = np.reshape(a, (4, 3)) - a = pad(a, ((2, 3), (3, 2)), 'edge') + a = np.pad(a, ((2, 3), (3, 2)), 'edge') b = np.array( [[0, 0, 0, 0, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 2, 2], @@ -1077,56 +1168,128 @@ class TestEdge(object): # Check a pad_width of the form ((1, 2),). # Regression test for issue gh-7808. a = np.array([1, 2, 3]) - padded = pad(a, ((1, 2),), 'edge') + padded = np.pad(a, ((1, 2),), 'edge') expected = np.array([1, 1, 2, 3, 3, 3]) assert_array_equal(padded, expected) a = np.array([[1, 2, 3], [4, 5, 6]]) - padded = pad(a, ((1, 2),), 'edge') - expected = pad(a, ((1, 2), (1, 2)), 'edge') + padded = np.pad(a, ((1, 2),), 'edge') + expected = np.pad(a, ((1, 2), (1, 2)), 'edge') assert_array_equal(padded, expected) a = np.arange(24).reshape(2, 3, 4) - padded = pad(a, ((1, 2),), 'edge') - expected = pad(a, ((1, 2), (1, 2), (1, 2)), 'edge') + padded = np.pad(a, ((1, 2),), 'edge') + expected = np.pad(a, ((1, 2), (1, 2), (1, 2)), 'edge') assert_array_equal(padded, expected) -class TestZeroPadWidth(object): - def test_zero_pad_width(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - for pad_width in (0, (0, 0), ((0, 0), (0, 0))): - assert_array_equal(arr, pad(arr, pad_width, mode='constant')) +class TestEmpty(object): + def test_simple(self): + arr = np.arange(24).reshape(4, 6) + result = np.pad(arr, [(2, 3), (3, 1)], mode="empty") + assert result.shape == (9, 10) + assert_equal(arr, result[2:-3, 3:-1]) + def test_pad_empty_dimension(self): + arr = np.zeros((3, 0, 2)) + result = np.pad(arr, [(0,), (2,), (1,)], mode="empty") + assert result.shape == (3, 4, 4) -class TestLegacyVectorFunction(object): - def test_legacy_vector_functionality(self): - def _padwithtens(vector, pad_width, iaxis, kwargs): - vector[:pad_width[0]] = 10 - vector[-pad_width[1]:] = 10 - return vector - a = np.arange(6).reshape(2, 3) - a = pad(a, 2, _padwithtens) - b = np.array( - [[10, 10, 10, 10, 10, 10, 10], - [10, 10, 10, 10, 10, 10, 10], +def test_legacy_vector_functionality(): + def _padwithtens(vector, pad_width, iaxis, kwargs): + vector[:pad_width[0]] = 10 + vector[-pad_width[1]:] = 10 - [10, 10, 0, 1, 2, 10, 10], - [10, 10, 3, 4, 5, 10, 10], + a = np.arange(6).reshape(2, 3) + a = np.pad(a, 2, _padwithtens) + b = np.array( + [[10, 10, 10, 10, 10, 10, 10], + [10, 10, 10, 10, 10, 10, 10], - [10, 10, 10, 10, 10, 10, 10], - [10, 10, 10, 10, 10, 10, 10]] - ) - assert_array_equal(a, b) + [10, 10, 0, 1, 2, 10, 10], + [10, 10, 3, 4, 5, 10, 10], + [10, 10, 10, 10, 10, 10, 10], + [10, 10, 10, 10, 10, 10, 10]] + ) + assert_array_equal(a, b) -class TestNdarrayPadWidth(object): - def test_check_simple(self): + +def test_unicode_mode(): + a = np.pad([1], 2, mode=u'constant') + b = np.array([0, 0, 1, 0, 0]) + assert_array_equal(a, b) + + +@pytest.mark.parametrize("mode", ["edge", "symmetric", "reflect", "wrap"]) +def test_object_input(mode): + # Regression test for issue gh-11395. + a = np.full((4, 3), fill_value=None) + pad_amt = ((2, 3), (3, 2)) + b = np.full((9, 8), fill_value=None) + assert_array_equal(np.pad(a, pad_amt, mode=mode), b) + + +class TestPadWidth(object): + @pytest.mark.parametrize("pad_width", [ + (4, 5, 6, 7), + ((1,), (2,), (3,)), + ((1, 2), (3, 4), (5, 6)), + ((3, 4, 5), (0, 1, 2)), + ]) + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_misshaped_pad_width(self, pad_width, mode): + arr = np.arange(30).reshape((6, 5)) + match = "operands could not be broadcast together" + with pytest.raises(ValueError, match=match): + np.pad(arr, pad_width, mode) + + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_misshaped_pad_width_2(self, mode): + arr = np.arange(30).reshape((6, 5)) + match = ("input operand has more dimensions than allowed by the axis " + "remapping") + with pytest.raises(ValueError, match=match): + np.pad(arr, (((3,), (4,), (5,)), ((0,), (1,), (2,))), mode) + + @pytest.mark.parametrize( + "pad_width", [-2, (-2,), (3, -1), ((5, 2), (-2, 3)), ((-4,), (2,))]) + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_negative_pad_width(self, pad_width, mode): + arr = np.arange(30).reshape((6, 5)) + match = "index can't contain negative values" + with pytest.raises(ValueError, match=match): + np.pad(arr, pad_width, mode) + + @pytest.mark.parametrize("pad_width, dtype", [ + ("3", None), + ("word", None), + (None, None), + (object(), None), + (3.4, None), + (((2, 3, 4), (3, 2)), object), + (complex(1, -1), None), + (((-2.1, 3), (3, 2)), None), + ]) + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_bad_type(self, pad_width, dtype, mode): + arr = np.arange(30).reshape((6, 5)) + match = "`pad_width` must be of integral type." + if dtype is not None: + # avoid DeprecationWarning when not specifying dtype + with pytest.raises(TypeError, match=match): + np.pad(arr, np.array(pad_width, dtype=dtype), mode) + else: + with pytest.raises(TypeError, match=match): + np.pad(arr, pad_width, mode) + with pytest.raises(TypeError, match=match): + np.pad(arr, np.array(pad_width), mode) + + def test_pad_width_as_ndarray(self): a = np.arange(12) a = np.reshape(a, (4, 3)) - a = pad(a, np.array(((2, 3), (3, 2))), 'edge') + a = np.pad(a, np.array(((2, 3), (3, 2))), 'edge') b = np.array( [[0, 0, 0, 0, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 2, 2], @@ -1142,121 +1305,62 @@ class TestNdarrayPadWidth(object): ) assert_array_equal(a, b) - -class TestUnicodeInput(object): - def test_unicode_mode(self): - constant_mode = u'constant' - a = np.pad([1], 2, mode=constant_mode) - b = np.array([0, 0, 1, 0, 0]) - assert_array_equal(a, b) - - -class TestObjectInput(object): - def test_object_input(self): - # Regression test for issue gh-11395. - a = np.full((4, 3), None) - pad_amt = ((2, 3), (3, 2)) - b = np.full((9, 8), None) - modes = ['edge', - 'symmetric', - 'reflect', - 'wrap', - ] - for mode in modes: - assert_array_equal(pad(a, pad_amt, mode=mode), b) - - -class TestValueError1(object): - def test_check_simple(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - kwargs = dict(mode='mean', stat_length=(3, )) - assert_raises(ValueError, pad, arr, ((2, 3), (3, 2), (4, 5)), - **kwargs) - - def test_check_negative_stat_length(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - kwargs = dict(mode='mean', stat_length=(-3, )) - assert_raises(ValueError, pad, arr, ((2, 3), (3, 2)), - **kwargs) - - def test_check_negative_pad_width(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - kwargs = dict(mode='mean', stat_length=(3, )) - assert_raises(ValueError, pad, arr, ((-2, 3), (3, 2)), - **kwargs) - - def test_check_empty_array(self): - assert_raises(ValueError, pad, [], 4, mode='reflect') - assert_raises(ValueError, pad, np.ndarray(0), 4, mode='reflect') - assert_raises(ValueError, pad, np.zeros((0, 3)), ((1,), (0,)), - mode='reflect') - - -class TestValueError2(object): - def test_check_negative_pad_amount(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - kwargs = dict(mode='mean', stat_length=(3, )) - assert_raises(ValueError, pad, arr, ((-2, 3), (3, 2)), - **kwargs) - - -class TestValueError3(object): - def test_check_kwarg_not_allowed(self): - arr = np.arange(30).reshape(5, 6) - assert_raises(ValueError, pad, arr, 4, mode='mean', - reflect_type='odd') - - def test_mode_not_set(self): - arr = np.arange(30).reshape(5, 6) - assert_raises(TypeError, pad, arr, 4) - - def test_malformed_pad_amount(self): - arr = np.arange(30).reshape(5, 6) - assert_raises(ValueError, pad, arr, (4, 5, 6, 7), mode='constant') - - def test_malformed_pad_amount2(self): - arr = np.arange(30).reshape(5, 6) - assert_raises(ValueError, pad, arr, ((3, 4, 5), (0, 1, 2)), - mode='constant') - - def test_pad_too_many_axes(self): - arr = np.arange(30).reshape(5, 6) - - # Attempt to pad using a 3D array equivalent - bad_shape = (((3,), (4,), (5,)), ((0,), (1,), (2,))) - assert_raises(ValueError, pad, arr, bad_shape, - mode='constant') - - -class TestTypeError1(object): - def test_float(self): - arr = np.arange(30) - assert_raises(TypeError, pad, arr, ((-2.1, 3), (3, 2))) - assert_raises(TypeError, pad, arr, np.array(((-2.1, 3), (3, 2)))) - - def test_str(self): - arr = np.arange(30) - assert_raises(TypeError, pad, arr, 'foo') - assert_raises(TypeError, pad, arr, np.array('foo')) - - def test_object(self): - class FooBar(object): - pass - arr = np.arange(30) - assert_raises(TypeError, pad, arr, FooBar()) - - def test_complex(self): - arr = np.arange(30) - assert_raises(TypeError, pad, arr, complex(1, -1)) - assert_raises(TypeError, pad, arr, np.array(complex(1, -1))) - - def test_check_wrong_pad_amount(self): - arr = np.arange(30) - arr = np.reshape(arr, (6, 5)) - kwargs = dict(mode='mean', stat_length=(3, )) - assert_raises(TypeError, pad, arr, ((2, 3, 4), (3, 2)), - **kwargs) + @pytest.mark.parametrize("pad_width", [0, (0, 0), ((0, 0), (0, 0))]) + @pytest.mark.parametrize("mode", _all_modes.keys()) + def test_zero_pad_width(self, pad_width, mode): + arr = np.arange(30).reshape(6, 5) + assert_array_equal(arr, np.pad(arr, pad_width, mode=mode)) + + +@pytest.mark.parametrize("mode", _all_modes.keys()) +def test_kwargs(mode): + """Test behavior of pad's kwargs for the given mode.""" + allowed = _all_modes[mode] + not_allowed = {} + for kwargs in _all_modes.values(): + if kwargs != allowed: + not_allowed.update(kwargs) + # Test if allowed keyword arguments pass + np.pad([1, 2, 3], 1, mode, **allowed) + # Test if prohibited keyword arguments of other modes raise an error + for key, value in not_allowed.items(): + match = "unsupported keyword arguments for mode '{}'".format(mode) + with pytest.raises(ValueError, match=match): + np.pad([1, 2, 3], 1, mode, **{key: value}) + + +def test_constant_zero_default(): + arr = np.array([1, 1]) + assert_array_equal(np.pad(arr, 2), [0, 0, 1, 1, 0, 0]) + + +@pytest.mark.parametrize("mode", [1, "const", object(), None, True, False]) +def test_unsupported_mode(mode): + match= "mode '{}' is not supported".format(mode) + with pytest.raises(ValueError, match=match): + np.pad([1, 2, 3], 4, mode=mode) + + +@pytest.mark.parametrize("mode", _all_modes.keys()) +def test_non_contiguous_array(mode): + arr = np.arange(24).reshape(4, 6)[::2, ::2] + result = np.pad(arr, (2, 3), mode) + assert result.shape == (7, 8) + assert_equal(result[2:-3, 2:-3], arr) + + +@pytest.mark.parametrize("mode", _all_modes.keys()) +def test_memory_layout_persistence(mode): + """Test if C and F order is preserved for all pad modes.""" + x = np.ones((5, 10), order='C') + assert np.pad(x, 5, mode).flags["C_CONTIGUOUS"] + x = np.ones((5, 10), order='F') + assert np.pad(x, 5, mode).flags["F_CONTIGUOUS"] + + +@pytest.mark.parametrize("dtype", _numeric_dtypes) +@pytest.mark.parametrize("mode", _all_modes.keys()) +def test_dtype_persistence(dtype, mode): + arr = np.zeros((3, 2, 1), dtype=dtype) + result = np.pad(arr, 1, mode=mode) + assert result.dtype == dtype diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index a17fc66e5..fd21a7f76 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -136,8 +136,8 @@ class TestSetOps(object): np.nan), # should fail because attempting # to downcast to smaller int type: - (np.array([1, 2, 3], dtype=np.int32), - np.array([5, 7, 2], dtype=np.int64), + (np.array([1, 2, 3], dtype=np.int16), + np.array([5, 1<<20, 2], dtype=np.int32), None), # should fail because attempting to cast # two special floating point values @@ -152,8 +152,8 @@ class TestSetOps(object): # specifically, raise an appropriate # Exception when attempting to append or # prepend with an incompatible type - msg = 'must be compatible' - with assert_raises_regex(TypeError, msg): + msg = 'cannot convert' + with assert_raises_regex(ValueError, msg): ediff1d(ary=ary, to_end=append, to_begin=prepend) @@ -422,41 +422,41 @@ class TestUnique(object): assert_array_equal(v, b, msg) msg = base_msg.format('return_index', dt) - v, j = unique(a, 1, 0, 0) + v, j = unique(a, True, False, False) assert_array_equal(v, b, msg) assert_array_equal(j, i1, msg) msg = base_msg.format('return_inverse', dt) - v, j = unique(a, 0, 1, 0) + v, j = unique(a, False, True, False) assert_array_equal(v, b, msg) assert_array_equal(j, i2, msg) msg = base_msg.format('return_counts', dt) - v, j = unique(a, 0, 0, 1) + v, j = unique(a, False, False, True) assert_array_equal(v, b, msg) assert_array_equal(j, c, msg) msg = base_msg.format('return_index and return_inverse', dt) - v, j1, j2 = unique(a, 1, 1, 0) + v, j1, j2 = unique(a, True, True, False) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, i2, msg) msg = base_msg.format('return_index and return_counts', dt) - v, j1, j2 = unique(a, 1, 0, 1) + v, j1, j2 = unique(a, True, False, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, c, msg) msg = base_msg.format('return_inverse and return_counts', dt) - v, j1, j2 = unique(a, 0, 1, 1) + v, j1, j2 = unique(a, False, True, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i2, msg) assert_array_equal(j2, c, msg) msg = base_msg.format(('return_index, return_inverse ' 'and return_counts'), dt) - v, j1, j2, j3 = unique(a, 1, 1, 1) + v, j1, j2, j3 = unique(a, True, True, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, i2, msg) @@ -600,8 +600,11 @@ class TestUnique(object): assert_array_equal(unique(data, axis=1), result.astype(dtype), msg) msg = 'Unique with 3d array and axis=2 failed' - data3d = np.dstack([data] * 3) - result = data3d[..., :1] + data3d = np.array([[[1, 1], + [1, 0]], + [[0, 1], + [0, 0]]]).astype(dtype) + result = np.take(data3d, [1, 0], axis=2) assert_array_equal(unique(data3d, axis=2), result, msg) uniq, idx, inv, cnt = unique(data, axis=0, return_index=True, diff --git a/numpy/lib/tests/test_financial.py b/numpy/lib/tests/test_financial.py index 524915041..cb67f7c0f 100644 --- a/numpy/lib/tests/test_financial.py +++ b/numpy/lib/tests/test_financial.py @@ -1,5 +1,6 @@ from __future__ import division, absolute_import, print_function +import warnings from decimal import Decimal import numpy as np @@ -8,16 +9,35 @@ from numpy.testing import ( ) +def filter_deprecation(func): + def newfunc(*args, **kwargs): + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings('always', category=DeprecationWarning) + func(*args, **kwargs) + assert_(all(w.category is DeprecationWarning for w in ws)) + return newfunc + + class TestFinancial(object): + @filter_deprecation + def test_npv_irr_congruence(self): + # IRR is defined as the rate required for the present value of a + # a series of cashflows to be zero i.e. NPV(IRR(x), x) = 0 + cashflows = np.array([-40000, 5000, 8000, 12000, 30000]) + assert_allclose(np.npv(np.irr(cashflows), cashflows), 0, atol=1e-10, rtol=0) + + @filter_deprecation def test_rate(self): assert_almost_equal( np.rate(10, 0, -3500, 10000), 0.1107, 4) + @filter_deprecation def test_rate_decimal(self): rate = np.rate(Decimal('10'), Decimal('0'), Decimal('-3500'), Decimal('10000')) assert_equal(Decimal('0.1106908537142689284704528100'), rate) + @filter_deprecation def test_irr(self): v = [-150000, 15000, 25000, 35000, 45000, 60000] assert_almost_equal(np.irr(v), 0.0524, 2) @@ -37,20 +57,25 @@ class TestFinancial(object): v = [-1, -2, -3] assert_equal(np.irr(v), np.nan) + @filter_deprecation def test_pv(self): assert_almost_equal(np.pv(0.07, 20, 12000, 0), -127128.17, 2) + @filter_deprecation def test_pv_decimal(self): assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), Decimal('-127128.1709461939327295222005')) + @filter_deprecation def test_fv(self): assert_equal(np.fv(0.075, 20, -2000, 0, 0), 86609.362673042924) + @filter_deprecation def test_fv_decimal(self): assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), 0, 0), Decimal('86609.36267304300040536731624')) + @filter_deprecation def test_pmt(self): res = np.pmt(0.08 / 12, 5 * 12, 15000) tgt = -304.145914 @@ -65,6 +90,7 @@ class TestFinancial(object): tgt = np.array([[-166.66667, -19311.258], [-626.90814, -19311.258]]) assert_allclose(res, tgt) + @filter_deprecation def test_pmt_decimal(self): res = np.pmt(Decimal('0.08') / Decimal('12'), 5 * 12, 15000) tgt = Decimal('-304.1459143262052370338701494') @@ -88,18 +114,22 @@ class TestFinancial(object): assert_equal(res[1][0], tgt[1][0]) assert_equal(res[1][1], tgt[1][1]) + @filter_deprecation def test_ppmt(self): assert_equal(np.round(np.ppmt(0.1 / 12, 1, 60, 55000), 2), -710.25) + @filter_deprecation def test_ppmt_decimal(self): assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000')), Decimal('-710.2541257864217612489830917')) # Two tests showing how Decimal is actually getting at a more exact result # .23 / 12 does not come out nicely as a float but does as a decimal + @filter_deprecation def test_ppmt_special_rate(self): assert_equal(np.round(np.ppmt(0.23 / 12, 1, 60, 10000000000), 8), -90238044.232277036) + @filter_deprecation def test_ppmt_special_rate_decimal(self): # When rounded out to 8 decimal places like the float based test, this should not equal the same value # as the float, substituted for the decimal @@ -112,31 +142,38 @@ class TestFinancial(object): assert_equal(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), Decimal('-90238044.2322778884413969909')) + @filter_deprecation def test_ipmt(self): assert_almost_equal(np.round(np.ipmt(0.1 / 12, 1, 24, 2000), 2), -16.67) + @filter_deprecation def test_ipmt_decimal(self): result = np.ipmt(Decimal('0.1') / Decimal('12'), 1, 24, 2000) assert_equal(result.flat[0], Decimal('-16.66666666666666666666666667')) + @filter_deprecation def test_nper(self): assert_almost_equal(np.nper(0.075, -2000, 0, 100000.), 21.54, 2) + @filter_deprecation def test_nper2(self): assert_almost_equal(np.nper(0.0, -2000, 0, 100000.), 50.0, 1) + @filter_deprecation def test_npv(self): assert_almost_equal( np.npv(0.05, [-15000, 1500, 2500, 3500, 4500, 6000]), 122.89, 2) + @filter_deprecation def test_npv_decimal(self): assert_equal( np.npv(Decimal('0.05'), [-15000, 1500, 2500, 3500, 4500, 6000]), Decimal('122.894854950942692161628715')) + @filter_deprecation def test_mirr(self): val = [-4500, -800, 800, 800, 600, 600, 800, 800, 700, 3000] assert_almost_equal(np.mirr(val, 0.08, 0.055), 0.0666, 4) @@ -150,6 +187,7 @@ class TestFinancial(object): val = [39000, 30000, 21000, 37000, 46000] assert_(np.isnan(np.mirr(val, 0.10, 0.12))) + @filter_deprecation def test_mirr_decimal(self): val = [Decimal('-4500'), Decimal('-800'), Decimal('800'), Decimal('800'), Decimal('600'), Decimal('600'), Decimal('800'), Decimal('800'), @@ -168,6 +206,7 @@ class TestFinancial(object): val = [Decimal('39000'), Decimal('30000'), Decimal('21000'), Decimal('37000'), Decimal('46000')] assert_(np.isnan(np.mirr(val, Decimal('0.10'), Decimal('0.12')))) + @filter_deprecation def test_when(self): # begin assert_equal(np.rate(10, 20, -3500, 10000, 1), @@ -232,6 +271,7 @@ class TestFinancial(object): assert_equal(np.nper(0.075, -2000, 0, 100000., 0), np.nper(0.075, -2000, 0, 100000., 'end')) + @filter_deprecation def test_decimal_with_when(self): """Test that decimals are still supported if the when argument is passed""" # begin @@ -306,6 +346,7 @@ class TestFinancial(object): np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), Decimal('0'), 'end').flat[0]) + @filter_deprecation def test_broadcast(self): assert_almost_equal(np.nper(0.075, -2000, 0, 100000., [0, 1]), [21.5449442, 20.76156441], 4) @@ -323,6 +364,7 @@ class TestFinancial(object): [-74.998201, -75.62318601, -75.62318601, -76.88882405, -76.88882405], 4) + @filter_deprecation def test_broadcast_decimal(self): # Use almost equal because precision is tested in the explicit tests, this test is to ensure # broadcast with Decimal is not broken. diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 077507082..0592e0b12 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -287,6 +287,7 @@ from io import BytesIO import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_raises, assert_raises_regex, + assert_warns ) from numpy.lib import format @@ -411,6 +412,7 @@ record_arrays = [ np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')), np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')), np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')), + np.zeros(1, dtype=[('c', ('<f8', (5,)), (2,))]) ] @@ -426,7 +428,7 @@ def roundtrip(arr): f = BytesIO() format.write_array(f, arr) f2 = BytesIO(f.getvalue()) - arr2 = format.read_array(f2) + arr2 = format.read_array(f2, allow_pickle=True) return arr2 @@ -576,7 +578,7 @@ def test_pickle_python2_python3(): path = os.path.join(data_dir, fname) for encoding in ['bytes', 'latin1']: - data_f = np.load(path, encoding=encoding) + data_f = np.load(path, allow_pickle=True, encoding=encoding) if fname.endswith('.npz'): data = data_f['x'] data_f.close() @@ -598,16 +600,19 @@ def test_pickle_python2_python3(): if sys.version_info[0] >= 3: if fname.startswith('py2'): if fname.endswith('.npz'): - data = np.load(path) + data = np.load(path, allow_pickle=True) assert_raises(UnicodeError, data.__getitem__, 'x') data.close() - data = np.load(path, fix_imports=False, encoding='latin1') + data = np.load(path, allow_pickle=True, fix_imports=False, + encoding='latin1') assert_raises(ImportError, data.__getitem__, 'x') data.close() else: - assert_raises(UnicodeError, np.load, path) + assert_raises(UnicodeError, np.load, path, + allow_pickle=True) assert_raises(ImportError, np.load, path, - encoding='latin1', fix_imports=False) + allow_pickle=True, fix_imports=False, + encoding='latin1') def test_pickle_disallow(): @@ -625,6 +630,61 @@ def test_pickle_disallow(): assert_raises(ValueError, np.save, path, np.array([None], dtype=object), allow_pickle=False) +@pytest.mark.parametrize('dt', [ + np.dtype(np.dtype([('a', np.int8), + ('b', np.int16), + ('c', np.int32), + ], align=True), + (3,)), + np.dtype([('x', np.dtype({'names':['a','b'], + 'formats':['i1','i1'], + 'offsets':[0,4], + 'itemsize':8, + }, + (3,)), + (4,), + )]), + np.dtype([('x', + ('<f8', (5,)), + (2,), + )]), + np.dtype([('x', np.dtype(( + np.dtype(( + np.dtype({'names':['a','b'], + 'formats':['i1','i1'], + 'offsets':[0,4], + 'itemsize':8}), + (3,) + )), + (4,) + ))) + ]), + np.dtype([ + ('a', np.dtype(( + np.dtype(( + np.dtype(( + np.dtype([ + ('a', int), + ('b', np.dtype({'names':['a','b'], + 'formats':['i1','i1'], + 'offsets':[0,4], + 'itemsize':8})), + ]), + (3,), + )), + (4,), + )), + (5,), + ))) + ]), + ]) + +def test_descr_to_dtype(dt): + dt1 = format.descr_to_dtype(dt.descr) + assert_equal_(dt1, dt) + arr1 = np.zeros(3, dt) + arr2 = roundtrip(arr1) + assert_array_equal(arr1, arr2) def test_version_2_0(): f = BytesIO() @@ -879,3 +939,57 @@ def test_empty_npz(): fname = os.path.join(tempdir, "nothing.npz") np.savez(fname) np.load(fname) + + +def test_unicode_field_names(): + # gh-7391 + arr = np.array([ + (1, 3), + (1, 2), + (1, 3), + (1, 2) + ], dtype=[ + ('int', int), + (u'\N{CJK UNIFIED IDEOGRAPH-6574}\N{CJK UNIFIED IDEOGRAPH-5F62}', int) + ]) + fname = os.path.join(tempdir, "unicode.npy") + with open(fname, 'wb') as f: + format.write_array(f, arr, version=(3, 0)) + with open(fname, 'rb') as f: + arr2 = format.read_array(f) + assert_array_equal(arr, arr2) + + # notifies the user that 3.0 is selected + with open(fname, 'wb') as f: + with assert_warns(UserWarning): + format.write_array(f, arr, version=None) + + +@pytest.mark.parametrize('dt, fail', [ + (np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3', + metadata={'some': 'stuff'})]}), True), + (np.dtype(int, metadata={'some': 'stuff'}), False), + (np.dtype([('subarray', (int, (2,)))], metadata={'some': 'stuff'}), False), + # recursive: metadata on the field of a dtype + (np.dtype({'names': ['a', 'b'], 'formats': [ + float, np.dtype({'names': ['c'], 'formats': [np.dtype(int, metadata={})]}) + ]}), False) + ]) +def test_metadata_dtype(dt, fail): + # gh-14142 + arr = np.ones(10, dtype=dt) + buf = BytesIO() + with assert_warns(UserWarning): + np.save(buf, arr) + buf.seek(0) + if fail: + with assert_raises(ValueError): + np.load(buf) + else: + arr2 = np.load(buf) + # BUG: assert_array_equal does not check metadata + from numpy.lib.format import _has_metadata + assert_array_equal(arr, arr2) + assert _has_metadata(arr.dtype) + assert not _has_metadata(arr2.dtype) + diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 3d4b0e3b2..9075ff538 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -4,13 +4,15 @@ import operator import warnings import sys import decimal +import types +from fractions import Fraction import pytest import numpy as np from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, - assert_array_almost_equal, assert_raises, assert_allclose, + assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY, assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, ) import numpy.lib.function_base as nfb @@ -24,6 +26,7 @@ from numpy.lib import ( from numpy.compat import long +PY2 = sys.version_info[0] == 2 def get_mat(n): data = np.arange(n) @@ -31,6 +34,17 @@ def get_mat(n): return data +def _make_complex(real, imag): + """ + Like real + 1j * imag, but behaves as expected when imag contains non-finite + values + """ + ret = np.zeros(np.broadcast(real, imag).shape, np.complex_) + ret.real = real + ret.imag = imag + return ret + + class TestRot90(object): def test_basic(self): assert_raises(ValueError, rot90, np.ones(4)) @@ -353,9 +367,9 @@ class TestAverage(object): assert_equal(type(np.average(a, weights=w)), subclass) def test_upcasting(self): - types = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'), + typs = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'), ('f4', 'f4', 'f4'), ('f4', 'f8', 'f8')] - for at, wt, rt in types: + for at, wt, rt in typs: a = np.array([[1,2],[3,4]], dtype=at) w = np.array([[1,2],[3,4]], dtype=wt) assert_equal(np.average(a, weights=w).dtype, np.dtype(rt)) @@ -409,27 +423,17 @@ class TestSelect(object): assert_equal(select([m], [d]), [0, 0, 0, np.nan, 0, 0]) def test_deprecated_empty(self): - with warnings.catch_warnings(record=True): - warnings.simplefilter("always") - assert_equal(select([], [], 3j), 3j) - - with warnings.catch_warnings(): - warnings.simplefilter("always") - assert_warns(DeprecationWarning, select, [], []) - warnings.simplefilter("error") - assert_raises(DeprecationWarning, select, [], []) + assert_raises(ValueError, select, [], [], 3j) + assert_raises(ValueError, select, [], []) def test_non_bool_deprecation(self): choices = self.choices conditions = self.conditions[:] - with warnings.catch_warnings(): - warnings.filterwarnings("always") - conditions[0] = conditions[0].astype(np.int_) - assert_warns(DeprecationWarning, select, conditions, choices) - conditions[0] = conditions[0].astype(np.uint8) - assert_warns(DeprecationWarning, select, conditions, choices) - warnings.filterwarnings("error") - assert_raises(DeprecationWarning, select, conditions, choices) + conditions[0] = conditions[0].astype(np.int_) + assert_raises(TypeError, select, conditions, choices) + conditions[0] = conditions[0].astype(np.uint8) + assert_raises(TypeError, select, conditions, choices) + assert_raises(TypeError, select, conditions, choices) def test_many_arguments(self): # This used to be limited by NPY_MAXARGS == 32 @@ -682,6 +686,9 @@ class TestDiff(object): assert_raises(np.AxisError, diff, x, axis=3) assert_raises(np.AxisError, diff, x, axis=-4) + x = np.array(1.11111111111, np.float64) + assert_raises(ValueError, diff, x) + def test_nd(self): x = 20 * rand(10, 20, 30) out1 = x[:, :, 1:] - x[:, :, :-1] @@ -931,7 +938,7 @@ class TestGradient(object): assert_equal(type(out), type(x)) # And make sure that the output and input don't have aliased mask # arrays - assert_(x.mask is not out.mask) + assert_(x._mask is not out._mask) # Also check that edge_order=2 doesn't alter the original mask x2 = np.ma.arange(5) x2[2] = np.ma.masked @@ -1088,7 +1095,7 @@ class TestAngle(object): np.arctan(3.0 / 1.0), np.arctan(1.0), 0, np.pi / 2, np.pi, -np.pi / 2.0, -np.arctan(3.0 / 1.0), np.pi - np.arctan(3.0 / 1.0)] - z = angle(x, deg=1) + z = angle(x, deg=True) zo = np.array(yo) * 180 / np.pi assert_array_almost_equal(y, yo, 11) assert_array_almost_equal(z, zo, 11) @@ -1498,6 +1505,49 @@ class TestVectorize(object): f(x) +class TestLeaks(object): + class A(object): + iters = 20 + + def bound(self, *args): + return 0 + + @staticmethod + def unbound(*args): + return 0 + + @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") + @pytest.mark.parametrize('name, incr', [ + ('bound', A.iters), + ('unbound', 0), + ]) + def test_frompyfunc_leaks(self, name, incr): + # exposed in gh-11867 as np.vectorized, but the problem stems from + # frompyfunc. + # class.attribute = np.frompyfunc(<method>) creates a + # reference cycle if <method> is a bound class method. It requires a + # gc collection cycle to break the cycle (on CPython 3) + import gc + A_func = getattr(self.A, name) + gc.disable() + try: + refcount = sys.getrefcount(A_func) + for i in range(self.A.iters): + a = self.A() + a.f = np.frompyfunc(getattr(a, name), 1, 1) + out = a.f(np.arange(10)) + a = None + if PY2: + assert_equal(sys.getrefcount(A_func), refcount) + else: + # A.func is part of a reference cycle if incr is non-zero + assert_equal(sys.getrefcount(A_func), refcount + incr) + for i in range(5): + gc.collect() + assert_equal(sys.getrefcount(A_func), refcount) + finally: + gc.enable() + class TestDigitize(object): def test_forward(self): @@ -1860,9 +1910,9 @@ class TestCov(object): [-np.inf, np.inf]])) def test_1D_rowvar(self): - assert_allclose(cov(self.x3), cov(self.x3, rowvar=0)) + assert_allclose(cov(self.x3), cov(self.x3, rowvar=False)) y = np.array([0.0780, 0.3107, 0.2111, 0.0334, 0.8501]) - assert_allclose(cov(self.x3, y), cov(self.x3, y, rowvar=0)) + assert_allclose(cov(self.x3, y), cov(self.x3, y, rowvar=False)) def test_1D_variance(self): assert_allclose(cov(self.x3, ddof=1), np.var(self.x3, ddof=1)) @@ -1924,9 +1974,9 @@ class Test_I0(object): np.array(1.0634833707413234)) A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549]) - assert_almost_equal( - i0(A), - np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049])) + expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049]) + assert_almost_equal(i0(A), expected) + assert_almost_equal(i0(-A), expected) B = np.array([[0.827002, 0.99959078], [0.89694769, 0.39298162], @@ -1940,6 +1990,26 @@ class Test_I0(object): [1.03633899, 1.00067775], [1.03352052, 1.13557954], [1.05884290, 1.06432317]])) + # Regression test for gh-11205 + i0_0 = np.i0([0.]) + assert_equal(i0_0.shape, (1,)) + assert_array_equal(np.i0([0.]), np.array([1.])) + + def test_non_array(self): + a = np.arange(4) + + class array_like: + __array_interface__ = a.__array_interface__ + + def __array_wrap__(self, arr): + return self + + # E.g. pandas series survive ufunc calls through array-wrap: + assert isinstance(np.abs(array_like()), array_like) + exp = np.i0(a) + res = np.i0(array_like()) + + assert_array_equal(exp, res) class TestKaiser(object): @@ -2309,7 +2379,7 @@ class TestInterp(object): x0 = np.nan assert_almost_equal(np.interp(x0, x, y), x0) - def test_non_finite_behavior(self): + def test_non_finite_behavior_exact_x(self): x = [1, 2, 2.5, 3, 4] xp = [1, 2, 3, 4] fp = [1, 2, np.inf, 4] @@ -2317,6 +2387,64 @@ class TestInterp(object): fp = [1, 2, np.nan, 4] assert_almost_equal(np.interp(x, xp, fp), [1, 2, np.nan, np.nan, 4]) + @pytest.fixture(params=[ + lambda x: np.float_(x), + lambda x: _make_complex(x, 0), + lambda x: _make_complex(0, x), + lambda x: _make_complex(x, np.multiply(x, -2)) + ], ids=[ + 'real', + 'complex-real', + 'complex-imag', + 'complex-both' + ]) + def sc(self, request): + """ scale function used by the below tests """ + return request.param + + def test_non_finite_any_nan(self, sc): + """ test that nans are propagated """ + assert_equal(np.interp(0.5, [np.nan, 1], sc([ 0, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, np.nan], sc([ 0, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, 1], sc([np.nan, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, 1], sc([ 0, np.nan])), sc(np.nan)) + + def test_non_finite_inf(self, sc): + """ Test that interp between opposite infs gives nan """ + assert_equal(np.interp(0.5, [-np.inf, +np.inf], sc([ 0, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, 1], sc([-np.inf, +np.inf])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, 1], sc([+np.inf, -np.inf])), sc(np.nan)) + + # unless the y values are equal + assert_equal(np.interp(0.5, [-np.inf, +np.inf], sc([ 10, 10])), sc(10)) + + def test_non_finite_half_inf_xf(self, sc): + """ Test that interp where both axes have a bound at inf gives nan """ + assert_equal(np.interp(0.5, [-np.inf, 1], sc([-np.inf, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [-np.inf, 1], sc([+np.inf, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [-np.inf, 1], sc([ 0, -np.inf])), sc(np.nan)) + assert_equal(np.interp(0.5, [-np.inf, 1], sc([ 0, +np.inf])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, +np.inf], sc([-np.inf, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, +np.inf], sc([+np.inf, 10])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, +np.inf], sc([ 0, -np.inf])), sc(np.nan)) + assert_equal(np.interp(0.5, [ 0, +np.inf], sc([ 0, +np.inf])), sc(np.nan)) + + def test_non_finite_half_inf_x(self, sc): + """ Test interp where the x axis has a bound at inf """ + assert_equal(np.interp(0.5, [-np.inf, -np.inf], sc([0, 10])), sc(10)) + assert_equal(np.interp(0.5, [-np.inf, 1 ], sc([0, 10])), sc(10)) + assert_equal(np.interp(0.5, [ 0, +np.inf], sc([0, 10])), sc(0)) + assert_equal(np.interp(0.5, [+np.inf, +np.inf], sc([0, 10])), sc(0)) + + def test_non_finite_half_inf_f(self, sc): + """ Test interp where the f axis has a bound at inf """ + assert_equal(np.interp(0.5, [0, 1], sc([ 0, -np.inf])), sc(-np.inf)) + assert_equal(np.interp(0.5, [0, 1], sc([ 0, +np.inf])), sc(+np.inf)) + assert_equal(np.interp(0.5, [0, 1], sc([-np.inf, 10])), sc(-np.inf)) + assert_equal(np.interp(0.5, [0, 1], sc([+np.inf, 10])), sc(+np.inf)) + assert_equal(np.interp(0.5, [0, 1], sc([-np.inf, -np.inf])), sc(-np.inf)) + assert_equal(np.interp(0.5, [0, 1], sc([+np.inf, +np.inf])), sc(+np.inf)) + def test_complex_interp(self): # test complex interpolation x = np.linspace(0, 1, 5) @@ -2391,11 +2519,23 @@ class TestPercentile(object): assert_equal(np.percentile(x, 100), 3.5) assert_equal(np.percentile(x, 50), 1.75) x[1] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(x, 0), np.nan) - assert_equal(np.percentile(x, 0, interpolation='nearest'), np.nan) - assert_(w[0].category is RuntimeWarning) + assert_equal(np.percentile(x, 0), np.nan) + assert_equal(np.percentile(x, 0, interpolation='nearest'), np.nan) + + def test_fraction(self): + x = [Fraction(i, 2) for i in range(8)] + + p = np.percentile(x, Fraction(0)) + assert_equal(p, Fraction(0)) + assert_equal(type(p), Fraction) + + p = np.percentile(x, Fraction(100)) + assert_equal(p, Fraction(7, 2)) + assert_equal(type(p), Fraction) + + p = np.percentile(x, Fraction(50)) + assert_equal(p, Fraction(7, 4)) + assert_equal(type(p), Fraction) def test_api(self): d = np.ones(5) @@ -2733,85 +2873,63 @@ class TestPercentile(object): def test_nan_behavior(self): a = np.arange(24, dtype=float) a[2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, 0.3), np.nan) - assert_equal(np.percentile(a, 0.3, axis=0), np.nan) - assert_equal(np.percentile(a, [0.3, 0.6], axis=0), - np.array([np.nan] * 2)) - assert_(w[0].category is RuntimeWarning) - assert_(w[1].category is RuntimeWarning) - assert_(w[2].category is RuntimeWarning) + assert_equal(np.percentile(a, 0.3), np.nan) + assert_equal(np.percentile(a, 0.3, axis=0), np.nan) + assert_equal(np.percentile(a, [0.3, 0.6], axis=0), + np.array([np.nan] * 2)) a = np.arange(24, dtype=float).reshape(2, 3, 4) a[1, 2, 3] = np.nan a[1, 1, 2] = np.nan # no axis - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, 0.3), np.nan) - assert_equal(np.percentile(a, 0.3).ndim, 0) - assert_(w[0].category is RuntimeWarning) + assert_equal(np.percentile(a, 0.3), np.nan) + assert_equal(np.percentile(a, 0.3).ndim, 0) # axis0 zerod b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, 0) b[2, 3] = np.nan b[1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, 0.3, 0), b) + assert_equal(np.percentile(a, 0.3, 0), b) # axis0 not zerod b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), [0.3, 0.6], 0) b[:, 2, 3] = np.nan b[:, 1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, [0.3, 0.6], 0), b) + assert_equal(np.percentile(a, [0.3, 0.6], 0), b) # axis1 zerod b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, 1) b[1, 3] = np.nan b[1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, 0.3, 1), b) + assert_equal(np.percentile(a, 0.3, 1), b) # axis1 not zerod b = np.percentile( np.arange(24, dtype=float).reshape(2, 3, 4), [0.3, 0.6], 1) b[:, 1, 3] = np.nan b[:, 1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, [0.3, 0.6], 1), b) + assert_equal(np.percentile(a, [0.3, 0.6], 1), b) # axis02 zerod b = np.percentile( np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, (0, 2)) b[1] = np.nan b[2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, 0.3, (0, 2)), b) + assert_equal(np.percentile(a, 0.3, (0, 2)), b) # axis02 not zerod b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), [0.3, 0.6], (0, 2)) b[:, 1] = np.nan b[:, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile(a, [0.3, 0.6], (0, 2)), b) + assert_equal(np.percentile(a, [0.3, 0.6], (0, 2)), b) # axis02 not zerod with nearest interpolation b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), [0.3, 0.6], (0, 2), interpolation='nearest') b[:, 1] = np.nan b[:, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.percentile( - a, [0.3, 0.6], (0, 2), interpolation='nearest'), b) + assert_equal(np.percentile( + a, [0.3, 0.6], (0, 2), interpolation='nearest'), b) class TestQuantile(object): @@ -2823,6 +2941,26 @@ class TestQuantile(object): assert_equal(np.quantile(x, 1), 3.5) assert_equal(np.quantile(x, 0.5), 1.75) + def test_fraction(self): + # fractional input, integral quantile + x = [Fraction(i, 2) for i in range(8)] + + q = np.quantile(x, 0) + assert_equal(q, 0) + assert_equal(type(q), Fraction) + + q = np.quantile(x, 1) + assert_equal(q, Fraction(7, 2)) + assert_equal(type(q), Fraction) + + q = np.quantile(x, Fraction(1, 2)) + assert_equal(q, Fraction(7, 4)) + assert_equal(type(q), Fraction) + + # repeat with integral input but fractional quantile + x = np.arange(8) + assert_equal(np.quantile(x, Fraction(1, 2)), Fraction(7, 2)) + def test_no_p_overwrite(self): # this is worth retesting, because quantile does not make a copy p0 = np.array([0, 0.75, 0.25, 0.5, 1.0]) @@ -2858,10 +2996,7 @@ class TestMedian(object): # check array scalar result assert_equal(np.median(a).ndim, 0) a[1] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a).ndim, 0) - assert_(w[0].category is RuntimeWarning) + assert_equal(np.median(a).ndim, 0) def test_axis_keyword(self): a3 = np.array([[2, 3], @@ -2960,58 +3095,43 @@ class TestMedian(object): def test_nan_behavior(self): a = np.arange(24, dtype=float) a[2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a), np.nan) - assert_equal(np.median(a, axis=0), np.nan) - assert_(w[0].category is RuntimeWarning) - assert_(w[1].category is RuntimeWarning) + assert_equal(np.median(a), np.nan) + assert_equal(np.median(a, axis=0), np.nan) a = np.arange(24, dtype=float).reshape(2, 3, 4) a[1, 2, 3] = np.nan a[1, 1, 2] = np.nan # no axis - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a), np.nan) - assert_equal(np.median(a).ndim, 0) - assert_(w[0].category is RuntimeWarning) + assert_equal(np.median(a), np.nan) + assert_equal(np.median(a).ndim, 0) # axis0 b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), 0) b[2, 3] = np.nan b[1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a, 0), b) - assert_equal(len(w), 1) + assert_equal(np.median(a, 0), b) # axis1 b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), 1) b[1, 3] = np.nan b[1, 2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a, 1), b) - assert_equal(len(w), 1) + assert_equal(np.median(a, 1), b) # axis02 b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), (0, 2)) b[1] = np.nan b[2] = np.nan - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', RuntimeWarning) - assert_equal(np.median(a, (0, 2)), b) - assert_equal(len(w), 1) + assert_equal(np.median(a, (0, 2)), b) def test_empty(self): - # empty arrays + # mean(empty array) emits two warnings: empty slice and divide by 0 a = np.array([], dtype=float) with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', RuntimeWarning) assert_equal(np.median(a), np.nan) assert_(w[0].category is RuntimeWarning) + assert_equal(len(w), 2) # multiple dimensions a = np.array([], dtype=float, ndmin=3) @@ -3106,6 +3226,7 @@ class TestAdd_newdoc_ufunc(object): class TestAdd_newdoc(object): @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_add_doc(self): # test np.add_newdoc tgt = "Current flat index into the array." diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index c96b01d42..dbf189f3e 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -8,6 +8,7 @@ from numpy.testing import ( assert_array_almost_equal, assert_raises, assert_allclose, assert_array_max_ulp, assert_raises_regex, suppress_warnings, ) +import pytest class TestHistogram(object): @@ -554,15 +555,11 @@ class TestHistogramOptimBinNums(object): return a / (a + b) ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)] - for seed in range(256)] + for seed in range(10)] # the average difference between the two methods decreases as the dataset size increases. - assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5), - [0.1065248, - 0.0968844, - 0.0331818, - 0.0178057], - decimal=3) + avg = abs(np.mean(ll, axis=0) - 0.5) + assert_almost_equal(avg, [0.15, 0.09, 0.08, 0.03], decimal=2) def test_simple_range(self): """ @@ -595,6 +592,16 @@ class TestHistogramOptimBinNums(object): msg += " with datasize of {0}".format(testlen) assert_equal(len(a), numbins, err_msg=msg) + @pytest.mark.parametrize("bins", ['auto', 'fd', 'doane', 'scott', + 'stone', 'rice', 'sturges']) + def test_signed_integer_data(self, bins): + # Regression test for gh-14379. + a = np.array([-2, 0, 127], dtype=np.int8) + hist, edges = np.histogram(a, bins=bins) + hist32, edges32 = np.histogram(a.astype(np.int32), bins=bins) + assert_array_equal(hist, hist32) + assert_array_equal(edges, edges32) + def test_simple_weighted(self): """ Check that weighted data raises a TypeError @@ -802,7 +809,7 @@ class TestHistogramdd(object): hist, edges = histogramdd((y, x), bins=(y_edges, x_edges)) assert_equal(hist, relative_areas) - # resulting histogram should be uniform, since counts and areas are propotional + # resulting histogram should be uniform, since counts and areas are proportional hist, edges = histogramdd((y, x), bins=(y_edges, x_edges), density=True) assert_equal(hist, 1 / (8*8)) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 3246f68ff..dbe445c2c 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -77,6 +77,26 @@ class TestRavelUnravelIndex(object): [[3, 6, 6], [4, 5, 1]]) assert_equal(np.unravel_index(1621, (6, 7, 8, 9)), [3, 1, 4, 1]) + def test_empty_indices(self): + msg1 = 'indices must be integral: the provided empty sequence was' + msg2 = 'only int indices permitted' + assert_raises_regex(TypeError, msg1, np.unravel_index, [], (10, 3, 5)) + assert_raises_regex(TypeError, msg1, np.unravel_index, (), (10, 3, 5)) + assert_raises_regex(TypeError, msg2, np.unravel_index, np.array([]), + (10, 3, 5)) + assert_equal(np.unravel_index(np.array([],dtype=int), (10, 3, 5)), + [[], [], []]) + assert_raises_regex(TypeError, msg1, np.ravel_multi_index, ([], []), + (10, 3)) + assert_raises_regex(TypeError, msg1, np.ravel_multi_index, ([], ['abc']), + (10, 3)) + assert_raises_regex(TypeError, msg2, np.ravel_multi_index, + (np.array([]), np.array([])), (5, 3)) + assert_equal(np.ravel_multi_index( + (np.array([], dtype=int), np.array([], dtype=int)), (5, 3)), []) + assert_equal(np.ravel_multi_index(np.array([[], []], dtype=int), + (5, 3)), []) + def test_big_indices(self): # ravel_multi_index for big indices (issue #7546) if np.intp == np.int64: @@ -86,6 +106,9 @@ class TestRavelUnravelIndex(object): np.ravel_multi_index(arr, (41, 7, 120, 36, 2706, 8, 6)), [5627771580, 117259570957]) + # test unravel_index for big indices (issue #9538) + assert_raises(ValueError, np.unravel_index, 1, (2**32-1, 2**31+1)) + # test overflow checking for too big array (issue #7546) dummy_arr = ([0],[0]) half_max = np.iinfo(np.intp).max // 2 @@ -152,6 +175,24 @@ class TestRavelUnravelIndex(object): assert_raises_regex( ValueError, "out of bounds", np.unravel_index, [1], ()) + @pytest.mark.parametrize("mode", ["clip", "wrap", "raise"]) + def test_empty_array_ravel(self, mode): + res = np.ravel_multi_index( + np.zeros((3, 0), dtype=np.intp), (2, 1, 0), mode=mode) + assert(res.shape == (0,)) + + with assert_raises(ValueError): + np.ravel_multi_index( + np.zeros((3, 1), dtype=np.intp), (2, 1, 0), mode=mode) + + def test_empty_array_unravel(self): + res = np.unravel_index(np.zeros(0, dtype=np.intp), (2, 1, 0)) + # res is a tuple of three empty arrays + assert(len(res) == 3) + assert(all(a.shape == (0,) for a in res)) + + with assert_raises(ValueError): + np.unravel_index([1], (2, 1, 0)) class TestGrid(object): def test_basic(self): @@ -167,7 +208,7 @@ class TestGrid(object): assert_almost_equal(a[1]-a[0], 2.0/9.0, 11) def test_linspace_equivalence(self): - y, st = np.linspace(2, 10, retstep=1) + y, st = np.linspace(2, 10, retstep=True) assert_almost_equal(st, 8/49.0) assert_array_almost_equal(y, mgrid[2:10:50j], 13) @@ -272,11 +313,16 @@ class TestIndexExpression(object): class TestIx_(object): def test_regression_1(self): - # Test empty inputs create outputs of indexing type, gh-5804 - # Test both lists and arrays - for func in (range, np.arange): - a, = np.ix_(func(0)) - assert_equal(a.dtype, np.intp) + # Test empty untyped inputs create outputs of indexing type, gh-5804 + a, = np.ix_(range(0)) + assert_equal(a.dtype, np.intp) + + a, = np.ix_([]) + assert_equal(a.dtype, np.intp) + + # but if the type is specified, don't change it + a, = np.ix_(np.array([], dtype=np.float32)) + assert_equal(a.dtype, np.float32) def test_shape_and_dtype(self): sizes = (4, 5, 3, 2) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 7ef25538b..4188265e8 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -22,8 +22,9 @@ from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, - HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, + HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings ) +from numpy.testing._private.utils import requires_memory class TextIO(BytesIO): @@ -87,7 +88,7 @@ class RoundtripTest(object): """ save_kwds = kwargs.get('save_kwds', {}) - load_kwds = kwargs.get('load_kwds', {}) + load_kwds = kwargs.get('load_kwds', {"allow_pickle": True}) file_on_disk = kwargs.get('file_on_disk', False) if file_on_disk: @@ -347,13 +348,23 @@ class TestSaveTxt(object): assert_raises(ValueError, np.savetxt, c, np.array(1)) assert_raises(ValueError, np.savetxt, c, np.array([[[1], [2]]])) - def test_record(self): + def test_structured(self): a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) + def test_structured_padded(self): + # gh-13297 + a = np.array([(1, 2, 3),(4, 5, 6)], dtype=[ + ('foo', 'i4'), ('bar', 'i4'), ('baz', 'i4') + ]) + c = BytesIO() + np.savetxt(c, a[['foo', 'baz']], fmt='%d') + c.seek(0) + assert_equal(c.readlines(), [b'1 3\n', b'4 6\n']) + @pytest.mark.skipif(Path is None, reason="No pathlib.Path") def test_multifield_view(self): a = np.ones(1, dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'f4')]) @@ -494,8 +505,6 @@ class TestSaveTxt(object): b' (3.142e+00-2.718e+00j) (3.142e+00-2.718e+00j)\n']) - - def test_custom_writer(self): class CustomWriter(list): @@ -510,7 +519,7 @@ class TestSaveTxt(object): def test_unicode(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) with tempdir() as tmpdir: # set encoding as on windows it may not be unicode even on py3 np.savetxt(os.path.join(tmpdir, 'test.csv'), a, fmt=['%s'], @@ -518,7 +527,7 @@ class TestSaveTxt(object): def test_unicode_roundtrip(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) # our gz wrapper support encoding suffixes = ['', '.gz'] # stdlib 2 versions do not support encoding @@ -532,12 +541,12 @@ class TestSaveTxt(object): np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a, fmt=['%s'], encoding='UTF-16-LE') b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix), - encoding='UTF-16-LE', dtype=np.unicode) + encoding='UTF-16-LE', dtype=np.unicode_) assert_array_equal(a, b) def test_unicode_bytestream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = BytesIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -545,12 +554,35 @@ class TestSaveTxt(object): def test_unicode_stringstream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = StringIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) assert_equal(s.read(), utf8 + '\n') + @pytest.mark.parametrize("fmt", [u"%f", b"%f"]) + @pytest.mark.parametrize("iotype", [StringIO, BytesIO]) + def test_unicode_and_bytes_fmt(self, fmt, iotype): + # string type of fmt should not matter, see also gh-4053 + a = np.array([1.]) + s = iotype() + np.savetxt(s, a, fmt=fmt) + s.seek(0) + if iotype is StringIO: + assert_equal(s.read(), u"%f\n" % 1.) + else: + assert_equal(s.read(), b"%f\n" % 1.) + + @pytest.mark.skipif(sys.platform=='win32', + reason="large files cause problems") + @pytest.mark.slow + @requires_memory(7e9) + def test_large_zip(self): + # The test takes at least 6GB of memory, writes a file larger than 4GB + test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) + for i in range(800000)], dtype=object) + with tempdir() as tmpdir: + np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) class LoadTxtBase(object): def check_compressed(self, fopen, suffixes): @@ -597,12 +629,12 @@ class LoadTxtBase(object): with temppath() as path: with open(path, "wb") as f: f.write(nonascii.encode("UTF-16")) - x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode) + x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode_) assert_array_equal(x, nonascii) def test_binary_decode(self): utf16 = b'\xff\xfeh\x04 \x00i\x04 \x00j\x04' - v = self.loadfunc(BytesIO(utf16), dtype=np.unicode, encoding='UTF-16') + v = self.loadfunc(BytesIO(utf16), dtype=np.unicode_, encoding='UTF-16') assert_array_equal(v, np.array(utf16.decode('UTF-16').split())) def test_converters_decode(self): @@ -610,7 +642,7 @@ class LoadTxtBase(object): c = TextIO() c.write(b'\xcf\x96') c.seek(0) - x = self.loadfunc(c, dtype=np.unicode, + x = self.loadfunc(c, dtype=np.unicode_, converters={0: lambda x: x.decode('UTF-8')}) a = np.array([b'\xcf\x96'.decode('UTF-8')]) assert_array_equal(x, a) @@ -621,7 +653,7 @@ class LoadTxtBase(object): with temppath() as path: with io.open(path, 'wt', encoding='UTF-8') as f: f.write(utf8) - x = self.loadfunc(path, dtype=np.unicode, + x = self.loadfunc(path, dtype=np.unicode_, converters={0: lambda x: x + 't'}, encoding='UTF-8') a = np.array([utf8 + 't']) @@ -1069,7 +1101,7 @@ class TestLoadTxt(LoadTxtBase): with open(path, "wb") as f: f.write(butf8) with open(path, "rb") as f: - x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode) + x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode_) assert_array_equal(x, sutf8) # test broken latin1 conversion people now rely on with open(path, "rb") as f: @@ -1192,7 +1224,7 @@ class TestFromTxt(LoadTxtBase): def test_record(self): # Test w/ explicit dtype data = TextIO('1 2\n3 4') - test = np.ndfromtxt(data, dtype=[('x', np.int32), ('y', np.int32)]) + test = np.genfromtxt(data, dtype=[('x', np.int32), ('y', np.int32)]) control = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) assert_equal(test, control) # @@ -1201,14 +1233,14 @@ class TestFromTxt(LoadTxtBase): 'formats': ('S1', 'i4', 'f4')} control = np.array([('M', 64.0, 75.0), ('F', 25.0, 60.0)], dtype=descriptor) - test = np.ndfromtxt(data, dtype=descriptor) + test = np.genfromtxt(data, dtype=descriptor) assert_equal(test, control) def test_array(self): # Test outputting a standard ndarray data = TextIO('1 2\n3 4') control = np.array([[1, 2], [3, 4]], dtype=int) - test = np.ndfromtxt(data, dtype=int) + test = np.genfromtxt(data, dtype=int) assert_array_equal(test, control) # data.seek(0) @@ -1221,11 +1253,11 @@ class TestFromTxt(LoadTxtBase): control = np.array([1, 2, 3, 4], int) # data = TextIO('1\n2\n3\n4\n') - test = np.ndfromtxt(data, dtype=int) + test = np.genfromtxt(data, dtype=int) assert_array_equal(test, control) # data = TextIO('1,2,3,4\n') - test = np.ndfromtxt(data, dtype=int, delimiter=',') + test = np.genfromtxt(data, dtype=int, delimiter=',') assert_array_equal(test, control) def test_comments(self): @@ -1233,11 +1265,11 @@ class TestFromTxt(LoadTxtBase): control = np.array([1, 2, 3, 5], int) # Comment on its own line data = TextIO('# comment\n1,2,3,5\n') - test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#') + test = np.genfromtxt(data, dtype=int, delimiter=',', comments='#') assert_equal(test, control) # Comment at the end of a line data = TextIO('1,2,3,5# comment\n') - test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#') + test = np.genfromtxt(data, dtype=int, delimiter=',', comments='#') assert_equal(test, control) def test_skiprows(self): @@ -1246,7 +1278,7 @@ class TestFromTxt(LoadTxtBase): kwargs = dict(dtype=int, delimiter=',') # data = TextIO('comment\n1,2,3,5\n') - test = np.ndfromtxt(data, skip_header=1, **kwargs) + test = np.genfromtxt(data, skip_header=1, **kwargs) assert_equal(test, control) # data = TextIO('# comment\n1,2,3,5\n') @@ -1293,7 +1325,7 @@ class TestFromTxt(LoadTxtBase): data = TextIO('gender age weight\nM 64.0 75.0\nF 25.0 60.0') with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - test = np.ndfromtxt(data, dtype=None, names=True) + test = np.genfromtxt(data, dtype=None, names=True) assert_(w[0].category is np.VisibleDeprecationWarning) control = {'gender': np.array([b'M', b'F']), 'age': np.array([64.0, 25.0]), @@ -1307,7 +1339,7 @@ class TestFromTxt(LoadTxtBase): data = TextIO('A 64 75.0 3+4j True\nBCD 25 60.0 5+6j False') with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - test = np.ndfromtxt(data, dtype=None) + test = np.genfromtxt(data, dtype=None) assert_(w[0].category is np.VisibleDeprecationWarning) control = [np.array([b'A', b'BCD']), np.array([64, 25]), @@ -1321,7 +1353,7 @@ class TestFromTxt(LoadTxtBase): def test_auto_dtype_uniform(self): # Tests whether the output dtype can be uniformized data = TextIO('1 2 3 4\n5 6 7 8\n') - test = np.ndfromtxt(data, dtype=None) + test = np.genfromtxt(data, dtype=None) control = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]) assert_equal(test, control) @@ -1329,7 +1361,7 @@ class TestFromTxt(LoadTxtBase): # Check that a nested dtype isn't MIA data = TextIO('1,2,3.0\n4,5,6.0\n') fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])]) - test = np.ndfromtxt(data, dtype=fancydtype, delimiter=',') + test = np.genfromtxt(data, dtype=fancydtype, delimiter=',') control = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype) assert_equal(test, control) @@ -1339,7 +1371,7 @@ class TestFromTxt(LoadTxtBase): 'formats': ('S1', 'i4', 'f4')} data = TextIO(b'M 64.0 75.0\nF 25.0 60.0') names = ('gender', 'age', 'weight') - test = np.ndfromtxt(data, dtype=descriptor, names=names) + test = np.genfromtxt(data, dtype=descriptor, names=names) descriptor['names'] = names control = np.array([('M', 64.0, 75.0), ('F', 25.0, 60.0)], dtype=descriptor) @@ -1381,12 +1413,25 @@ M 33 21.99 control = np.array([(1, 2), (3, 4)], dtype=[('col1', int), ('col2', int)]) assert_equal(test, control) + def test_file_is_closed_on_error(self): + # gh-13200 + with tempdir() as tmpdir: + fpath = os.path.join(tmpdir, "test.csv") + with open(fpath, "wb") as f: + f.write(u'\N{GREEK PI SYMBOL}'.encode('utf8')) + + # ResourceWarnings are emitted from a destructor, so won't be + # detected by regular propagation to errors. + with assert_no_warnings(): + with pytest.raises(UnicodeDecodeError): + np.genfromtxt(fpath, encoding="ascii") + def test_autonames_and_usecols(self): # Tests names and usecols data = TextIO('A B C D\n aaaa 121 45 9.1') with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - test = np.ndfromtxt(data, usecols=('A', 'C', 'D'), + test = np.genfromtxt(data, usecols=('A', 'C', 'D'), names=True, dtype=None) assert_(w[0].category is np.VisibleDeprecationWarning) control = np.array(('aaaa', 45, 9.1), @@ -1396,7 +1441,7 @@ M 33 21.99 def test_converters_with_usecols(self): # Test the combination user-defined converters and usecol data = TextIO('1,2,3,,5\n6,7,8,9,10\n') - test = np.ndfromtxt(data, dtype=int, delimiter=',', + test = np.genfromtxt(data, dtype=int, delimiter=',', converters={3: lambda s: int(s or - 999)}, usecols=(1, 3,)) control = np.array([[2, -999], [7, 9]], int) @@ -1407,7 +1452,7 @@ M 33 21.99 data = TextIO('A B C D\n aaaa 121 45 9.1') with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - test = np.ndfromtxt(data, usecols=('A', 'C', 'D'), names=True, + test = np.genfromtxt(data, usecols=('A', 'C', 'D'), names=True, dtype=None, converters={'C': lambda s: 2 * int(s)}) assert_(w[0].category is np.VisibleDeprecationWarning) @@ -1420,7 +1465,7 @@ M 33 21.99 converter = { 'date': lambda s: strptime(s, '%Y-%m-%d %H:%M:%SZ')} data = TextIO('2009-02-03 12:00:00Z, 72214.0') - test = np.ndfromtxt(data, delimiter=',', dtype=None, + test = np.genfromtxt(data, delimiter=',', dtype=None, names=['date', 'stid'], converters=converter) control = np.array((datetime(2009, 2, 3), 72214.), dtype=[('date', np.object_), ('stid', float)]) @@ -1431,7 +1476,7 @@ M 33 21.99 converter = { 'date': lambda s: np.datetime64(strptime(s, '%Y-%m-%d %H:%M:%SZ'))} data = TextIO('2009-02-03 12:00:00Z, 72214.0') - test = np.ndfromtxt(data, delimiter=',', dtype=None, + test = np.genfromtxt(data, delimiter=',', dtype=None, names=['date', 'stid'], converters=converter) control = np.array((datetime(2009, 2, 3), 72214.), dtype=[('date', 'datetime64[us]'), ('stid', float)]) @@ -1440,12 +1485,12 @@ M 33 21.99 def test_unused_converter(self): # Test whether unused converters are forgotten data = TextIO("1 21\n 3 42\n") - test = np.ndfromtxt(data, usecols=(1,), + test = np.genfromtxt(data, usecols=(1,), converters={0: lambda s: int(s, 16)}) assert_equal(test, [21, 42]) # data.seek(0) - test = np.ndfromtxt(data, usecols=(1,), + test = np.genfromtxt(data, usecols=(1,), converters={1: lambda s: int(s, 16)}) assert_equal(test, [33, 66]) @@ -1472,12 +1517,12 @@ M 33 21.99 def test_dtype_with_converters(self): dstr = "2009; 23; 46" - test = np.ndfromtxt(TextIO(dstr,), + test = np.genfromtxt(TextIO(dstr,), delimiter=";", dtype=float, converters={0: bytes}) control = np.array([('2009', 23., 46)], dtype=[('f0', '|S4'), ('f1', float), ('f2', float)]) assert_equal(test, control) - test = np.ndfromtxt(TextIO(dstr,), + test = np.genfromtxt(TextIO(dstr,), delimiter=";", dtype=float, converters={0: float}) control = np.array([2009., 23., 46],) assert_equal(test, control) @@ -1517,6 +1562,13 @@ M 33 21.99 test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype, converters=converters) + # nested but empty fields also aren't supported + ndtype = [('idx', int), ('code', object), ('nest', [])] + with assert_raises_regex(NotImplementedError, + 'Nested fields.* not supported.*'): + test = np.genfromtxt(TextIO(data), delimiter=";", + dtype=ndtype, converters=converters) + def test_userconverters_with_explicit_dtype(self): # Test user_converters w/ explicit (standard) dtype data = TextIO('skip,skip,2001-01-01,1.0,skip') @@ -1532,7 +1584,7 @@ M 33 21.99 with open(path, 'wb') as f: f.write(b'skip,skip,2001-01-01' + utf8 + b',1.0,skip') test = np.genfromtxt(path, delimiter=",", names=None, dtype=float, - usecols=(2, 3), converters={2: np.unicode}, + usecols=(2, 3), converters={2: np.compat.unicode}, encoding='UTF-8') control = np.array([('2001-01-01' + utf8.decode('UTF-8'), 1.)], dtype=[('', '|U11'), ('', float)]) @@ -1541,7 +1593,7 @@ M 33 21.99 def test_spacedelimiter(self): # Test space delimiter data = TextIO("1 2 3 4 5\n6 7 8 9 10") - test = np.ndfromtxt(data) + test = np.genfromtxt(data) control = np.array([[1., 2., 3., 4., 5.], [6., 7., 8., 9., 10.]]) assert_equal(test, control) @@ -1555,7 +1607,7 @@ M 33 21.99 def test_missing(self): data = TextIO('1,2,3,,5\n') - test = np.ndfromtxt(data, dtype=int, delimiter=',', + test = np.genfromtxt(data, dtype=int, delimiter=',', converters={3: lambda s: int(s or - 999)}) control = np.array([1, 2, 3, -999, 5], int) assert_equal(test, control) @@ -1577,18 +1629,18 @@ M 33 21.99 data = TextIO() np.savetxt(data, control) data.seek(0) - test = np.ndfromtxt(data, dtype=float, usecols=(1,)) + test = np.genfromtxt(data, dtype=float, usecols=(1,)) assert_equal(test, control[:, 1]) # control = np.array([[1, 2, 3], [3, 4, 5]], float) data = TextIO() np.savetxt(data, control) data.seek(0) - test = np.ndfromtxt(data, dtype=float, usecols=(1, 2)) + test = np.genfromtxt(data, dtype=float, usecols=(1, 2)) assert_equal(test, control[:, 1:]) # Testing with arrays instead of tuples. data.seek(0) - test = np.ndfromtxt(data, dtype=float, usecols=np.array([1, 2])) + test = np.genfromtxt(data, dtype=float, usecols=np.array([1, 2])) assert_equal(test, control[:, 1:]) def test_usecols_as_css(self): @@ -1604,7 +1656,7 @@ M 33 21.99 data = TextIO("JOE 70.1 25.3\nBOB 60.5 27.9") names = ['stid', 'temp'] dtypes = ['S4', 'f8'] - test = np.ndfromtxt( + test = np.genfromtxt( data, usecols=(0, 2), dtype=list(zip(names, dtypes))) assert_equal(test['stid'], [b"JOE", b"BOB"]) assert_equal(test['temp'], [25.3, 27.9]) @@ -1633,11 +1685,15 @@ M 33 21.99 test = np.genfromtxt(data) assert_equal(test, np.array([])) + # when skip_header > 0 + test = np.genfromtxt(data, skip_header=1) + assert_equal(test, np.array([])) + def test_fancy_dtype_alt(self): # Check that a nested dtype isn't MIA data = TextIO('1,2,3.0\n4,5,6.0\n') fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])]) - test = np.mafromtxt(data, dtype=fancydtype, delimiter=',') + test = np.genfromtxt(data, dtype=fancydtype, delimiter=',', usemask=True) control = ma.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype) assert_equal(test, control) @@ -1645,7 +1701,7 @@ M 33 21.99 c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 3))]) - x = np.ndfromtxt(c, dtype=dt) + x = np.genfromtxt(c, dtype=dt) a = np.array([('aaaa', 1.0, 8.0, [[1, 2, 3], [4, 5, 6]])], dtype=dt) assert_array_equal(x, a) @@ -1653,7 +1709,7 @@ M 33 21.99 def test_withmissing(self): data = TextIO('A,B\n0,1\n2,N/A') kwargs = dict(delimiter=",", missing_values="N/A", names=True) - test = np.mafromtxt(data, dtype=None, **kwargs) + test = np.genfromtxt(data, dtype=None, usemask=True, **kwargs) control = ma.array([(0, 1), (2, -1)], mask=[(False, False), (False, True)], dtype=[('A', int), ('B', int)]) @@ -1661,7 +1717,7 @@ M 33 21.99 assert_equal(test.mask, control.mask) # data.seek(0) - test = np.mafromtxt(data, **kwargs) + test = np.genfromtxt(data, usemask=True, **kwargs) control = ma.array([(0, 1), (2, -1)], mask=[(False, False), (False, True)], dtype=[('A', float), ('B', float)]) @@ -1673,7 +1729,7 @@ M 33 21.99 basekwargs = dict(dtype=None, delimiter=",", names=True,) mdtype = [('A', int), ('B', float), ('C', complex)] # - test = np.mafromtxt(TextIO(data), missing_values="N/A", + test = np.genfromtxt(TextIO(data), missing_values="N/A", **basekwargs) control = ma.array([(0, 0.0, 0j), (1, -999, 1j), (-9, 2.2, -999j), (3, -99, 3j)], @@ -1682,16 +1738,17 @@ M 33 21.99 assert_equal(test, control) # basekwargs['dtype'] = mdtype - test = np.mafromtxt(TextIO(data), - missing_values={0: -9, 1: -99, 2: -999j}, **basekwargs) + test = np.genfromtxt(TextIO(data), + missing_values={0: -9, 1: -99, 2: -999j}, usemask=True, **basekwargs) control = ma.array([(0, 0.0, 0j), (1, -999, 1j), (-9, 2.2, -999j), (3, -99, 3j)], mask=[(0, 0, 0), (0, 1, 0), (1, 0, 1), (0, 1, 0)], dtype=mdtype) assert_equal(test, control) # - test = np.mafromtxt(TextIO(data), + test = np.genfromtxt(TextIO(data), missing_values={0: -9, 'B': -99, 'C': -999j}, + usemask=True, **basekwargs) control = ma.array([(0, 0.0, 0j), (1, -999, 1j), (-9, 2.2, -999j), (3, -99, 3j)], @@ -1729,8 +1786,8 @@ M 33 21.99 def test_withmissing_float(self): data = TextIO('A,B\n0,1.5\n2,-999.00') - test = np.mafromtxt(data, dtype=None, delimiter=',', - missing_values='-999.0', names=True,) + test = np.genfromtxt(data, dtype=None, delimiter=',', + missing_values='-999.0', names=True, usemask=True) control = ma.array([(0, 1.5), (2, -1.)], mask=[(False, False), (False, True)], dtype=[('A', int), ('B', float)]) @@ -1769,14 +1826,14 @@ M 33 21.99 ret = {} def f(_ret={}): - _ret['mtest'] = np.ndfromtxt(mdata, invalid_raise=False, **kwargs) + _ret['mtest'] = np.genfromtxt(mdata, invalid_raise=False, **kwargs) assert_warns(ConversionWarning, f, _ret=ret) mtest = ret['mtest'] assert_equal(len(mtest), 45) assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'abcde'])) # mdata.seek(0) - assert_raises(ValueError, np.ndfromtxt, mdata, + assert_raises(ValueError, np.genfromtxt, mdata, delimiter=",", names=True) def test_invalid_raise_with_usecols(self): @@ -1793,14 +1850,14 @@ M 33 21.99 ret = {} def f(_ret={}): - _ret['mtest'] = np.ndfromtxt(mdata, usecols=(0, 4), **kwargs) + _ret['mtest'] = np.genfromtxt(mdata, usecols=(0, 4), **kwargs) assert_warns(ConversionWarning, f, _ret=ret) mtest = ret['mtest'] assert_equal(len(mtest), 45) assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'ae'])) # mdata.seek(0) - mtest = np.ndfromtxt(mdata, usecols=(0, 1), **kwargs) + mtest = np.genfromtxt(mdata, usecols=(0, 1), **kwargs) assert_equal(len(mtest), 50) control = np.ones(50, dtype=[(_, int) for _ in 'ab']) control[[10 * _ for _ in range(5)]] = (2, 2) @@ -1811,7 +1868,7 @@ M 33 21.99 data = ["1, 1, 1, 1, -1.1"] * 50 mdata = TextIO("\n".join(data)) - converters = {4: lambda x: "(%s)" % x} + converters = {4: lambda x: "(%s)" % x.decode()} kwargs = dict(delimiter=",", converters=converters, dtype=[(_, int) for _ in 'abcde'],) assert_raises(ValueError, np.genfromtxt, mdata, **kwargs) @@ -1819,7 +1876,7 @@ M 33 21.99 def test_default_field_format(self): # Test default format data = "0, 1, 2.3\n4, 5, 6.7" - mtest = np.ndfromtxt(TextIO(data), + mtest = np.genfromtxt(TextIO(data), delimiter=",", dtype=None, defaultfmt="f%02i") ctrl = np.array([(0, 1, 2.3), (4, 5, 6.7)], dtype=[("f00", int), ("f01", int), ("f02", float)]) @@ -1828,7 +1885,7 @@ M 33 21.99 def test_single_dtype_wo_names(self): # Test single dtype w/o names data = "0, 1, 2.3\n4, 5, 6.7" - mtest = np.ndfromtxt(TextIO(data), + mtest = np.genfromtxt(TextIO(data), delimiter=",", dtype=float, defaultfmt="f%02i") ctrl = np.array([[0., 1., 2.3], [4., 5., 6.7]], dtype=float) assert_equal(mtest, ctrl) @@ -1836,7 +1893,7 @@ M 33 21.99 def test_single_dtype_w_explicit_names(self): # Test single dtype w explicit names data = "0, 1, 2.3\n4, 5, 6.7" - mtest = np.ndfromtxt(TextIO(data), + mtest = np.genfromtxt(TextIO(data), delimiter=",", dtype=float, names="a, b, c") ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)], dtype=[(_, float) for _ in "abc"]) @@ -1845,7 +1902,7 @@ M 33 21.99 def test_single_dtype_w_implicit_names(self): # Test single dtype w implicit names data = "a, b, c\n0, 1, 2.3\n4, 5, 6.7" - mtest = np.ndfromtxt(TextIO(data), + mtest = np.genfromtxt(TextIO(data), delimiter=",", dtype=float, names=True) ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)], dtype=[(_, float) for _ in "abc"]) @@ -1854,7 +1911,7 @@ M 33 21.99 def test_easy_structured_dtype(self): # Test easy structured dtype data = "0, 1, 2.3\n4, 5, 6.7" - mtest = np.ndfromtxt(TextIO(data), delimiter=",", + mtest = np.genfromtxt(TextIO(data), delimiter=",", dtype=(int, float, float), defaultfmt="f_%02i") ctrl = np.array([(0, 1., 2.3), (4, 5., 6.7)], dtype=[("f_00", int), ("f_01", float), ("f_02", float)]) @@ -1866,14 +1923,14 @@ M 33 21.99 kwargs = dict(delimiter=",", dtype=None) with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - mtest = np.ndfromtxt(TextIO(data), **kwargs) + mtest = np.genfromtxt(TextIO(data), **kwargs) assert_(w[0].category is np.VisibleDeprecationWarning) ctrl = np.array([('01/01/2003 ', 1.3, ' abcde')], dtype=[('f0', '|S12'), ('f1', float), ('f2', '|S8')]) assert_equal(mtest, ctrl) with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) - mtest = np.ndfromtxt(TextIO(data), autostrip=True, **kwargs) + mtest = np.genfromtxt(TextIO(data), autostrip=True, **kwargs) assert_(w[0].category is np.VisibleDeprecationWarning) ctrl = np.array([('01/01/2003', 1.3, 'abcde')], dtype=[('f0', '|S10'), ('f1', float), ('f2', '|S5')]) @@ -1934,12 +1991,12 @@ M 33 21.99 # w/ dtype=None ctrl = np.array([(0, 1, 2), (3, 4, 5)], dtype=[(_, int) for _ in ('A', 'f0', 'C')]) - test = np.ndfromtxt(TextIO(data), dtype=None, **kwargs) + test = np.genfromtxt(TextIO(data), dtype=None, **kwargs) assert_equal(test, ctrl) # w/ default dtype ctrl = np.array([(0, 1, 2), (3, 4, 5)], dtype=[(_, float) for _ in ('A', 'f0', 'C')]) - test = np.ndfromtxt(TextIO(data), **kwargs) + test = np.genfromtxt(TextIO(data), **kwargs) def test_names_auto_completion(self): # Make sure that names are properly completed @@ -1975,13 +2032,13 @@ M 33 21.99 kwargs = dict(delimiter=(5, 5, 4), names=True, dtype=None) ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)], dtype=[('A', int), ('B', int), ('C', float)]) - test = np.ndfromtxt(TextIO(data), **kwargs) + test = np.genfromtxt(TextIO(data), **kwargs) assert_equal(test, ctrl) # kwargs = dict(delimiter=5, names=True, dtype=None) ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)], dtype=[('A', int), ('B', int), ('C', float)]) - test = np.ndfromtxt(TextIO(data), **kwargs) + test = np.genfromtxt(TextIO(data), **kwargs) assert_equal(test, ctrl) def test_filling_values(self): @@ -1989,7 +2046,7 @@ M 33 21.99 data = b"1, 2, 3\n1, , 5\n0, 6, \n" kwargs = dict(delimiter=",", dtype=None, filling_values=-999) ctrl = np.array([[1, 2, 3], [1, -999, 5], [0, 6, -999]], dtype=int) - test = np.ndfromtxt(TextIO(data), **kwargs) + test = np.genfromtxt(TextIO(data), **kwargs) assert_equal(test, ctrl) def test_comments_is_none(self): @@ -2066,7 +2123,7 @@ M 33 21.99 ctl = np.array([ ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"], ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) # test a mixed dtype @@ -2109,7 +2166,7 @@ M 33 21.99 ["norm1", "norm2", "norm3"], ["norm1", latin1, "norm3"], ["test1", "testNonethe" + utf8, "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) def test_recfromtxt(self): @@ -2278,7 +2335,7 @@ M 33 21.99 data = TextIO('73786976294838206464 17179869184 1024') - test = np.ndfromtxt(data, dtype=None) + test = np.genfromtxt(data, dtype=None) assert_equal(test.dtype.names, ['f0', 'f1', 'f2']) @@ -2342,7 +2399,7 @@ class TestPathUsage(object): np.savez(path, lab='place holder') with np.load(path) as data: assert_array_equal(data['lab'], 'place holder') - + def test_savez_compressed_load(self): # Test that pathlib.Path instances can be used with savez. with temppath(suffix='.npz') as path: @@ -2368,7 +2425,7 @@ class TestPathUsage(object): f.write(u'1 2\n3 4') control = np.array([[1, 2], [3, 4]], dtype=int) - test = np.ndfromtxt(path, dtype=int) + test = np.genfromtxt(path, dtype=int) assert_array_equal(test, control) def test_mafromtxt(self): @@ -2378,7 +2435,7 @@ class TestPathUsage(object): with path.open('w') as f: f.write(u'1,2,3.0\n4,5,6.0\n') - test = np.mafromtxt(path, delimiter=',') + test = np.genfromtxt(path, delimiter=',', usemask=True) control = ma.array([(1.0, 2.0, 3.0), (4.0, 5.0, 6.0)]) assert_equal(test, control) @@ -2423,6 +2480,44 @@ def test_gzip_load(): assert_array_equal(np.load(f), a) +# These next two classes encode the minimal API needed to save()/load() arrays. +# The `test_ducktyping` ensures they work correctly +class JustWriter(object): + def __init__(self, base): + self.base = base + + def write(self, s): + return self.base.write(s) + + def flush(self): + return self.base.flush() + +class JustReader(object): + def __init__(self, base): + self.base = base + + def read(self, n): + return self.base.read(n) + + def seek(self, off, whence=0): + return self.base.seek(off, whence) + + +def test_ducktyping(): + a = np.random.random((5, 5)) + + s = BytesIO() + f = JustWriter(s) + + np.save(f, a) + f.flush() + s.seek(0) + + f = JustReader(s) + assert_array_equal(np.load(f), a) + + + def test_gzip_loadtxt(): # Thanks to another windows brokenness, we can't use # NamedTemporaryFile: a file created from this function cannot be diff --git a/numpy/lib/tests/test_nanfunctions.py b/numpy/lib/tests/test_nanfunctions.py index 504372faf..da2d0cc52 100644 --- a/numpy/lib/tests/test_nanfunctions.py +++ b/numpy/lib/tests/test_nanfunctions.py @@ -1,8 +1,10 @@ from __future__ import division, absolute_import, print_function import warnings +import pytest import numpy as np +from numpy.lib.nanfunctions import _nan_mask, _replace_nan from numpy.testing import ( assert_, assert_equal, assert_almost_equal, assert_no_warnings, assert_raises, assert_array_equal, suppress_warnings @@ -925,3 +927,56 @@ class TestNanFunctions_Quantile(object): p = p.tolist() np.nanquantile(np.arange(100.), p, interpolation="midpoint") assert_array_equal(p, p0) + +@pytest.mark.parametrize("arr, expected", [ + # array of floats with some nans + (np.array([np.nan, 5.0, np.nan, np.inf]), + np.array([False, True, False, True])), + # int64 array that can't possibly have nans + (np.array([1, 5, 7, 9], dtype=np.int64), + True), + # bool array that can't possibly have nans + (np.array([False, True, False, True]), + True), + # 2-D complex array with nans + (np.array([[np.nan, 5.0], + [np.nan, np.inf]], dtype=np.complex64), + np.array([[False, True], + [False, True]])), + ]) +def test__nan_mask(arr, expected): + for out in [None, np.empty(arr.shape, dtype=np.bool_)]: + actual = _nan_mask(arr, out=out) + assert_equal(actual, expected) + # the above won't distinguish between True proper + # and an array of True values; we want True proper + # for types that can't possibly contain NaN + if type(expected) is not np.ndarray: + assert actual is True + + +def test__replace_nan(): + """ Test that _replace_nan returns the original array if there are no + NaNs, not a copy. + """ + for dtype in [np.bool, np.int32, np.int64]: + arr = np.array([0, 1], dtype=dtype) + result, mask = _replace_nan(arr, 0) + assert mask is None + # do not make a copy if there are no nans + assert result is arr + + for dtype in [np.float32, np.float64]: + arr = np.array([0, 1], dtype=dtype) + result, mask = _replace_nan(arr, 2) + assert (mask == False).all() + # mask is not None, so we make a copy + assert result is not arr + assert_equal(result, arr) + + arr_nan = np.array([0, 1, np.nan], dtype=dtype) + result_nan, mask_nan = _replace_nan(arr_nan, 2) + assert_equal(mask_nan, np.array([False, False, True])) + assert result_nan is not arr_nan + assert_equal(result_nan, np.array([0, 1, 2])) + assert np.isnan(arr_nan[-1]) diff --git a/numpy/lib/tests/test_packbits.py b/numpy/lib/tests/test_packbits.py index fde5c37f2..95a465c36 100644 --- a/numpy/lib/tests/test_packbits.py +++ b/numpy/lib/tests/test_packbits.py @@ -2,7 +2,8 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.testing import assert_array_equal, assert_equal, assert_raises - +import pytest +from itertools import chain def test_packbits(): # Copied from the docstring. @@ -50,8 +51,8 @@ def test_packbits_empty_with_axis(): assert_equal(b.dtype, np.uint8) assert_equal(b.shape, out_shape) - -def test_packbits_large(): +@pytest.mark.parametrize('bitorder', ('little', 'big')) +def test_packbits_large(bitorder): # test data large enough for 16 byte vectorization a = np.array([1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, @@ -71,7 +72,7 @@ def test_packbits_large(): a = a.repeat(3) for dtype in '?bBhHiIlLqQ': arr = np.array(a, dtype=dtype) - b = np.packbits(arr, axis=None) + b = np.packbits(arr, axis=None, bitorder=bitorder) assert_equal(b.dtype, np.uint8) r = [252, 127, 192, 3, 254, 7, 252, 0, 7, 31, 240, 0, 28, 1, 255, 252, 113, 248, 3, 255, 192, 28, 15, 192, 28, 126, 0, 224, 127, 255, @@ -81,9 +82,10 @@ def test_packbits_large(): 255, 224, 1, 255, 252, 126, 63, 0, 1, 192, 252, 14, 63, 0, 15, 199, 252, 113, 255, 3, 128, 56, 252, 14, 7, 0, 113, 255, 255, 142, 56, 227, 129, 248, 227, 129, 199, 31, 128] - assert_array_equal(b, r) + if bitorder == 'big': + assert_array_equal(b, r) # equal for size being multiple of 8 - assert_array_equal(np.unpackbits(b)[:-4], a) + assert_array_equal(np.unpackbits(b, bitorder=bitorder)[:-4], a) # check last byte of different remainders (16 byte vectorization) b = [np.packbits(arr[:-i], axis=None)[-1] for i in range(1, 16)] @@ -229,6 +231,20 @@ def test_unpackbits(): [0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 0, 1, 1, 1]])) +def test_pack_unpack_order(): + a = np.array([[2], [7], [23]], dtype=np.uint8) + b = np.unpackbits(a, axis=1) + assert_equal(b.dtype, np.uint8) + b_little = np.unpackbits(a, axis=1, bitorder='little') + b_big = np.unpackbits(a, axis=1, bitorder='big') + assert_array_equal(b, b_big) + assert_array_equal(a, np.packbits(b_little, axis=1, bitorder='little')) + assert_array_equal(b[:,::-1], b_little) + assert_array_equal(a, np.packbits(b_big, axis=1, bitorder='big')) + assert_raises(ValueError, np.unpackbits, a, bitorder='r') + assert_raises(TypeError, np.unpackbits, a, bitorder=10) + + def test_unpackbits_empty(): a = np.empty((0,), dtype=np.uint8) @@ -266,3 +282,97 @@ def test_unpackbits_large(): assert_array_equal(np.packbits(np.unpackbits(d, axis=1), axis=1), d) d = d.T.copy() assert_array_equal(np.packbits(np.unpackbits(d, axis=0), axis=0), d) + + +class TestCount(): + x = np.array([ + [1, 0, 1, 0, 0, 1, 0], + [0, 1, 1, 1, 0, 0, 0], + [0, 0, 1, 0, 0, 1, 1], + [1, 1, 0, 0, 0, 1, 1], + [1, 0, 1, 0, 1, 0, 1], + [0, 0, 1, 1, 1, 0, 0], + [0, 1, 0, 1, 0, 1, 0], + ], dtype=np.uint8) + padded1 = np.zeros(57, dtype=np.uint8) + padded1[:49] = x.ravel() + padded1b = np.zeros(57, dtype=np.uint8) + padded1b[:49] = x[::-1].copy().ravel() + padded2 = np.zeros((9, 9), dtype=np.uint8) + padded2[:7, :7] = x + + @pytest.mark.parametrize('bitorder', ('little', 'big')) + @pytest.mark.parametrize('count', chain(range(58), range(-1, -57, -1))) + def test_roundtrip(self, bitorder, count): + if count < 0: + # one extra zero of padding + cutoff = count - 1 + else: + cutoff = count + # test complete invertibility of packbits and unpackbits with count + packed = np.packbits(self.x, bitorder=bitorder) + unpacked = np.unpackbits(packed, count=count, bitorder=bitorder) + assert_equal(unpacked.dtype, np.uint8) + assert_array_equal(unpacked, self.padded1[:cutoff]) + + @pytest.mark.parametrize('kwargs', [ + {}, {'count': None}, + ]) + def test_count(self, kwargs): + packed = np.packbits(self.x) + unpacked = np.unpackbits(packed, **kwargs) + assert_equal(unpacked.dtype, np.uint8) + assert_array_equal(unpacked, self.padded1[:-1]) + + @pytest.mark.parametrize('bitorder', ('little', 'big')) + # delta==-1 when count<0 because one extra zero of padding + @pytest.mark.parametrize('count', chain(range(8), range(-1, -9, -1))) + def test_roundtrip_axis(self, bitorder, count): + if count < 0: + # one extra zero of padding + cutoff = count - 1 + else: + cutoff = count + packed0 = np.packbits(self.x, axis=0, bitorder=bitorder) + unpacked0 = np.unpackbits(packed0, axis=0, count=count, + bitorder=bitorder) + assert_equal(unpacked0.dtype, np.uint8) + assert_array_equal(unpacked0, self.padded2[:cutoff, :self.x.shape[1]]) + + packed1 = np.packbits(self.x, axis=1, bitorder=bitorder) + unpacked1 = np.unpackbits(packed1, axis=1, count=count, + bitorder=bitorder) + assert_equal(unpacked1.dtype, np.uint8) + assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :cutoff]) + + @pytest.mark.parametrize('kwargs', [ + {}, {'count': None}, + {'bitorder' : 'little'}, + {'bitorder': 'little', 'count': None}, + {'bitorder' : 'big'}, + {'bitorder': 'big', 'count': None}, + ]) + def test_axis_count(self, kwargs): + packed0 = np.packbits(self.x, axis=0) + unpacked0 = np.unpackbits(packed0, axis=0, **kwargs) + assert_equal(unpacked0.dtype, np.uint8) + if kwargs.get('bitorder', 'big') == 'big': + assert_array_equal(unpacked0, self.padded2[:-1, :self.x.shape[1]]) + else: + assert_array_equal(unpacked0[::-1, :], self.padded2[:-1, :self.x.shape[1]]) + + packed1 = np.packbits(self.x, axis=1) + unpacked1 = np.unpackbits(packed1, axis=1, **kwargs) + assert_equal(unpacked1.dtype, np.uint8) + if kwargs.get('bitorder', 'big') == 'big': + assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :-1]) + else: + assert_array_equal(unpacked1[:, ::-1], self.padded2[:self.x.shape[0], :-1]) + + def test_bad_count(self): + packed0 = np.packbits(self.x, axis=0) + assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9) + packed1 = np.packbits(self.x, axis=1) + assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9) + packed = np.packbits(self.x) + assert_raises(ValueError, np.unpackbits, packed, count=-57) diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index 77414ba7c..89759bd83 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -246,16 +246,16 @@ class TestPolynomial(object): assert_equal(r.coeffs.dtype, np.complex128) assert_equal(q*a + r, b) - def test_poly_coeffs_immutable(self): - """ Coefficients should not be modifiable """ + def test_poly_coeffs_mutable(self): + """ Coefficients should be modifiable """ p = np.poly1d([1, 2, 3]) - try: - # despite throwing an exception, this used to change state - p.coeffs += 1 - except Exception: - pass - assert_equal(p.coeffs, [1, 2, 3]) + p.coeffs += 1 + assert_equal(p.coeffs, [2, 3, 4]) p.coeffs[2] += 10 - assert_equal(p.coeffs, [1, 2, 3]) + assert_equal(p.coeffs, [2, 3, 14]) + + # this never used to be allowed - let's not add features to deprecated + # APIs + assert_raises(AttributeError, setattr, p, 'coeffs', np.array(1)) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 069693613..fa5f4dec2 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -12,9 +12,11 @@ from numpy.lib.recfunctions import ( find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, repack_fields, unstructured_to_structured, structured_to_unstructured, apply_along_fields, require_fields, assign_fields_by_name) +get_fieldspec = np.lib.recfunctions._get_fieldspec get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat -zip_descr = np.lib.recfunctions.zip_descr +zip_descr = np.lib.recfunctions._zip_descr +zip_dtype = np.lib.recfunctions._zip_dtype class TestRecFunctions(object): @@ -89,8 +91,10 @@ class TestRecFunctions(object): control = np.array([(1,), (4,)], dtype=[('a', int)]) assert_equal(test, control) + # dropping all fields results in an array with no fields test = drop_fields(a, ['a', 'b']) - assert_(test is None) + control = np.array([(), ()], dtype=[]) + assert_equal(test, control) def test_rename_fields(self): # Test rename fields @@ -113,6 +117,14 @@ class TestRecFunctions(object): test = get_names(ndtype) assert_equal(test, ('a', ('b', ('ba', 'bb')))) + ndtype = np.dtype([('a', int), ('b', [])]) + test = get_names(ndtype) + assert_equal(test, ('a', ('b', ()))) + + ndtype = np.dtype([]) + test = get_names(ndtype) + assert_equal(test, ()) + def test_get_names_flat(self): # Test get_names_flat ndtype = np.dtype([('A', '|S3'), ('B', float)]) @@ -123,6 +135,14 @@ class TestRecFunctions(object): test = get_names_flat(ndtype) assert_equal(test, ('a', 'b', 'ba', 'bb')) + ndtype = np.dtype([('a', int), ('b', [])]) + test = get_names_flat(ndtype) + assert_equal(test, ('a', 'b')) + + ndtype = np.dtype([]) + test = get_names_flat(ndtype) + assert_equal(test, ()) + def test_get_fieldstructure(self): # Test get_fieldstructure @@ -145,6 +165,11 @@ class TestRecFunctions(object): 'BBA': ['B', 'BB'], 'BBB': ['B', 'BB']} assert_equal(test, control) + # 0 fields + ndtype = np.dtype([]) + test = get_fieldstructure(ndtype) + assert_equal(test, {}) + def test_find_duplicates(self): # Test find_duplicates a = ma.array([(2, (2., 'B')), (1, (2., 'B')), (2, (2., 'B')), @@ -214,6 +239,8 @@ class TestRecFunctions(object): dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + out = np.mean(structured_to_unstructured(b[['x']]), axis=-1) + assert_equal(out, np.array([ 1. , 4. , 7. , 10. ])) c = np.arange(20).reshape((4,5)) out = unstructured_to_structured(c, a.dtype) @@ -241,6 +268,16 @@ class TestRecFunctions(object): assert_(dd.base is d) assert_(ddd.base is d) + # including uniform fields with subarrays unpacked + d = np.array([(1, [2, 3], [[ 4, 5], [ 6, 7]]), + (8, [9, 10], [[11, 12], [13, 14]])], + dtype=[('x0', 'i4'), ('x1', ('i4', 2)), + ('x2', ('i4', (2, 2)))]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + # test that nested fields with identical names don't break anything point = np.dtype([('x', int), ('y', int)]) triangle = np.dtype([('a', point), ('b', point), ('c', point)]) @@ -249,6 +286,40 @@ class TestRecFunctions(object): assert_equal(res, np.zeros((10, 6), dtype=int)) + # test nested combinations of subarrays and structured arrays, gh-13333 + def subarray(dt, shape): + return np.dtype((dt, shape)) + + def structured(*dts): + return np.dtype([('x{}'.format(i), dt) for i, dt in enumerate(dts)]) + + def inspect(dt, dtype=None): + arr = np.zeros((), dt) + ret = structured_to_unstructured(arr, dtype=dtype) + backarr = unstructured_to_structured(ret, dt) + return ret.shape, ret.dtype, backarr.dtype + + dt = structured(subarray(structured(np.int32, np.int32), 3)) + assert_equal(inspect(dt), ((6,), np.int32, dt)) + + dt = structured(subarray(subarray(np.int32, 2), 2)) + assert_equal(inspect(dt), ((4,), np.int32, dt)) + + dt = structured(np.int32) + assert_equal(inspect(dt), ((1,), np.int32, dt)) + + dt = structured(np.int32, subarray(subarray(np.int32, 2), 2)) + assert_equal(inspect(dt), ((5,), np.int32, dt)) + + dt = structured() + assert_raises(ValueError, structured_to_unstructured, np.zeros(3, dt)) + + # these currently don't work, but we may make it work in the future + assert_raises(NotImplementedError, structured_to_unstructured, + np.zeros(3, dt), dtype=np.int32) + assert_raises(NotImplementedError, unstructured_to_structured, + np.zeros((3,0), dtype=np.int32)) + def test_field_assignment_by_name(self): a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) newdt = [('b', 'f4'), ('c', 'u1')] @@ -309,8 +380,8 @@ class TestMergeArrays(object): z = np.array( [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)]) w = np.array( - [(1, (2, 3.0)), (4, (5, 6.0))], - dtype=[('a', int), ('b', [('ba', float), ('bb', int)])]) + [(1, (2, 3.0, ())), (4, (5, 6.0, ()))], + dtype=[('a', int), ('b', [('ba', float), ('bb', int), ('bc', [])])]) self.data = (w, x, y, z) def test_solo(self): @@ -381,8 +452,8 @@ class TestMergeArrays(object): test = merge_arrays((x, w), flatten=False) controldtype = [('f0', int), ('f1', [('a', int), - ('b', [('ba', float), ('bb', int)])])] - control = np.array([(1., (1, (2, 3.0))), (2, (4, (5, 6.0)))], + ('b', [('ba', float), ('bb', int), ('bc', [])])])] + control = np.array([(1., (1, (2, 3.0, ()))), (2, (4, (5, 6.0, ())))], dtype=controldtype) assert_equal(test, control) diff --git a/numpy/lib/tests/test_regression.py b/numpy/lib/tests/test_regression.py index 4c46bc46b..4cd812f5d 100644 --- a/numpy/lib/tests/test_regression.py +++ b/numpy/lib/tests/test_regression.py @@ -21,8 +21,8 @@ class TestRegression(object): # Ticket #91 x = np.random.random((3, 3)) y = x.copy() - np.cov(x, rowvar=1) - np.cov(y, rowvar=0) + np.cov(x, rowvar=True) + np.cov(y, rowvar=False) assert_array_equal(x, y) def test_mem_digitize(self): @@ -56,7 +56,7 @@ class TestRegression(object): def test_poly1d_nan_roots(self): # Ticket #396 - p = np.poly1d([np.nan, np.nan, 1], r=0) + p = np.poly1d([np.nan, np.nan, 1], r=False) assert_raises(np.linalg.LinAlgError, getattr, p, "r") def test_mem_polymul(self): diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index 01ea028bb..be1604a75 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -289,14 +289,26 @@ class TestExpandDims(object): assert_(b.shape[axis] == 1) assert_(np.squeeze(b).shape == s) - def test_deprecations(self): - # 2017-05-17, 1.13.0 + def test_axis_tuple(self): + a = np.empty((3, 3, 3)) + assert np.expand_dims(a, axis=(0, 1, 2)).shape == (1, 1, 1, 3, 3, 3) + assert np.expand_dims(a, axis=(0, -1, -2)).shape == (1, 3, 3, 3, 1, 1) + assert np.expand_dims(a, axis=(0, 3, 5)).shape == (1, 3, 3, 1, 3, 1) + assert np.expand_dims(a, axis=(0, -3, -5)).shape == (1, 1, 3, 1, 3, 3) + + def test_axis_out_of_range(self): s = (2, 3, 4, 5) a = np.empty(s) - with warnings.catch_warnings(): - warnings.simplefilter("always") - assert_warns(DeprecationWarning, expand_dims, a, -6) - assert_warns(DeprecationWarning, expand_dims, a, 5) + assert_raises(np.AxisError, expand_dims, a, -6) + assert_raises(np.AxisError, expand_dims, a, 5) + + a = np.empty((3, 3, 3)) + assert_raises(np.AxisError, expand_dims, a, (0, -6)) + assert_raises(np.AxisError, expand_dims, a, (0, 5)) + + def test_repeated_axis(self): + a = np.empty((3, 3, 3)) + assert_raises(ValueError, expand_dims, a, axis=(1, 1)) def test_subclasses(self): a = np.arange(10).reshape((2, 5)) diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index b2bd7da3e..85fcceedc 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -4,7 +4,7 @@ import numpy as np from numpy.core._rational_tests import rational from numpy.testing import ( assert_equal, assert_array_equal, assert_raises, assert_, - assert_raises_regex + assert_raises_regex, assert_warns, ) from numpy.lib.stride_tricks import ( as_strided, broadcast_arrays, _broadcast_shape, broadcast_to @@ -415,12 +415,32 @@ def test_writeable(): assert_equal(result.flags.writeable, False) assert_raises(ValueError, result.__setitem__, slice(None), 0) - # but the result of broadcast_arrays needs to be writeable (for now), to + # but the result of broadcast_arrays needs to be writeable, to # preserve backwards compatibility + for is_broadcast, results in [(False, broadcast_arrays(original,)), + (True, broadcast_arrays(0, original))]: + for result in results: + # This will change to False in a future version + if is_broadcast: + with assert_warns(FutureWarning): + assert_equal(result.flags.writeable, True) + with assert_warns(DeprecationWarning): + result[:] = 0 + # Warning not emitted, writing to the array resets it + assert_equal(result.flags.writeable, True) + else: + # No warning: + assert_equal(result.flags.writeable, True) + for results in [broadcast_arrays(original), broadcast_arrays(0, original)]: for result in results: + # resets the warn_on_write DeprecationWarning + result.flags.writeable = True + # check: no warning emitted assert_equal(result.flags.writeable, True) + result[:] = 0 + # keep readonly input readonly original.flags.writeable = False _, result = broadcast_arrays(0, original) @@ -435,6 +455,25 @@ def test_writeable(): assert_(first.shape == second.shape) +def test_writeable_memoryview(): + # The result of broadcast_arrays exports as a non-writeable memoryview + # because otherwise there is no good way to opt in to the new behaviour + # (i.e. you would need to set writeable to False explicitly). + # See gh-13929. + original = np.array([1, 2, 3]) + + for is_broadcast, results in [(False, broadcast_arrays(original,)), + (True, broadcast_arrays(0, original))]: + for result in results: + # This will change to False in a future version + if is_broadcast: + # memoryview(result, writable=True) will give warning but cannot + # be tested using the python API. + assert memoryview(result).readonly + else: + assert not memoryview(result).readonly + + def test_reference_types(): input_array = np.array('a', dtype=object) expected = np.array(['a'] * 3, dtype=object) diff --git a/numpy/lib/tests/test_twodim_base.py b/numpy/lib/tests/test_twodim_base.py index bf93b4adb..bb844e4bd 100644 --- a/numpy/lib/tests/test_twodim_base.py +++ b/numpy/lib/tests/test_twodim_base.py @@ -5,7 +5,7 @@ from __future__ import division, absolute_import, print_function from numpy.testing import ( assert_equal, assert_array_equal, assert_array_max_ulp, - assert_array_almost_equal, assert_raises, + assert_array_almost_equal, assert_raises, assert_ ) from numpy import ( @@ -17,6 +17,9 @@ from numpy import ( import numpy as np +from numpy.core.tests.test_overrides import requires_array_function + + def get_mat(n): data = arange(n) data = add.outer(data, data) @@ -273,6 +276,27 @@ class TestHistogram2d(object): assert_array_equal(H, answer) assert_array_equal(xe, array([0., 0.25, 0.5, 0.75, 1])) + @requires_array_function + def test_dispatch(self): + class ShouldDispatch: + def __array_function__(self, function, types, args, kwargs): + return types, args, kwargs + + xy = [1, 2] + s_d = ShouldDispatch() + r = histogram2d(s_d, xy) + # Cannot use assert_equal since that dispatches... + assert_(r == ((ShouldDispatch,), (s_d, xy), {})) + r = histogram2d(xy, s_d) + assert_(r == ((ShouldDispatch,), (xy, s_d), {})) + r = histogram2d(xy, xy, bins=s_d) + assert_(r, ((ShouldDispatch,), (xy, xy), dict(bins=s_d))) + r = histogram2d(xy, xy, bins=[s_d, 5]) + assert_(r, ((ShouldDispatch,), (xy, xy), dict(bins=[s_d, 5]))) + assert_raises(Exception, histogram2d, xy, xy, bins=[s_d]) + r = histogram2d(xy, xy, weights=s_d) + assert_(r, ((ShouldDispatch,), (xy, xy), dict(weights=s_d))) + class TestTri(object): def test_dtype(self): diff --git a/numpy/lib/tests/test_type_check.py b/numpy/lib/tests/test_type_check.py index 2982ca31a..b3f114b92 100644 --- a/numpy/lib/tests/test_type_check.py +++ b/numpy/lib/tests/test_type_check.py @@ -360,6 +360,14 @@ class TestNanToNum(object): assert_(vals[1] == 0) assert_all(vals[2] > 1e10) and assert_all(np.isfinite(vals[2])) assert_equal(type(vals), np.ndarray) + + # perform the same tests but with nan, posinf and neginf keywords + with np.errstate(divide='ignore', invalid='ignore'): + vals = nan_to_num(np.array((-1., 0, 1))/0., + nan=10, posinf=20, neginf=30) + assert_equal(vals, [30, 10, 20]) + assert_all(np.isfinite(vals[[0, 2]])) + assert_equal(type(vals), np.ndarray) # perform the same test but in-place with np.errstate(divide='ignore', invalid='ignore'): @@ -371,26 +379,48 @@ class TestNanToNum(object): assert_(vals[1] == 0) assert_all(vals[2] > 1e10) and assert_all(np.isfinite(vals[2])) assert_equal(type(vals), np.ndarray) + + # perform the same test but in-place + with np.errstate(divide='ignore', invalid='ignore'): + vals = np.array((-1., 0, 1))/0. + result = nan_to_num(vals, copy=False, nan=10, posinf=20, neginf=30) + + assert_(result is vals) + assert_equal(vals, [30, 10, 20]) + assert_all(np.isfinite(vals[[0, 2]])) + assert_equal(type(vals), np.ndarray) def test_array(self): vals = nan_to_num([1]) assert_array_equal(vals, np.array([1], int)) assert_equal(type(vals), np.ndarray) + vals = nan_to_num([1], nan=10, posinf=20, neginf=30) + assert_array_equal(vals, np.array([1], int)) + assert_equal(type(vals), np.ndarray) def test_integer(self): vals = nan_to_num(1) assert_all(vals == 1) assert_equal(type(vals), np.int_) + vals = nan_to_num(1, nan=10, posinf=20, neginf=30) + assert_all(vals == 1) + assert_equal(type(vals), np.int_) def test_float(self): vals = nan_to_num(1.0) assert_all(vals == 1.0) assert_equal(type(vals), np.float_) + vals = nan_to_num(1.1, nan=10, posinf=20, neginf=30) + assert_all(vals == 1.1) + assert_equal(type(vals), np.float_) def test_complex_good(self): vals = nan_to_num(1+1j) assert_all(vals == 1+1j) assert_equal(type(vals), np.complex_) + vals = nan_to_num(1+1j, nan=10, posinf=20, neginf=30) + assert_all(vals == 1+1j) + assert_equal(type(vals), np.complex_) def test_complex_bad(self): with np.errstate(divide='ignore', invalid='ignore'): @@ -414,6 +444,16 @@ class TestNanToNum(object): # !! inf. Comment out for now, and see if it # !! changes #assert_all(vals.real < -1e10) and assert_all(np.isfinite(vals)) + + def test_do_not_rewrite_previous_keyword(self): + # This is done to test that when, for instance, nan=np.inf then these + # values are not rewritten by posinf keyword to the posinf value. + with np.errstate(divide='ignore', invalid='ignore'): + vals = nan_to_num(np.array((-1., 0, 1))/0., nan=np.inf, posinf=999) + assert_all(np.isfinite(vals[[0, 2]])) + assert_all(vals[0] < -1e10) + assert_equal(vals[[1, 2]], [np.inf, 999]) + assert_equal(type(vals), np.ndarray) class TestRealIfClose(object): diff --git a/numpy/lib/tests/test_ufunclike.py b/numpy/lib/tests/test_ufunclike.py index 0f06876a1..64280616f 100644 --- a/numpy/lib/tests/test_ufunclike.py +++ b/numpy/lib/tests/test_ufunclike.py @@ -21,7 +21,7 @@ class TestUfunclike(object): assert_equal(res, tgt) assert_equal(out, tgt) - a = a.astype(np.complex) + a = a.astype(np.complex_) with assert_raises(TypeError): ufl.isposinf(a) @@ -36,7 +36,7 @@ class TestUfunclike(object): assert_equal(res, tgt) assert_equal(out, tgt) - a = a.astype(np.complex) + a = a.astype(np.complex_) with assert_raises(TypeError): ufl.isneginf(a) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 2723f3440..9673a05fa 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -1,5 +1,6 @@ from __future__ import division, absolute_import, print_function +import inspect import sys import pytest @@ -38,6 +39,32 @@ def old_func3(self, x): new_func3 = deprecate(old_func3, old_name="old_func3", new_name="new_func3") +def old_func4(self, x): + """Summary. + + Further info. + """ + return x +new_func4 = deprecate(old_func4) + + +def old_func5(self, x): + """Summary. + + Bizarre indentation. + """ + return x +new_func5 = deprecate(old_func5) + + +def old_func6(self, x): + """ + Also in PEP-257. + """ + return x +new_func6 = deprecate(old_func6) + + def test_deprecate_decorator(): assert_('deprecated' in old_func.__doc__) @@ -51,6 +78,25 @@ def test_deprecate_fn(): assert_('new_func3' in new_func3.__doc__) +@pytest.mark.skipif(sys.flags.optimize == 2, reason="-OO discards docstrings") +def test_deprecate_help_indentation(): + _compare_docs(old_func4, new_func4) + _compare_docs(old_func5, new_func5) + _compare_docs(old_func6, new_func6) + + +def _compare_docs(old_func, new_func): + old_doc = inspect.getdoc(old_func) + new_doc = inspect.getdoc(new_func) + index = new_doc.index('\n\n') + 2 + assert_equal(new_doc[index:], old_doc) + + +@pytest.mark.skipif(sys.flags.optimize == 2, reason="-OO discards docstrings") +def test_deprecate_preserve_whitespace(): + assert_('\n Bizarre' in new_func5.__doc__) + + def test_safe_eval_nameconstant(): # Test if safe_eval supports Python 3.4 _ast.NameConstant utils.safe_eval('None') diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index e165c9b02..f45392188 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -368,7 +368,7 @@ def tri(N, M=None, k=0, dtype=float): ------- tri : ndarray of shape (N, M) Array with its lower triangle filled with ones and zero elsewhere; - in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise. + in other words ``T[i,j] == 1`` for ``j <= i + k``, 0 otherwise. Examples -------- @@ -565,7 +565,20 @@ def vander(x, N=None, increasing=False): def _histogram2d_dispatcher(x, y, bins=None, range=None, normed=None, weights=None, density=None): - return (x, y, bins, weights) + yield x + yield y + + # This terrible logic is adapted from the checks in histogram2d + try: + N = len(bins) + except TypeError: + N = 1 + if N == 2: + yield from bins # bins=[x, y] + else: + yield bins + + yield weights @array_function_dispatch(_histogram2d_dispatcher) diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index f55517732..977117235 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -68,16 +68,14 @@ def mintypecode(typechars, typeset='GDFgdf', default='d'): 'G' """ - typecodes = [(isinstance(t, str) and t) or asarray(t).dtype.char - for t in typechars] - intersection = [t for t in typecodes if t in typeset] + typecodes = ((isinstance(t, str) and t) or asarray(t).dtype.char + for t in typechars) + intersection = set(t for t in typecodes if t in typeset) if not intersection: return default if 'F' in intersection and 'd' in intersection: return 'D' - l = [(_typecodes_by_elsize.index(t), t) for t in intersection] - l.sort() - return l[0][1] + return min(intersection, key=_typecodes_by_elsize.index) def _asfarray_dispatcher(a, dtype=None): @@ -363,18 +361,23 @@ def _getmaxmin(t): return f.max, f.min -def _nan_to_num_dispatcher(x, copy=None): +def _nan_to_num_dispatcher(x, copy=None, nan=None, posinf=None, neginf=None): return (x,) @array_function_dispatch(_nan_to_num_dispatcher) -def nan_to_num(x, copy=True): +def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None): """ - Replace NaN with zero and infinity with large finite numbers. + Replace NaN with zero and infinity with large finite numbers (default + behaviour) or with the numbers defined by the user using the `nan`, + `posinf` and/or `neginf` keywords. - If `x` is inexact, NaN is replaced by zero, and infinity and -infinity - replaced by the respectively largest and most negative finite floating - point values representable by ``x.dtype``. + If `x` is inexact, NaN is replaced by zero or by the user defined value in + `nan` keyword, infinity is replaced by the largest finite floating point + values representable by ``x.dtype`` or by the user defined value in + `posinf` keyword and -infinity is replaced by the most negative finite + floating point values representable by ``x.dtype`` or by the user defined + value in `neginf` keyword. For complex dtypes, the above is applied to each of the real and imaginary components of `x` separately. @@ -390,8 +393,27 @@ def nan_to_num(x, copy=True): in-place (False). The in-place operation only occurs if casting to an array does not require a copy. Default is True. - + .. versionadded:: 1.13 + nan : int, float, optional + Value to be used to fill NaN values. If no value is passed + then NaN values will be replaced with 0.0. + + .. versionadded:: 1.17 + posinf : int, float, optional + Value to be used to fill positive infinity values. If no value is + passed then positive infinity values will be replaced with a very + large number. + + .. versionadded:: 1.17 + neginf : int, float, optional + Value to be used to fill negative infinity values. If no value is + passed then negative infinity values will be replaced with a very + small (or negative) number. + + .. versionadded:: 1.17 + + Returns ------- @@ -424,6 +446,9 @@ def nan_to_num(x, copy=True): >>> np.nan_to_num(x) array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary -1.28000000e+002, 1.28000000e+002]) + >>> np.nan_to_num(x, nan=-9999, posinf=33333333, neginf=33333333) + array([ 3.3333333e+07, 3.3333333e+07, -9.9990000e+03, + -1.2800000e+02, 1.2800000e+02]) >>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)]) array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary -1.28000000e+002, 1.28000000e+002]) @@ -431,6 +456,8 @@ def nan_to_num(x, copy=True): array([ 1.79769313e+308 +0.00000000e+000j, # may vary 0.00000000e+000 +0.00000000e+000j, 0.00000000e+000 +1.79769313e+308j]) + >>> np.nan_to_num(y, nan=111111, posinf=222222) + array([222222.+111111.j, 111111. +0.j, 111111.+222222.j]) """ x = _nx.array(x, subok=True, copy=copy) xtype = x.dtype.type @@ -444,10 +471,17 @@ def nan_to_num(x, copy=True): dest = (x.real, x.imag) if iscomplex else (x,) maxf, minf = _getmaxmin(x.real.dtype) + if posinf is not None: + maxf = posinf + if neginf is not None: + minf = neginf for d in dest: - _nx.copyto(d, 0.0, where=isnan(d)) - _nx.copyto(d, maxf, where=isposinf(d)) - _nx.copyto(d, minf, where=isneginf(d)) + idx_nan = isnan(d) + idx_posinf = isposinf(d) + idx_neginf = isneginf(d) + _nx.copyto(d, nan, where=idx_nan) + _nx.copyto(d, maxf, where=idx_posinf) + _nx.copyto(d, minf, where=idx_neginf) return x[()] if isscalar else x #----------------------------------------------------------------------------- @@ -513,6 +547,9 @@ def real_if_close(a, tol=100): def _asscalar_dispatcher(a): + # 2018-10-10, 1.16 + warnings.warn('np.asscalar(a) is deprecated since NumPy v1.16, use ' + 'a.item() instead', DeprecationWarning, stacklevel=3) return (a,) @@ -541,10 +578,6 @@ def asscalar(a): >>> np.asscalar(np.array([24])) 24 """ - - # 2018-10-10, 1.16 - warnings.warn('np.asscalar(a) is deprecated since NumPy v1.16, use ' - 'a.item() instead', DeprecationWarning, stacklevel=1) return a.item() #----------------------------------------------------------------------------- diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index 5c411e8c8..96fd5b319 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -8,7 +8,9 @@ from __future__ import division, absolute_import, print_function __all__ = ['fix', 'isneginf', 'isposinf'] import numpy.core.numeric as nx -from numpy.core.overrides import array_function_dispatch, ENABLE_ARRAY_FUNCTION +from numpy.core.overrides import ( + array_function_dispatch, ARRAY_FUNCTION_ENABLED, +) import warnings import functools @@ -43,7 +45,7 @@ def _fix_out_named_y(f): Allow the out argument to be passed as the name `y` (deprecated) This decorator should only be used if _deprecate_out_named_y is used on - a corresponding dispatcher fucntion. + a corresponding dispatcher function. """ @functools.wraps(f) def func(x, out=None, **kwargs): @@ -55,8 +57,14 @@ def _fix_out_named_y(f): return func -if not ENABLE_ARRAY_FUNCTION: - _fix_out_named_y = _deprecate_out_named_y +def _fix_and_maybe_deprecate_out_named_y(f): + """ + Use the appropriate decorator, depending upon if dispatching is being used. + """ + if ARRAY_FUNCTION_ENABLED: + return _fix_out_named_y(f) + else: + return _deprecate_out_named_y(f) @_deprecate_out_named_y @@ -65,7 +73,7 @@ def _dispatcher(x, out=None): @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_out_named_y +@_fix_and_maybe_deprecate_out_named_y def fix(x, out=None): """ Round to nearest integer towards zero. @@ -112,7 +120,7 @@ def fix(x, out=None): @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_out_named_y +@_fix_and_maybe_deprecate_out_named_y def isposinf(x, out=None): """ Test element-wise for positive infinity, return result as bool array. @@ -181,7 +189,7 @@ def isposinf(x, out=None): @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_out_named_y +@_fix_and_maybe_deprecate_out_named_y def isneginf(x, out=None): """ Test element-wise for negative infinity, return result as bool array. diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 5a4cae235..3c71d2a7c 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -105,6 +105,20 @@ class _Deprecate(object): if doc is None: doc = depdoc else: + lines = doc.expandtabs().split('\n') + indent = _get_indent(lines[1:]) + if lines[0].lstrip(): + # Indent the original first line to let inspect.cleandoc() + # dedent the docstring despite the deprecation notice. + doc = indent * ' ' + doc + else: + # Remove the same leading blank lines as cleandoc() would. + skip = len(lines[0]) + 1 + for line in lines[1:]: + if len(line) > indent: + break + skip += len(line) + 1 + doc = doc[skip:] doc = '\n\n'.join([depdoc, doc]) newfunc.__doc__ = doc try: @@ -115,6 +129,21 @@ class _Deprecate(object): newfunc.__dict__.update(d) return newfunc + +def _get_indent(lines): + """ + Determines the leading whitespace that could be removed from all the lines. + """ + indent = sys.maxsize + for line in lines: + content = len(line.lstrip()) + if content: + indent = min(indent, len(line) - content) + if indent == sys.maxsize: + indent = 0 + return indent + + def deprecate(*args, **kwargs): """ Issues a DeprecationWarning, adds warning to `old_name`'s @@ -759,13 +788,8 @@ def lookfor(what, module=None, import_modules=True, regenerate=False, if kind in ('module', 'object'): # don't show modules or objects continue - ok = True doc = docstring.lower() - for w in whats: - if w not in doc: - ok = False - break - if ok: + if all(w in doc for w in whats): found.append(name) # Relevance sort @@ -974,93 +998,6 @@ def _getmembers(item): if hasattr(item, x)] return members -#----------------------------------------------------------------------------- - -# The following SafeEval class and company are adapted from Michael Spencer's -# ASPN Python Cookbook recipe: https://code.activestate.com/recipes/364469/ -# -# Accordingly it is mostly Copyright 2006 by Michael Spencer. -# The recipe, like most of the other ASPN Python Cookbook recipes was made -# available under the Python license. -# https://en.wikipedia.org/wiki/Python_License - -# It has been modified to: -# * handle unary -/+ -# * support True/False/None -# * raise SyntaxError instead of a custom exception. - -class SafeEval(object): - """ - Object to evaluate constant string expressions. - - This includes strings with lists, dicts and tuples using the abstract - syntax tree created by ``compiler.parse``. - - .. deprecated:: 1.10.0 - - See Also - -------- - safe_eval - - """ - def __init__(self): - # 2014-10-15, 1.10 - warnings.warn("SafeEval is deprecated in 1.10 and will be removed.", - DeprecationWarning, stacklevel=2) - - def visit(self, node): - cls = node.__class__ - meth = getattr(self, 'visit' + cls.__name__, self.default) - return meth(node) - - def default(self, node): - raise SyntaxError("Unsupported source construct: %s" - % node.__class__) - - def visitExpression(self, node): - return self.visit(node.body) - - def visitNum(self, node): - return node.n - - def visitStr(self, node): - return node.s - - def visitBytes(self, node): - return node.s - - def visitDict(self, node,**kw): - return dict([(self.visit(k), self.visit(v)) - for k, v in zip(node.keys, node.values)]) - - def visitTuple(self, node): - return tuple([self.visit(i) for i in node.elts]) - - def visitList(self, node): - return [self.visit(i) for i in node.elts] - - def visitUnaryOp(self, node): - import ast - if isinstance(node.op, ast.UAdd): - return +self.visit(node.operand) - elif isinstance(node.op, ast.USub): - return -self.visit(node.operand) - else: - raise SyntaxError("Unknown unary op: %r" % node.op) - - def visitName(self, node): - if node.id == 'False': - return False - elif node.id == 'True': - return True - elif node.id == 'None': - return None - else: - raise SyntaxError("Unknown name: %s" % node.id) - - def visitNameConstant(self, node): - return node.value - def safe_eval(source): """ @@ -1107,7 +1044,6 @@ def safe_eval(source): """ # Local import to speed up numpy's import time. import ast - return ast.literal_eval(source) @@ -1140,17 +1076,12 @@ def _median_nancheck(data, result, axis, out): n = n.filled(False) if result.ndim == 0: if n == True: - warnings.warn("Invalid value encountered in median", - RuntimeWarning, stacklevel=3) if out is not None: out[...] = data.dtype.type(np.nan) result = out else: result = data.dtype.type(np.nan) elif np.count_nonzero(n.ravel()) > 0: - warnings.warn("Invalid value encountered in median for" + - " %d results" % np.count_nonzero(n.ravel()), - RuntimeWarning, stacklevel=3) result[n] = np.nan return result |