diff options
| author | scoder <stefan_ml@behnel.de> | 2023-05-04 09:29:53 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-04 09:29:53 +0200 |
| commit | 442c8f48d3146ec32c7d5387310e171276cf10ac (patch) | |
| tree | d8911d1a64e384b7955d3fc09a07edd218a9f1ee /numpy/lib | |
| parent | 3e4a6cba2da27bbe2a6e12c163238e503c9f6a07 (diff) | |
| parent | 9163e933df91b516b6f0c7a9ba8ad1750e642f37 (diff) | |
| download | numpy-442c8f48d3146ec32c7d5387310e171276cf10ac.tar.gz | |
Merge branch 'main' into cython3_noexcept
Diffstat (limited to 'numpy/lib')
65 files changed, 10581 insertions, 3611 deletions
diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index cb0de0d15..d3cc9fee4 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -11,7 +11,6 @@ Most contains basic functions that are used by several submodules and are useful to have in the main name-space. """ -import math from numpy.version import version as __version__ @@ -21,6 +20,24 @@ from . import mixins from . import scimath as emath # Private submodules +# load module names. See https://github.com/networkx/networkx/issues/5838 +from . import type_check +from . import index_tricks +from . import function_base +from . import nanfunctions +from . import shape_base +from . import stride_tricks +from . import twodim_base +from . import ufunclike +from . import histograms +from . import polynomial +from . import utils +from . import arraysetops +from . import npyio +from . import arrayterator +from . import arraypad +from . import _version + from .type_check import * from .index_tricks import * from .function_base import * @@ -35,13 +52,12 @@ from .polynomial import * from .utils import * from .arraysetops import * from .npyio import * -from .financial import * from .arrayterator import Arrayterator from .arraypad import * from ._version import * from numpy.core._multiarray_umath import tracemalloc_domain -__all__ = ['emath', 'math', 'tracemalloc_domain', 'Arrayterator'] +__all__ = ['emath', 'tracemalloc_domain', 'Arrayterator'] __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ @@ -54,10 +70,25 @@ __all__ += polynomial.__all__ __all__ += utils.__all__ __all__ += arraysetops.__all__ __all__ += npyio.__all__ -__all__ += financial.__all__ __all__ += nanfunctions.__all__ __all__ += histograms.__all__ from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester + +def __getattr__(attr): + # Warn for reprecated attributes + import math + import warnings + + if attr == 'math': + warnings.warn( + "`np.lib.math` is a deprecated alias for the standard library " + "`math` module (Deprecated Numpy 1.25). Replace usages of " + "`numpy.lib.math` with `math`", DeprecationWarning, stacklevel=2) + return math + else: + raise AttributeError("module {!r} has no attribute " + "{!r}".format(__name__, attr)) + diff --git a/numpy/lib/__init__.pyi b/numpy/lib/__init__.pyi new file mode 100644 index 000000000..d3553bbcc --- /dev/null +++ b/numpy/lib/__init__.pyi @@ -0,0 +1,245 @@ +import math as math +from typing import Any + +from numpy._pytesttester import PytestTester + +from numpy import ( + ndenumerate as ndenumerate, + ndindex as ndindex, +) + +from numpy.version import version + +from numpy.lib import ( + format as format, + mixins as mixins, + scimath as scimath, + stride_tricks as stride_tricks, +) + +from numpy.lib._version import ( + NumpyVersion as NumpyVersion, +) + +from numpy.lib.arraypad import ( + pad as pad, +) + +from numpy.lib.arraysetops import ( + ediff1d as ediff1d, + intersect1d as intersect1d, + setxor1d as setxor1d, + union1d as union1d, + setdiff1d as setdiff1d, + unique as unique, + in1d as in1d, + isin as isin, +) + +from numpy.lib.arrayterator import ( + Arrayterator as Arrayterator, +) + +from numpy.lib.function_base import ( + select as select, + piecewise as piecewise, + trim_zeros as trim_zeros, + copy as copy, + iterable as iterable, + percentile as percentile, + diff as diff, + gradient as gradient, + angle as angle, + unwrap as unwrap, + sort_complex as sort_complex, + disp as disp, + flip as flip, + rot90 as rot90, + extract as extract, + place as place, + vectorize as vectorize, + asarray_chkfinite as asarray_chkfinite, + average as average, + bincount as bincount, + digitize as digitize, + cov as cov, + corrcoef as corrcoef, + median as median, + sinc as sinc, + hamming as hamming, + hanning as hanning, + bartlett as bartlett, + blackman as blackman, + kaiser as kaiser, + trapz as trapz, + i0 as i0, + add_newdoc as add_newdoc, + add_docstring as add_docstring, + meshgrid as meshgrid, + delete as delete, + insert as insert, + append as append, + interp as interp, + add_newdoc_ufunc as add_newdoc_ufunc, + quantile as quantile, +) + +from numpy.lib.histograms import ( + histogram_bin_edges as histogram_bin_edges, + histogram as histogram, + histogramdd as histogramdd, +) + +from numpy.lib.index_tricks import ( + ravel_multi_index as ravel_multi_index, + unravel_index as unravel_index, + mgrid as mgrid, + ogrid as ogrid, + r_ as r_, + c_ as c_, + s_ as s_, + index_exp as index_exp, + ix_ as ix_, + fill_diagonal as fill_diagonal, + diag_indices as diag_indices, + diag_indices_from as diag_indices_from, +) + +from numpy.lib.nanfunctions import ( + nansum as nansum, + nanmax as nanmax, + nanmin as nanmin, + nanargmax as nanargmax, + nanargmin as nanargmin, + nanmean as nanmean, + nanmedian as nanmedian, + nanpercentile as nanpercentile, + nanvar as nanvar, + nanstd as nanstd, + nanprod as nanprod, + nancumsum as nancumsum, + nancumprod as nancumprod, + nanquantile as nanquantile, +) + +from numpy.lib.npyio import ( + savetxt as savetxt, + loadtxt as loadtxt, + genfromtxt as genfromtxt, + recfromtxt as recfromtxt, + recfromcsv as recfromcsv, + load as load, + save as save, + savez as savez, + savez_compressed as savez_compressed, + packbits as packbits, + unpackbits as unpackbits, + fromregex as fromregex, + DataSource as DataSource, +) + +from numpy.lib.polynomial import ( + poly as poly, + roots as roots, + polyint as polyint, + polyder as polyder, + polyadd as polyadd, + polysub as polysub, + polymul as polymul, + polydiv as polydiv, + polyval as polyval, + polyfit as polyfit, + RankWarning as RankWarning, + poly1d as poly1d, +) + +from numpy.lib.shape_base import ( + column_stack as column_stack, + row_stack as row_stack, + dstack as dstack, + array_split as array_split, + split as split, + hsplit as hsplit, + vsplit as vsplit, + dsplit as dsplit, + apply_over_axes as apply_over_axes, + expand_dims as expand_dims, + apply_along_axis as apply_along_axis, + kron as kron, + tile as tile, + get_array_wrap as get_array_wrap, + take_along_axis as take_along_axis, + put_along_axis as put_along_axis, +) + +from numpy.lib.stride_tricks import ( + broadcast_to as broadcast_to, + broadcast_arrays as broadcast_arrays, + broadcast_shapes as broadcast_shapes, +) + +from numpy.lib.twodim_base import ( + diag as diag, + diagflat as diagflat, + eye as eye, + fliplr as fliplr, + flipud as flipud, + tri as tri, + triu as triu, + tril as tril, + vander as vander, + histogram2d as histogram2d, + mask_indices as mask_indices, + tril_indices as tril_indices, + tril_indices_from as tril_indices_from, + triu_indices as triu_indices, + triu_indices_from as triu_indices_from, +) + +from numpy.lib.type_check import ( + mintypecode as mintypecode, + asfarray as asfarray, + real as real, + imag as imag, + iscomplex as iscomplex, + isreal as isreal, + iscomplexobj as iscomplexobj, + isrealobj as isrealobj, + nan_to_num as nan_to_num, + real_if_close as real_if_close, + typename as typename, + common_type as common_type, +) + +from numpy.lib.ufunclike import ( + fix as fix, + isposinf as isposinf, + isneginf as isneginf, +) + +from numpy.lib.utils import ( + issubclass_ as issubclass_, + issubsctype as issubsctype, + issubdtype as issubdtype, + deprecate as deprecate, + deprecate_with_doc as deprecate_with_doc, + get_include as get_include, + info as info, + source as source, + who as who, + lookfor as lookfor, + byte_bounds as byte_bounds, + safe_eval as safe_eval, + show_runtime as show_runtime, +) + +from numpy.core.multiarray import ( + tracemalloc_domain as tracemalloc_domain, +) + +__all__: list[str] +__path__: list[str] +test: PytestTester + +__version__ = version +emath = scimath diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index 7a23b1651..613733fa5 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -35,10 +35,9 @@ Example:: """ import os -import shutil import io -from numpy.core.overrides import set_module +from .._utils import set_module _open = open @@ -257,6 +256,8 @@ class DataSource: def __del__(self): # Remove temp directories if hasattr(self, '_istmpdest') and self._istmpdest: + import shutil + shutil.rmtree(self._destpath) def _iszip(self, filename): @@ -279,8 +280,9 @@ class DataSource: def _splitzipext(self, filename): """Split zip extension from filename and return filename. - *Returns*: - base, zip_ext : {tuple} + Returns + ------- + base, zip_ext : {tuple} """ @@ -319,10 +321,10 @@ class DataSource: Creates a copy of the file in the datasource cache. """ - # We import these here because importing urllib is slow and + # We import these here because importing them is slow and # a significant fraction of numpy's total import time. + import shutil from urllib.request import urlopen - from urllib.error import URLError upath = self.abspath(path) @@ -528,7 +530,7 @@ class DataSource: return _file_openers[ext](found, mode=mode, encoding=encoding, newline=newline) else: - raise IOError("%s not found." % path) + raise FileNotFoundError(f"{path} not found.") class Repository (DataSource): diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 7560bf4da..534d1b3ee 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -5,7 +5,7 @@ __docformat__ = "restructuredtext en" import numpy as np import numpy.core.numeric as nx -from numpy.compat import asbytes, asunicode, bytes +from numpy.compat import asbytes, asunicode def _decode_line(line, encoding=None): @@ -18,18 +18,18 @@ def _decode_line(line, encoding=None): ---------- line : str or bytes Line to be decoded. + encoding : str + Encoding used to decode `line`. Returns ------- - decoded_line : unicode - Unicode in Python 2, a str (unicode) in Python 3. + decoded_line : str """ if type(line) is bytes: if encoding is None: - line = line.decode('latin1') - else: - line = line.decode(encoding) + encoding = "latin1" + line = line.decode(encoding) return line @@ -513,8 +513,8 @@ class StringConverter: (nx.complexfloating, complex, nx.nan + 0j), # Last, try with the string types (must be last, because # `_mapper[-1]` is used as default in some cases) - (nx.unicode_, asunicode, '???'), - (nx.string_, asbytes, '???'), + (nx.str_, asunicode, '???'), + (nx.bytes_, asbytes, '???'), ]) @classmethod diff --git a/numpy/lib/_version.py b/numpy/lib/_version.py index d4098acb5..bfac5f814 100644 --- a/numpy/lib/_version.py +++ b/numpy/lib/_version.py @@ -15,7 +15,7 @@ class NumpyVersion(): """Parse and compare numpy version strings. NumPy has the following versioning scheme (numbers given are examples; they - can be > 9) in principle): + can be > 9 in principle): - Released version: '1.8.0', '1.8.1', etc. - Alpha: '1.8.0a1', '1.8.0a2', etc. @@ -54,7 +54,7 @@ class NumpyVersion(): def __init__(self, vstring): self.vstring = vstring - ver_main = re.match(r'\d[.]\d+[.]\d+', vstring) + ver_main = re.match(r'\d+\.\d+\.\d+', vstring) if not ver_main: raise ValueError("Not a valid numpy version string") @@ -151,5 +151,5 @@ class NumpyVersion(): def __ge__(self, other): return self._compare(other) >= 0 - def __repr(self): + def __repr__(self): return "NumpyVersion(%s)" % self.vstring diff --git a/numpy/lib/_version.pyi b/numpy/lib/_version.pyi new file mode 100644 index 000000000..1c82c99b6 --- /dev/null +++ b/numpy/lib/_version.pyi @@ -0,0 +1,17 @@ +__all__: list[str] + +class NumpyVersion: + vstring: str + version: str + major: int + minor: int + bugfix: int + pre_release: str + is_devversion: bool + def __init__(self, vstring: str) -> None: ... + def __lt__(self, other: str | NumpyVersion) -> bool: ... + def __le__(self, other: str | NumpyVersion) -> bool: ... + def __eq__(self, other: str | NumpyVersion) -> bool: ... # type: ignore[override] + def __ne__(self, other: str | NumpyVersion) -> bool: ... # type: ignore[override] + def __gt__(self, other: str | NumpyVersion) -> bool: ... + def __ge__(self, other: str | NumpyVersion) -> bool: ... diff --git a/numpy/lib/arraypad.py b/numpy/lib/arraypad.py index 8830b8147..b06a645d8 100644 --- a/numpy/lib/arraypad.py +++ b/numpy/lib/arraypad.py @@ -378,7 +378,7 @@ def _set_reflect_both(padded, axis, width_pair, method, include_edge=False): return left_pad, right_pad -def _set_wrap_both(padded, axis, width_pair): +def _set_wrap_both(padded, axis, width_pair, original_period): """ Pad `axis` of `arr` with wrapped values. @@ -391,6 +391,8 @@ def _set_wrap_both(padded, axis, width_pair): width_pair : (int, int) Pair of widths that mark the pad area on both sides in the given dimension. + original_period : int + Original length of data on `axis` of `arr`. Returns ------- @@ -400,6 +402,9 @@ def _set_wrap_both(padded, axis, width_pair): """ left_pad, right_pad = width_pair period = padded.shape[axis] - right_pad - left_pad + # Avoid wrapping with only a subset of the original area by ensuring period + # can only be a multiple of the original area's length. + period = period // original_period * original_period # If the current dimension of `arr` doesn't contain enough valid values # (not part of the undefined pad area) we need to pad multiple times. @@ -410,14 +415,12 @@ def _set_wrap_both(padded, axis, width_pair): if left_pad > 0: # Pad with wrapped values on left side - # First slice chunk from right side of the non-pad area. + # First slice chunk from left side of the non-pad area. # Use min(period, left_pad) to ensure that chunk is not larger than - # pad area - right_slice = _slice_at_axis( - slice(-right_pad - min(period, left_pad), - -right_pad if right_pad != 0 else None), - axis - ) + # pad area. + slice_end = left_pad + period + slice_start = slice_end - min(period, left_pad) + right_slice = _slice_at_axis(slice(slice_start, slice_end), axis) right_chunk = padded[right_slice] if left_pad > period: @@ -431,11 +434,12 @@ def _set_wrap_both(padded, axis, width_pair): if right_pad > 0: # Pad with wrapped values on right side - # First slice chunk from left side of the non-pad area. + # First slice chunk from right side of the non-pad area. # Use min(period, right_pad) to ensure that chunk is not larger than - # pad area - left_slice = _slice_at_axis( - slice(left_pad, left_pad + min(period, right_pad),), axis) + # pad area. + slice_start = -right_pad - period + slice_end = slice_start + min(period, right_pad) + left_slice = _slice_at_axis(slice(slice_start, slice_end), axis) left_chunk = padded[left_slice] if right_pad > period: @@ -537,11 +541,12 @@ def pad(array, pad_width, mode='constant', **kwargs): The array to pad. pad_width : {sequence, array_like, int} Number of values padded to the edges of each axis. - ((before_1, after_1), ... (before_N, after_N)) unique pad widths + ``((before_1, after_1), ... (before_N, after_N))`` unique pad widths for each axis. - ((before, after),) yields same before and after pad for each axis. - (pad,) or int is a shortcut for before = after = pad width for all - axes. + ``(before, after)`` or ``((before, after),)`` yields same before + and after pad for each axis. + ``(pad,)`` or ``int`` is a shortcut for before = after = pad width + for all axes. mode : str or function, optional One of the following string values or a user supplied function. @@ -586,14 +591,14 @@ def pad(array, pad_width, mode='constant', **kwargs): Used in 'maximum', 'mean', 'median', and 'minimum'. Number of values at edge of each axis used to calculate the statistic value. - ((before_1, after_1), ... (before_N, after_N)) unique statistic + ``((before_1, after_1), ... (before_N, after_N))`` unique statistic lengths for each axis. - ((before, after),) yields same before and after statistic lengths - for each axis. + ``(before, after)`` or ``((before, after),)`` yields same before + and after statistic lengths for each axis. - (stat_length,) or int is a shortcut for before = after = statistic - length for all axes. + ``(stat_length,)`` or ``int`` is a shortcut for + ``before = after = statistic`` length for all axes. Default is ``None``, to use the entire axis. constant_values : sequence or scalar, optional @@ -603,11 +608,11 @@ def pad(array, pad_width, mode='constant', **kwargs): ``((before_1, after_1), ... (before_N, after_N))`` unique pad constants for each axis. - ``((before, after),)`` yields same before and after constants for each - axis. + ``(before, after)`` or ``((before, after),)`` yields same before + and after constants for each axis. - ``(constant,)`` or ``constant`` is a shortcut for ``before = after = constant`` for - all axes. + ``(constant,)`` or ``constant`` is a shortcut for + ``before = after = constant`` for all axes. Default is 0. end_values : sequence or scalar, optional @@ -617,11 +622,11 @@ def pad(array, pad_width, mode='constant', **kwargs): ``((before_1, after_1), ... (before_N, after_N))`` unique end values for each axis. - ``((before, after),)`` yields same before and after end values for each - axis. + ``(before, after)`` or ``((before, after),)`` yields same before + and after end values for each axis. - ``(constant,)`` or ``constant`` is a shortcut for ``before = after = constant`` for - all axes. + ``(constant,)`` or ``constant`` is a shortcut for + ``before = after = constant`` for all axes. Default is 0. reflect_type : {'even', 'odd'}, optional @@ -866,11 +871,12 @@ def pad(array, pad_width, mode='constant', **kwargs): elif mode == "wrap": for axis, (left_index, right_index) in zip(axes, pad_width): roi = _view_roi(padded, original_area_slice, axis) + original_period = padded.shape[axis] - right_index - left_index while left_index > 0 or right_index > 0: # Iteratively pad until dimension is filled with wrapped # values. This is necessary if the pad area is larger than # the length of the original values in the current dimension. left_index, right_index = _set_wrap_both( - roi, axis, (left_index, right_index)) + roi, axis, (left_index, right_index), original_period) return padded diff --git a/numpy/lib/arraypad.pyi b/numpy/lib/arraypad.pyi new file mode 100644 index 000000000..1ac6fc7d9 --- /dev/null +++ b/numpy/lib/arraypad.pyi @@ -0,0 +1,85 @@ +from typing import ( + Literal as L, + Any, + overload, + TypeVar, + Protocol, +) + +from numpy import generic + +from numpy._typing import ( + ArrayLike, + NDArray, + _ArrayLikeInt, + _ArrayLike, +) + +_SCT = TypeVar("_SCT", bound=generic) + +class _ModeFunc(Protocol): + def __call__( + self, + vector: NDArray[Any], + iaxis_pad_width: tuple[int, int], + iaxis: int, + kwargs: dict[str, Any], + /, + ) -> None: ... + +_ModeKind = L[ + "constant", + "edge", + "linear_ramp", + "maximum", + "mean", + "median", + "minimum", + "reflect", + "symmetric", + "wrap", + "empty", +] + +__all__: list[str] + +# TODO: In practice each keyword argument is exclusive to one or more +# specific modes. Consider adding more overloads to express this in the future. + +# Expand `**kwargs` into explicit keyword-only arguments +@overload +def pad( + array: _ArrayLike[_SCT], + pad_width: _ArrayLikeInt, + mode: _ModeKind = ..., + *, + stat_length: None | _ArrayLikeInt = ..., + constant_values: ArrayLike = ..., + end_values: ArrayLike = ..., + reflect_type: L["odd", "even"] = ..., +) -> NDArray[_SCT]: ... +@overload +def pad( + array: ArrayLike, + pad_width: _ArrayLikeInt, + mode: _ModeKind = ..., + *, + stat_length: None | _ArrayLikeInt = ..., + constant_values: ArrayLike = ..., + end_values: ArrayLike = ..., + reflect_type: L["odd", "even"] = ..., +) -> NDArray[Any]: ... +@overload +def pad( + array: _ArrayLike[_SCT], + pad_width: _ArrayLikeInt, + mode: _ModeFunc, + **kwargs: Any, +) -> NDArray[_SCT]: ... +@overload +def pad( + array: ArrayLike, + pad_width: _ArrayLikeInt, + mode: _ModeFunc, + **kwargs: Any, +) -> NDArray[Any]: ... diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 6cca1738b..300bbda26 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -1,28 +1,17 @@ """ Set operations for arrays based on sorting. -:Contains: - unique, - isin, - ediff1d, - intersect1d, - setxor1d, - in1d, - union1d, - setdiff1d - -:Notes: +Notes +----- For floating point arrays, inaccurate results may appear due to usual round-off and floating point comparison issues. Speed could be gained in some operations by an implementation of -sort(), that can provide directly the permutation vectors, avoiding -thus calls to argsort(). +`numpy.sort`, that can provide directly the permutation vectors, thus avoiding +calls to `numpy.argsort`. -To do: Optionally return indices analogously to unique for all functions. - -:Author: Robert Cimrman +Original author: Robert Cimrman """ import functools @@ -104,7 +93,7 @@ def ediff1d(ary, to_end=None, to_begin=None): else: to_begin = np.asanyarray(to_begin) if not np.can_cast(to_begin, dtype_req, casting="same_kind"): - raise TypeError("dtype of `to_end` must be compatible " + raise TypeError("dtype of `to_begin` must be compatible " "with input `ary` under the `same_kind` rule.") to_begin = to_begin.ravel() @@ -142,13 +131,13 @@ def _unpack_tuple(x): def _unique_dispatcher(ar, return_index=None, return_inverse=None, - return_counts=None, axis=None): + return_counts=None, axis=None, *, equal_nan=None): return (ar,) @array_function_dispatch(_unique_dispatcher) def unique(ar, return_index=False, return_inverse=False, - return_counts=False, axis=None): + return_counts=False, axis=None, *, equal_nan=True): """ Find the unique elements of an array. @@ -173,9 +162,6 @@ def unique(ar, return_index=False, return_inverse=False, return_counts : bool, optional If True, also return the number of times each unique item appears in `ar`. - - .. versionadded:: 1.9.0 - axis : int or None, optional The axis to operate on. If None, `ar` will be flattened. If an integer, the subarrays indexed by the given axis will be flattened and treated @@ -186,6 +172,11 @@ def unique(ar, return_index=False, return_inverse=False, .. versionadded:: 1.13.0 + equal_nan : bool, optional + If True, collapses multiple NaN values in the return array into one. + + .. versionadded:: 1.24 + Returns ------- unique : ndarray @@ -220,6 +211,16 @@ def unique(ar, return_index=False, return_inverse=False, flattened subarrays are sorted in lexicographic order starting with the first element. + .. versionchanged: NumPy 1.21 + If nan values are in the input array, a single nan is put + to the end of the sorted unique values. + + Also for complex arrays all NaN values are considered equivalent + (no matter whether the NaN is in the real or imaginary part). + As the representant for the returned array the smallest one in the + lexicographical order is chosen - see np.sort for how the lexicographical + order is defined for complex arrays. + Examples -------- >>> np.unique([1, 1, 2, 2, 3, 3]) @@ -270,7 +271,8 @@ def unique(ar, return_index=False, return_inverse=False, """ ar = np.asanyarray(ar) if axis is None: - ret = _unique1d(ar, return_index, return_inverse, return_counts) + ret = _unique1d(ar, return_index, return_inverse, return_counts, + equal_nan=equal_nan) return _unpack_tuple(ret) # axis was specified and not None @@ -278,7 +280,7 @@ def unique(ar, return_index=False, return_inverse=False, ar = np.moveaxis(ar, axis, 0) except np.AxisError: # this removes the "axis1" or "axis2" prefix from the error message - raise np.AxisError(axis, ar.ndim) + raise np.AxisError(axis, ar.ndim) from None # Must reshape to a contiguous 2D array for this to work... orig_shape, orig_dtype = ar.shape, ar.dtype @@ -300,10 +302,10 @@ def unique(ar, return_index=False, return_inverse=False, # array with shape `(len(ar),)`. Because `dtype` in this case has # itemsize 0, the total size of the result is still 0 bytes. consolidated = np.empty(len(ar), dtype=dtype) - except TypeError: + except TypeError as e: # There's no good way to do this for object arrays, etc... msg = 'The axis argument to unique is not supported for dtype {dt}' - raise TypeError(msg.format(dt=ar.dtype)) + raise TypeError(msg.format(dt=ar.dtype)) from e def reshape_uniq(uniq): n = len(uniq) @@ -313,13 +315,13 @@ def unique(ar, return_index=False, return_inverse=False, return uniq output = _unique1d(consolidated, return_index, - return_inverse, return_counts) + return_inverse, return_counts, equal_nan=equal_nan) output = (reshape_uniq(output[0]),) + output[1:] return _unpack_tuple(output) def _unique1d(ar, return_index=False, return_inverse=False, - return_counts=False): + return_counts=False, *, equal_nan=True): """ Find the unique elements of an array, ignoring shape. """ @@ -335,7 +337,19 @@ def _unique1d(ar, return_index=False, return_inverse=False, aux = ar mask = np.empty(aux.shape, dtype=np.bool_) mask[:1] = True - mask[1:] = aux[1:] != aux[:-1] + if (equal_nan and aux.shape[0] > 0 and aux.dtype.kind in "cfmM" and + np.isnan(aux[-1])): + if aux.dtype.kind == "c": # for complex all NaNs are considered equivalent + aux_firstnan = np.searchsorted(np.isnan(aux), True, side='left') + else: + aux_firstnan = np.searchsorted(aux, aux[-1], side='left') + if aux_firstnan > 0: + mask[1:aux_firstnan] = ( + aux[1:aux_firstnan] != aux[:aux_firstnan - 1]) + mask[aux_firstnan] = True + mask[aux_firstnan + 1:] = False + else: + mask[1:] = aux[1:] != aux[:-1] ret = (aux[mask],) if return_index: @@ -369,7 +383,9 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): Input arrays. Will be flattened if not already 1D. assume_unique : bool If True, the input arrays are both assumed to be unique, which - can speed up the calculation. Default is False. + can speed up the calculation. If True but ``ar1`` or ``ar2`` are not + unique, incorrect results and out-of-bounds indices could result. + Default is False. return_indices : bool If True, the indices which correspond to the intersection of the two arrays are returned. The first instance of a value is used if there are @@ -500,12 +516,13 @@ def setxor1d(ar1, ar2, assume_unique=False): return aux[flag[1:] & flag[:-1]] -def _in1d_dispatcher(ar1, ar2, assume_unique=None, invert=None): +def _in1d_dispatcher(ar1, ar2, assume_unique=None, invert=None, *, + kind=None): return (ar1, ar2) @array_function_dispatch(_in1d_dispatcher) -def in1d(ar1, ar2, assume_unique=False, invert=False): +def in1d(ar1, ar2, assume_unique=False, invert=False, *, kind=None): """ Test whether each element of a 1-D array is also present in a second array. @@ -528,6 +545,26 @@ def in1d(ar1, ar2, assume_unique=False, invert=False): False where an element of `ar1` is in `ar2` and True otherwise). Default is False. ``np.in1d(a, b, invert=True)`` is equivalent to (but is faster than) ``np.invert(in1d(a, b))``. + kind : {None, 'sort', 'table'}, optional + The algorithm to use. This will not affect the final result, + but will affect the speed and memory use. The default, None, + will select automatically based on memory considerations. + + * If 'sort', will use a mergesort-based approach. This will have + a memory usage of roughly 6 times the sum of the sizes of + `ar1` and `ar2`, not accounting for size of dtypes. + * If 'table', will use a lookup table approach similar + to a counting sort. This is only available for boolean and + integer arrays. This will have a memory usage of the + size of `ar1` plus the max-min value of `ar2`. `assume_unique` + has no effect when the 'table' option is used. + * If None, will automatically choose 'table' if + the required memory allocation is less than or equal to + 6 times the sum of the sizes of `ar1` and `ar2`, + otherwise will use 'sort'. This is done to not use + a large amount of memory by default, even though + 'table' may be faster in most cases. If 'table' is chosen, + `assume_unique` will have no effect. .. versionadded:: 1.8.0 @@ -553,6 +590,13 @@ def in1d(ar1, ar2, assume_unique=False, invert=False): ``asarray(ar2)`` is an object array rather than the expected array of contained values. + Using ``kind='table'`` tends to be faster than `kind='sort'` if the + following relationship is true: + ``log10(len(ar2)) > (log10(max(ar2)-min(ar2)) - 2.27) / 0.927``, + but may use greater memory. The default value for `kind` will + be automatically selected based only on memory usage, so one may + manually set ``kind='table'`` if memory constraints can be relaxed. + .. versionadded:: 1.4.0 Examples @@ -574,6 +618,103 @@ def in1d(ar1, ar2, assume_unique=False, invert=False): ar1 = np.asarray(ar1).ravel() ar2 = np.asarray(ar2).ravel() + # Ensure that iteration through object arrays yields size-1 arrays + if ar2.dtype == object: + ar2 = ar2.reshape(-1, 1) + + if kind not in {None, 'sort', 'table'}: + raise ValueError( + f"Invalid kind: '{kind}'. Please use None, 'sort' or 'table'.") + + # Can use the table method if all arrays are integers or boolean: + is_int_arrays = all(ar.dtype.kind in ("u", "i", "b") for ar in (ar1, ar2)) + use_table_method = is_int_arrays and kind in {None, 'table'} + + if use_table_method: + if ar2.size == 0: + if invert: + return np.ones_like(ar1, dtype=bool) + else: + return np.zeros_like(ar1, dtype=bool) + + # Convert booleans to uint8 so we can use the fast integer algorithm + if ar1.dtype == bool: + ar1 = ar1.astype(np.uint8) + if ar2.dtype == bool: + ar2 = ar2.astype(np.uint8) + + ar2_min = np.min(ar2) + ar2_max = np.max(ar2) + + ar2_range = int(ar2_max) - int(ar2_min) + + # Constraints on whether we can actually use the table method: + # 1. Assert memory usage is not too large + below_memory_constraint = ar2_range <= 6 * (ar1.size + ar2.size) + # 2. Check overflows for (ar2 - ar2_min); dtype=ar2.dtype + range_safe_from_overflow = ar2_range <= np.iinfo(ar2.dtype).max + # 3. Check overflows for (ar1 - ar2_min); dtype=ar1.dtype + if ar1.size > 0: + ar1_min = np.min(ar1) + ar1_max = np.max(ar1) + + # After masking, the range of ar1 is guaranteed to be + # within the range of ar2: + ar1_upper = min(int(ar1_max), int(ar2_max)) + ar1_lower = max(int(ar1_min), int(ar2_min)) + + range_safe_from_overflow &= all(( + ar1_upper - int(ar2_min) <= np.iinfo(ar1.dtype).max, + ar1_lower - int(ar2_min) >= np.iinfo(ar1.dtype).min + )) + + # Optimal performance is for approximately + # log10(size) > (log10(range) - 2.27) / 0.927. + # However, here we set the requirement that by default + # the intermediate array can only be 6x + # the combined memory allocation of the original + # arrays. See discussion on + # https://github.com/numpy/numpy/pull/12065. + + if ( + range_safe_from_overflow and + (below_memory_constraint or kind == 'table') + ): + + if invert: + outgoing_array = np.ones_like(ar1, dtype=bool) + else: + outgoing_array = np.zeros_like(ar1, dtype=bool) + + # Make elements 1 where the integer exists in ar2 + if invert: + isin_helper_ar = np.ones(ar2_range + 1, dtype=bool) + isin_helper_ar[ar2 - ar2_min] = 0 + else: + isin_helper_ar = np.zeros(ar2_range + 1, dtype=bool) + isin_helper_ar[ar2 - ar2_min] = 1 + + # Mask out elements we know won't work + basic_mask = (ar1 <= ar2_max) & (ar1 >= ar2_min) + outgoing_array[basic_mask] = isin_helper_ar[ar1[basic_mask] - + ar2_min] + + return outgoing_array + elif kind == 'table': # not range_safe_from_overflow + raise RuntimeError( + "You have specified kind='table', " + "but the range of values in `ar2` or `ar1` exceed the " + "maximum integer of the datatype. " + "Please set `kind` to None or 'sort'." + ) + elif kind == 'table': + raise ValueError( + "The 'table' method is only " + "supported for boolean or integer arrays. " + "Please select 'sort' or None for kind." + ) + + # Check if one of the arrays may contain arbitrary objects contains_object = ar1.dtype.hasobject or ar2.dtype.hasobject @@ -617,14 +758,16 @@ def in1d(ar1, ar2, assume_unique=False, invert=False): return ret[rev_idx] -def _isin_dispatcher(element, test_elements, assume_unique=None, invert=None): +def _isin_dispatcher(element, test_elements, assume_unique=None, invert=None, + *, kind=None): return (element, test_elements) @array_function_dispatch(_isin_dispatcher) -def isin(element, test_elements, assume_unique=False, invert=False): +def isin(element, test_elements, assume_unique=False, invert=False, *, + kind=None): """ - Calculates `element in test_elements`, broadcasting over `element` only. + Calculates ``element in test_elements``, broadcasting over `element` only. Returns a boolean array of the same shape as `element` that is True where an element of `element` is in `test_elements` and False otherwise. @@ -644,6 +787,27 @@ def isin(element, test_elements, assume_unique=False, invert=False): calculating `element not in test_elements`. Default is False. ``np.isin(a, b, invert=True)`` is equivalent to (but faster than) ``np.invert(np.isin(a, b))``. + kind : {None, 'sort', 'table'}, optional + The algorithm to use. This will not affect the final result, + but will affect the speed and memory use. The default, None, + will select automatically based on memory considerations. + + * If 'sort', will use a mergesort-based approach. This will have + a memory usage of roughly 6 times the sum of the sizes of + `ar1` and `ar2`, not accounting for size of dtypes. + * If 'table', will use a lookup table approach similar + to a counting sort. This is only available for boolean and + integer arrays. This will have a memory usage of the + size of `ar1` plus the max-min value of `ar2`. `assume_unique` + has no effect when the 'table' option is used. + * If None, will automatically choose 'table' if + the required memory allocation is less than or equal to + 6 times the sum of the sizes of `ar1` and `ar2`, + otherwise will use 'sort'. This is done to not use + a large amount of memory by default, even though + 'table' may be faster in most cases. If 'table' is chosen, + `assume_unique` will have no effect. + Returns ------- @@ -671,6 +835,13 @@ def isin(element, test_elements, assume_unique=False, invert=False): of the `array` constructor's way of handling non-sequence collections. Converting the set to a list usually gives the desired behavior. + Using ``kind='table'`` tends to be faster than `kind='sort'` if the + following relationship is true: + ``log10(len(ar2)) > (log10(max(ar2)-min(ar2)) - 2.27) / 0.927``, + but may use greater memory. The default value for `kind` will + be automatically selected based only on memory usage, so one may + manually set ``kind='table'`` if memory constraints can be relaxed. + .. versionadded:: 1.13.0 Examples @@ -717,7 +888,7 @@ def isin(element, test_elements, assume_unique=False, invert=False): """ element = np.asarray(element) return in1d(element, test_elements, assume_unique=assume_unique, - invert=invert).reshape(element.shape) + invert=invert, kind=kind).reshape(element.shape) def _union1d_dispatcher(ar1, ar2): diff --git a/numpy/lib/arraysetops.pyi b/numpy/lib/arraysetops.pyi new file mode 100644 index 000000000..aa1310a32 --- /dev/null +++ b/numpy/lib/arraysetops.pyi @@ -0,0 +1,360 @@ +from typing import ( + Literal as L, + Any, + TypeVar, + overload, + SupportsIndex, +) + +from numpy import ( + generic, + number, + bool_, + ushort, + ubyte, + uintc, + uint, + ulonglong, + short, + int8, + byte, + intc, + int_, + intp, + longlong, + half, + single, + double, + longdouble, + csingle, + cdouble, + clongdouble, + timedelta64, + datetime64, + object_, + str_, + bytes_, + void, +) + +from numpy._typing import ( + ArrayLike, + NDArray, + _ArrayLike, + _ArrayLikeBool_co, + _ArrayLikeDT64_co, + _ArrayLikeTD64_co, + _ArrayLikeObject_co, + _ArrayLikeNumber_co, +) + +_SCT = TypeVar("_SCT", bound=generic) +_NumberType = TypeVar("_NumberType", bound=number[Any]) + +# Explicitly set all allowed values to prevent accidental castings to +# abstract dtypes (their common super-type). +# +# Only relevant if two or more arguments are parametrized, (e.g. `setdiff1d`) +# which could result in, for example, `int64` and `float64`producing a +# `number[_64Bit]` array +_SCTNoCast = TypeVar( + "_SCTNoCast", + bool_, + ushort, + ubyte, + uintc, + uint, + ulonglong, + short, + byte, + intc, + int_, + longlong, + half, + single, + double, + longdouble, + csingle, + cdouble, + clongdouble, + timedelta64, + datetime64, + object_, + str_, + bytes_, + void, +) + +__all__: list[str] + +@overload +def ediff1d( + ary: _ArrayLikeBool_co, + to_end: None | ArrayLike = ..., + to_begin: None | ArrayLike = ..., +) -> NDArray[int8]: ... +@overload +def ediff1d( + ary: _ArrayLike[_NumberType], + to_end: None | ArrayLike = ..., + to_begin: None | ArrayLike = ..., +) -> NDArray[_NumberType]: ... +@overload +def ediff1d( + ary: _ArrayLikeNumber_co, + to_end: None | ArrayLike = ..., + to_begin: None | ArrayLike = ..., +) -> NDArray[Any]: ... +@overload +def ediff1d( + ary: _ArrayLikeDT64_co | _ArrayLikeTD64_co, + to_end: None | ArrayLike = ..., + to_begin: None | ArrayLike = ..., +) -> NDArray[timedelta64]: ... +@overload +def ediff1d( + ary: _ArrayLikeObject_co, + to_end: None | ArrayLike = ..., + to_begin: None | ArrayLike = ..., +) -> NDArray[object_]: ... + +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[False] = ..., + return_inverse: L[False] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[False] = ..., + return_inverse: L[False] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> NDArray[Any]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[True] = ..., + return_inverse: L[False] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[True] = ..., + return_inverse: L[False] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[False] = ..., + return_inverse: L[True] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[False] = ..., + return_inverse: L[True] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[False] = ..., + return_inverse: L[False] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[False] = ..., + return_inverse: L[False] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[True] = ..., + return_inverse: L[True] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[True] = ..., + return_inverse: L[True] = ..., + return_counts: L[False] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[True] = ..., + return_inverse: L[False] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[True] = ..., + return_inverse: L[False] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[False] = ..., + return_inverse: L[True] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[False] = ..., + return_inverse: L[True] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: _ArrayLike[_SCT], + return_index: L[True] = ..., + return_inverse: L[True] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[_SCT], NDArray[intp], NDArray[intp], NDArray[intp]]: ... +@overload +def unique( + ar: ArrayLike, + return_index: L[True] = ..., + return_inverse: L[True] = ..., + return_counts: L[True] = ..., + axis: None | SupportsIndex = ..., + *, + equal_nan: bool = ..., +) -> tuple[NDArray[Any], NDArray[intp], NDArray[intp], NDArray[intp]]: ... + +@overload +def intersect1d( + ar1: _ArrayLike[_SCTNoCast], + ar2: _ArrayLike[_SCTNoCast], + assume_unique: bool = ..., + return_indices: L[False] = ..., +) -> NDArray[_SCTNoCast]: ... +@overload +def intersect1d( + ar1: ArrayLike, + ar2: ArrayLike, + assume_unique: bool = ..., + return_indices: L[False] = ..., +) -> NDArray[Any]: ... +@overload +def intersect1d( + ar1: _ArrayLike[_SCTNoCast], + ar2: _ArrayLike[_SCTNoCast], + assume_unique: bool = ..., + return_indices: L[True] = ..., +) -> tuple[NDArray[_SCTNoCast], NDArray[intp], NDArray[intp]]: ... +@overload +def intersect1d( + ar1: ArrayLike, + ar2: ArrayLike, + assume_unique: bool = ..., + return_indices: L[True] = ..., +) -> tuple[NDArray[Any], NDArray[intp], NDArray[intp]]: ... + +@overload +def setxor1d( + ar1: _ArrayLike[_SCTNoCast], + ar2: _ArrayLike[_SCTNoCast], + assume_unique: bool = ..., +) -> NDArray[_SCTNoCast]: ... +@overload +def setxor1d( + ar1: ArrayLike, + ar2: ArrayLike, + assume_unique: bool = ..., +) -> NDArray[Any]: ... + +def in1d( + ar1: ArrayLike, + ar2: ArrayLike, + assume_unique: bool = ..., + invert: bool = ..., +) -> NDArray[bool_]: ... + +def isin( + element: ArrayLike, + test_elements: ArrayLike, + assume_unique: bool = ..., + invert: bool = ..., +) -> NDArray[bool_]: ... + +@overload +def union1d( + ar1: _ArrayLike[_SCTNoCast], + ar2: _ArrayLike[_SCTNoCast], +) -> NDArray[_SCTNoCast]: ... +@overload +def union1d( + ar1: ArrayLike, + ar2: ArrayLike, +) -> NDArray[Any]: ... + +@overload +def setdiff1d( + ar1: _ArrayLike[_SCTNoCast], + ar2: _ArrayLike[_SCTNoCast], + assume_unique: bool = ..., +) -> NDArray[_SCTNoCast]: ... +@overload +def setdiff1d( + ar1: ArrayLike, + ar2: ArrayLike, + assume_unique: bool = ..., +) -> NDArray[Any]: ... diff --git a/numpy/lib/arrayterator.pyi b/numpy/lib/arrayterator.pyi new file mode 100644 index 000000000..aa192fb7c --- /dev/null +++ b/numpy/lib/arrayterator.pyi @@ -0,0 +1,49 @@ +from collections.abc import Generator +from typing import ( + Any, + TypeVar, + Union, + overload, +) + +from numpy import ndarray, dtype, generic +from numpy._typing import DTypeLike + +# TODO: Set a shape bound once we've got proper shape support +_Shape = TypeVar("_Shape", bound=Any) +_DType = TypeVar("_DType", bound=dtype[Any]) +_ScalarType = TypeVar("_ScalarType", bound=generic) + +_Index = Union[ + Union[ellipsis, int, slice], + tuple[Union[ellipsis, int, slice], ...], +] + +__all__: list[str] + +# NOTE: In reality `Arrayterator` does not actually inherit from `ndarray`, +# but its ``__getattr__` method does wrap around the former and thus has +# access to all its methods + +class Arrayterator(ndarray[_Shape, _DType]): + var: ndarray[_Shape, _DType] # type: ignore[assignment] + buf_size: None | int + start: list[int] + stop: list[int] + step: list[int] + + @property # type: ignore[misc] + def shape(self) -> tuple[int, ...]: ... + @property + def flat( # type: ignore[override] + self: ndarray[Any, dtype[_ScalarType]] + ) -> Generator[_ScalarType, None, None]: ... + def __init__( + self, var: ndarray[_Shape, _DType], buf_size: None | int = ... + ) -> None: ... + @overload + def __array__(self, dtype: None = ...) -> ndarray[Any, _DType]: ... + @overload + def __array__(self, dtype: DTypeLike) -> ndarray[Any, dtype[Any]]: ... + def __getitem__(self, index: _Index) -> Arrayterator[Any, _DType]: ... + def __iter__(self) -> Generator[ndarray[Any, _DType], None, None]: ... diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py deleted file mode 100644 index 709a79dc0..000000000 --- a/numpy/lib/financial.py +++ /dev/null @@ -1,967 +0,0 @@ -"""Some simple financial calculations - -patterned after spreadsheet computations. - -There is some complexity in each function -so that the functions behave like ufuncs with -broadcasting and being able to be called with scalars -or arrays (or other sequences). - -Functions support the :class:`decimal.Decimal` type unless -otherwise stated. -""" -import warnings -from decimal import Decimal -import functools - -import numpy as np -from numpy.core import overrides - - -_depmsg = ("numpy.{name} is deprecated and will be removed from NumPy 1.20. " - "Use numpy_financial.{name} instead " - "(https://pypi.org/project/numpy-financial/).") - -array_function_dispatch = functools.partial( - overrides.array_function_dispatch, module='numpy') - - -__all__ = ['fv', 'pmt', 'nper', 'ipmt', 'ppmt', 'pv', 'rate', - 'irr', 'npv', 'mirr'] - -_when_to_num = {'end':0, 'begin':1, - 'e':0, 'b':1, - 0:0, 1:1, - 'beginning':1, - 'start':1, - 'finish':0} - -def _convert_when(when): - #Test to see if when has already been converted to ndarray - #This will happen if one function calls another, for example ppmt - if isinstance(when, np.ndarray): - return when - try: - return _when_to_num[when] - except (KeyError, TypeError): - return [_when_to_num[x] for x in when] - - -def _fv_dispatcher(rate, nper, pmt, pv, when=None): - warnings.warn(_depmsg.format(name='fv'), - DeprecationWarning, stacklevel=3) - return (rate, nper, pmt, pv) - - -@array_function_dispatch(_fv_dispatcher) -def fv(rate, nper, pmt, pv, when='end'): - """ - Compute the future value. - - .. deprecated:: 1.18 - - `fv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a present value, `pv` - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * a (fixed) payment, `pmt`, paid either - * at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the value at the end of the `nper` periods - - Parameters - ---------- - rate : scalar or array_like of shape(M, ) - Rate of interest as decimal (not per cent) per period - nper : scalar or array_like of shape(M, ) - Number of compounding periods - pmt : scalar or array_like of shape(M, ) - Payment - pv : scalar or array_like of shape(M, ) - Present value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)). - Defaults to {'end', 0}. - - Returns - ------- - out : ndarray - Future values. If all input is scalar, returns a scalar float. If - any input is array_like, returns future values for each input element. - If multiple inputs are array_like, they all must have the same shape. - - Notes - ----- - The future value is computed by solving the equation:: - - fv + - pv*(1+rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) == 0 - - or, when ``rate == 0``:: - - fv + pv + pmt * nper == 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - - Examples - -------- - What is the future value after 10 years of saving $100 now, with - an additional monthly savings of $100. Assume the interest rate is - 5% (annually) compounded monthly? - - >>> np.fv(0.05/12, 10*12, -100, -100) - 15692.928894335748 - - By convention, the negative sign represents cash flow out (i.e. money not - available today). Thus, saving $100 a month at 5% annual interest leads - to $15,692.93 available to spend in 10 years. - - If any input is array_like, returns an array of equal shape. Let's - compare different interest rates from the example above. - - >>> a = np.array((0.05, 0.06, 0.07))/12 - >>> np.fv(a, 10*12, -100, -100) - array([ 15692.92889434, 16569.87435405, 17509.44688102]) # may vary - - """ - when = _convert_when(when) - (rate, nper, pmt, pv, when) = map(np.asarray, [rate, nper, pmt, pv, when]) - temp = (1+rate)**nper - fact = np.where(rate == 0, nper, - (1 + rate*when)*(temp - 1)/rate) - return -(pv*temp + pmt*fact) - - -def _pmt_dispatcher(rate, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='pmt'), - DeprecationWarning, stacklevel=3) - return (rate, nper, pv, fv) - - -@array_function_dispatch(_pmt_dispatcher) -def pmt(rate, nper, pv, fv=0, when='end'): - """ - Compute the payment against loan principal plus interest. - - .. deprecated:: 1.18 - - `pmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a present value, `pv` (e.g., an amount borrowed) - * a future value, `fv` (e.g., 0) - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * and (optional) specification of whether payment is made - at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the (fixed) periodic payment. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - nper : array_like - Number of compounding periods - pv : array_like - Present value - fv : array_like, optional - Future value (default = 0) - when : {{'begin', 1}, {'end', 0}}, {string, int} - When payments are due ('begin' (1) or 'end' (0)) - - Returns - ------- - out : ndarray - Payment against loan plus interest. If all input is scalar, returns a - scalar float. If any input is array_like, returns payment for each - input element. If multiple inputs are array_like, they all must have - the same shape. - - Notes - ----- - The payment is computed by solving the equation:: - - fv + - pv*(1 + rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) == 0 - - or, when ``rate == 0``:: - - fv + pv + pmt * nper == 0 - - for ``pmt``. - - Note that computing a monthly mortgage payment is only - one use for this function. For example, pmt returns the - periodic deposit one must make to achieve a specified - future balance given an initial deposit, a fixed, - periodically compounded interest rate, and the total - number of periods. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php - ?wg_abbrev=office-formulaOpenDocument-formula-20090508.odt - - Examples - -------- - What is the monthly payment needed to pay off a $200,000 loan in 15 - years at an annual interest rate of 7.5%? - - >>> np.pmt(0.075/12, 12*15, 200000) - -1854.0247200054619 - - In order to pay-off (i.e., have a future-value of 0) the $200,000 obtained - today, a monthly payment of $1,854.02 would be required. Note that this - example illustrates usage of `fv` having a default value of 0. - - """ - when = _convert_when(when) - (rate, nper, pv, fv, when) = map(np.array, [rate, nper, pv, fv, when]) - temp = (1 + rate)**nper - mask = (rate == 0) - masked_rate = np.where(mask, 1, rate) - fact = np.where(mask != 0, nper, - (1 + masked_rate*when)*(temp - 1)/masked_rate) - return -(fv + pv*temp) / fact - - -def _nper_dispatcher(rate, pmt, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='nper'), - DeprecationWarning, stacklevel=3) - return (rate, pmt, pv, fv) - - -@array_function_dispatch(_nper_dispatcher) -def nper(rate, pmt, pv, fv=0, when='end'): - """ - Compute the number of periodic payments. - - .. deprecated:: 1.18 - - `nper` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - :class:`decimal.Decimal` type is not supported. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - pmt : array_like - Payment - pv : array_like - Present value - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - - Notes - ----- - The number of periods ``nper`` is computed by solving the equation:: - - fv + pv*(1+rate)**nper + pmt*(1+rate*when)/rate*((1+rate)**nper-1) = 0 - - but if ``rate = 0`` then:: - - fv + pv + pmt*nper = 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - Examples - -------- - If you only had $150/month to pay towards the loan, how long would it take - to pay-off a loan of $8,000 at 7% annual interest? - - >>> print(np.round(np.nper(0.07/12, -150, 8000), 5)) - 64.07335 - - So, over 64 months would be required to pay off the loan. - - The same analysis could be done with several different interest rates - and/or payments and/or total amounts to produce an entire table. - - >>> np.nper(*(np.ogrid[0.07/12: 0.08/12: 0.01/12, - ... -150 : -99 : 50 , - ... 8000 : 9001 : 1000])) - array([[[ 64.07334877, 74.06368256], - [108.07548412, 127.99022654]], - [[ 66.12443902, 76.87897353], - [114.70165583, 137.90124779]]]) - - """ - when = _convert_when(when) - (rate, pmt, pv, fv, when) = map(np.asarray, [rate, pmt, pv, fv, when]) - - use_zero_rate = False - with np.errstate(divide="raise"): - try: - z = pmt*(1+rate*when)/rate - except FloatingPointError: - use_zero_rate = True - - if use_zero_rate: - return (-fv + pv) / pmt - else: - A = -(fv + pv)/(pmt+0) - B = np.log((-fv+z) / (pv+z))/np.log(1+rate) - return np.where(rate == 0, A, B) - - -def _ipmt_dispatcher(rate, per, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='ipmt'), - DeprecationWarning, stacklevel=3) - return (rate, per, nper, pv, fv) - - -@array_function_dispatch(_ipmt_dispatcher) -def ipmt(rate, per, nper, pv, fv=0, when='end'): - """ - Compute the interest portion of a payment. - - .. deprecated:: 1.18 - - `ipmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : scalar or array_like of shape(M, ) - Rate of interest as decimal (not per cent) per period - per : scalar or array_like of shape(M, ) - Interest paid against the loan changes during the life or the loan. - The `per` is the payment period to calculate the interest amount. - nper : scalar or array_like of shape(M, ) - Number of compounding periods - pv : scalar or array_like of shape(M, ) - Present value - fv : scalar or array_like of shape(M, ), optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)). - Defaults to {'end', 0}. - - Returns - ------- - out : ndarray - Interest portion of payment. If all input is scalar, returns a scalar - float. If any input is array_like, returns interest payment for each - input element. If multiple inputs are array_like, they all must have - the same shape. - - See Also - -------- - ppmt, pmt, pv - - Notes - ----- - The total payment is made up of payment against principal plus interest. - - ``pmt = ppmt + ipmt`` - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - Examples - -------- - What is the amortization schedule for a 1 year loan of $2500 at - 8.24% interest per year compounded monthly? - - >>> principal = 2500.00 - - The 'per' variable represents the periods of the loan. Remember that - financial equations start the period count at 1! - - >>> per = np.arange(1*12) + 1 - >>> ipmt = np.ipmt(0.0824/12, per, 1*12, principal) - >>> ppmt = np.ppmt(0.0824/12, per, 1*12, principal) - - Each element of the sum of the 'ipmt' and 'ppmt' arrays should equal - 'pmt'. - - >>> pmt = np.pmt(0.0824/12, 1*12, principal) - >>> np.allclose(ipmt + ppmt, pmt) - True - - >>> fmt = '{0:2d} {1:8.2f} {2:8.2f} {3:8.2f}' - >>> for payment in per: - ... index = payment - 1 - ... principal = principal + ppmt[index] - ... print(fmt.format(payment, ppmt[index], ipmt[index], principal)) - 1 -200.58 -17.17 2299.42 - 2 -201.96 -15.79 2097.46 - 3 -203.35 -14.40 1894.11 - 4 -204.74 -13.01 1689.37 - 5 -206.15 -11.60 1483.22 - 6 -207.56 -10.18 1275.66 - 7 -208.99 -8.76 1066.67 - 8 -210.42 -7.32 856.25 - 9 -211.87 -5.88 644.38 - 10 -213.32 -4.42 431.05 - 11 -214.79 -2.96 216.26 - 12 -216.26 -1.49 -0.00 - - >>> interestpd = np.sum(ipmt) - >>> np.round(interestpd, 2) - -112.98 - - """ - when = _convert_when(when) - rate, per, nper, pv, fv, when = np.broadcast_arrays(rate, per, nper, - pv, fv, when) - total_pmt = pmt(rate, nper, pv, fv, when) - ipmt = _rbl(rate, per, total_pmt, pv, when)*rate - try: - ipmt = np.where(when == 1, ipmt/(1 + rate), ipmt) - ipmt = np.where(np.logical_and(when == 1, per == 1), 0, ipmt) - except IndexError: - pass - return ipmt - - -def _rbl(rate, per, pmt, pv, when): - """ - This function is here to simply have a different name for the 'fv' - function to not interfere with the 'fv' keyword argument within the 'ipmt' - function. It is the 'remaining balance on loan' which might be useful as - its own function, but is easily calculated with the 'fv' function. - """ - return fv(rate, (per - 1), pmt, pv, when) - - -def _ppmt_dispatcher(rate, per, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='ppmt'), - DeprecationWarning, stacklevel=3) - return (rate, per, nper, pv, fv) - - -@array_function_dispatch(_ppmt_dispatcher) -def ppmt(rate, per, nper, pv, fv=0, when='end'): - """ - Compute the payment against loan principal. - - .. deprecated:: 1.18 - - `ppmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - per : array_like, int - Amount paid against the loan changes. The `per` is the period of - interest. - nper : array_like - Number of compounding periods - pv : array_like - Present value - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int} - When payments are due ('begin' (1) or 'end' (0)) - - See Also - -------- - pmt, pv, ipmt - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - """ - total = pmt(rate, nper, pv, fv, when) - return total - ipmt(rate, per, nper, pv, fv, when) - - -def _pv_dispatcher(rate, nper, pmt, fv=None, when=None): - warnings.warn(_depmsg.format(name='pv'), - DeprecationWarning, stacklevel=3) - return (rate, nper, nper, pv, fv) - - -@array_function_dispatch(_pv_dispatcher) -def pv(rate, nper, pmt, fv=0, when='end'): - """ - Compute the present value. - - .. deprecated:: 1.18 - - `pv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a future value, `fv` - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * a (fixed) payment, `pmt`, paid either - * at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the value now - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - nper : array_like - Number of compounding periods - pmt : array_like - Payment - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - - Returns - ------- - out : ndarray, float - Present value of a series of payments or investments. - - Notes - ----- - The present value is computed by solving the equation:: - - fv + - pv*(1 + rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) = 0 - - or, when ``rate = 0``:: - - fv + pv + pmt * nper = 0 - - for `pv`, which is then returned. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - Examples - -------- - What is the present value (e.g., the initial investment) - of an investment that needs to total $15692.93 - after 10 years of saving $100 every month? Assume the - interest rate is 5% (annually) compounded monthly. - - >>> np.pv(0.05/12, 10*12, -100, 15692.93) - -100.00067131625819 - - By convention, the negative sign represents cash flow out - (i.e., money not available today). Thus, to end up with - $15,692.93 in 10 years saving $100 a month at 5% annual - interest, one's initial deposit should also be $100. - - If any input is array_like, ``pv`` returns an array of equal shape. - Let's compare different interest rates in the example above: - - >>> a = np.array((0.05, 0.04, 0.03))/12 - >>> np.pv(a, 10*12, -100, 15692.93) - array([ -100.00067132, -649.26771385, -1273.78633713]) # may vary - - So, to end up with the same $15692.93 under the same $100 per month - "savings plan," for annual interest rates of 4% and 3%, one would - need initial investments of $649.27 and $1273.79, respectively. - - """ - when = _convert_when(when) - (rate, nper, pmt, fv, when) = map(np.asarray, [rate, nper, pmt, fv, when]) - temp = (1+rate)**nper - fact = np.where(rate == 0, nper, (1+rate*when)*(temp-1)/rate) - return -(fv + pmt*fact)/temp - -# Computed with Sage -# (y + (r + 1)^n*x + p*((r + 1)^n - 1)*(r*w + 1)/r)/(n*(r + 1)^(n - 1)*x - -# p*((r + 1)^n - 1)*(r*w + 1)/r^2 + n*p*(r + 1)^(n - 1)*(r*w + 1)/r + -# p*((r + 1)^n - 1)*w/r) - -def _g_div_gp(r, n, p, x, y, w): - t1 = (r+1)**n - t2 = (r+1)**(n-1) - return ((y + t1*x + p*(t1 - 1)*(r*w + 1)/r) / - (n*t2*x - p*(t1 - 1)*(r*w + 1)/(r**2) + n*p*t2*(r*w + 1)/r + - p*(t1 - 1)*w/r)) - - -def _rate_dispatcher(nper, pmt, pv, fv, when=None, guess=None, tol=None, - maxiter=None): - warnings.warn(_depmsg.format(name='rate'), - DeprecationWarning, stacklevel=3) - return (nper, pmt, pv, fv) - - -# Use Newton's iteration until the change is less than 1e-6 -# for all values or a maximum of 100 iterations is reached. -# Newton's rule is -# r_{n+1} = r_{n} - g(r_n)/g'(r_n) -# where -# g(r) is the formula -# g'(r) is the derivative with respect to r. -@array_function_dispatch(_rate_dispatcher) -def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): - """ - Compute the rate of interest per period. - - .. deprecated:: 1.18 - - `rate` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - nper : array_like - Number of compounding periods - pmt : array_like - Payment - pv : array_like - Present value - fv : array_like - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - guess : Number, optional - Starting guess for solving the rate of interest, default 0.1 - tol : Number, optional - Required tolerance for the solution, default 1e-6 - maxiter : int, optional - Maximum iterations in finding the solution - - Notes - ----- - The rate of interest is computed by iteratively solving the - (non-linear) equation:: - - fv + pv*(1+rate)**nper + pmt*(1+rate*when)/rate * ((1+rate)**nper - 1) = 0 - - for ``rate``. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - """ - when = _convert_when(when) - default_type = Decimal if isinstance(pmt, Decimal) else float - - # Handle casting defaults to Decimal if/when pmt is a Decimal and - # guess and/or tol are not given default values - if guess is None: - guess = default_type('0.1') - - if tol is None: - tol = default_type('1e-6') - - (nper, pmt, pv, fv, when) = map(np.asarray, [nper, pmt, pv, fv, when]) - - rn = guess - iterator = 0 - close = False - while (iterator < maxiter) and not close: - rnp1 = rn - _g_div_gp(rn, nper, pmt, pv, fv, when) - diff = abs(rnp1-rn) - close = np.all(diff < tol) - iterator += 1 - rn = rnp1 - if not close: - # Return nan's in array of the same shape as rn - return np.nan + rn - else: - return rn - - -def _irr_dispatcher(values): - warnings.warn(_depmsg.format(name='irr'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_irr_dispatcher) -def irr(values): - """ - Return the Internal Rate of Return (IRR). - - .. deprecated:: 1.18 - - `irr` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - This is the "average" periodically compounded rate of return - that gives a net present value of 0.0; for a more complete explanation, - see Notes below. - - :class:`decimal.Decimal` type is not supported. - - Parameters - ---------- - values : array_like, shape(N,) - Input cash flows per time period. By convention, net "deposits" - are negative and net "withdrawals" are positive. Thus, for - example, at least the first element of `values`, which represents - the initial investment, will typically be negative. - - Returns - ------- - out : float - Internal Rate of Return for periodic input values. - - Notes - ----- - The IRR is perhaps best understood through an example (illustrated - using np.irr in the Examples section below). Suppose one invests 100 - units and then makes the following withdrawals at regular (fixed) - intervals: 39, 59, 55, 20. Assuming the ending value is 0, one's 100 - unit investment yields 173 units; however, due to the combination of - compounding and the periodic withdrawals, the "average" rate of return - is neither simply 0.73/4 nor (1.73)^0.25-1. Rather, it is the solution - (for :math:`r`) of the equation: - - .. math:: -100 + \\frac{39}{1+r} + \\frac{59}{(1+r)^2} - + \\frac{55}{(1+r)^3} + \\frac{20}{(1+r)^4} = 0 - - In general, for `values` :math:`= [v_0, v_1, ... v_M]`, - irr is the solution of the equation: [2]_ - - .. math:: \\sum_{t=0}^M{\\frac{v_t}{(1+irr)^{t}}} = 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., - Addison-Wesley, 2003, pg. 348. - - Examples - -------- - >>> round(np.irr([-100, 39, 59, 55, 20]), 5) - 0.28095 - >>> round(np.irr([-100, 0, 0, 74]), 5) - -0.0955 - >>> round(np.irr([-100, 100, 0, -7]), 5) - -0.0833 - >>> round(np.irr([-100, 100, 0, 7]), 5) - 0.06206 - >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) - 0.0886 - - """ - # `np.roots` call is why this function does not support Decimal type. - # - # Ultimately Decimal support needs to be added to np.roots, which has - # greater implications on the entire linear algebra module and how it does - # eigenvalue computations. - res = np.roots(values[::-1]) - mask = (res.imag == 0) & (res.real > 0) - if not mask.any(): - return np.nan - res = res[mask].real - # NPV(rate) = 0 can have more than one solution so we return - # only the solution closest to zero. - rate = 1/res - 1 - rate = rate.item(np.argmin(np.abs(rate))) - return rate - - -def _npv_dispatcher(rate, values): - warnings.warn(_depmsg.format(name='npv'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_npv_dispatcher) -def npv(rate, values): - """ - Returns the NPV (Net Present Value) of a cash flow series. - - .. deprecated:: 1.18 - - `npv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : scalar - The discount rate. - values : array_like, shape(M, ) - The values of the time series of cash flows. The (fixed) time - interval between cash flow "events" must be the same as that for - which `rate` is given (i.e., if `rate` is per year, then precisely - a year is understood to elapse between each cash flow event). By - convention, investments or "deposits" are negative, income or - "withdrawals" are positive; `values` must begin with the initial - investment, thus `values[0]` will typically be negative. - - Returns - ------- - out : float - The NPV of the input cash flow series `values` at the discount - `rate`. - - Warnings - -------- - ``npv`` considers a series of cashflows starting in the present (t = 0). - NPV can also be defined with a series of future cashflows, paid at the - end, rather than the start, of each period. If future cashflows are used, - the first cashflow `values[0]` must be zeroed and added to the net - present value of the future cashflows. This is demonstrated in the - examples. - - Notes - ----- - Returns the result of: [2]_ - - .. math :: \\sum_{t=0}^{M-1}{\\frac{values_t}{(1+rate)^{t}}} - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., - Addison-Wesley, 2003, pg. 346. - - Examples - -------- - Consider a potential project with an initial investment of $40 000 and - projected cashflows of $5 000, $8 000, $12 000 and $30 000 at the end of - each period discounted at a rate of 8% per period. To find the project's - net present value: - - >>> rate, cashflows = 0.08, [-40_000, 5_000, 8_000, 12_000, 30_000] - >>> np.npv(rate, cashflows).round(5) - 3065.22267 - - It may be preferable to split the projected cashflow into an initial - investment and expected future cashflows. In this case, the value of - the initial cashflow is zero and the initial investment is later added - to the future cashflows net present value: - - >>> initial_cashflow = cashflows[0] - >>> cashflows[0] = 0 - >>> np.round(np.npv(rate, cashflows) + initial_cashflow, 5) - 3065.22267 - - """ - values = np.asarray(values) - return (values / (1+rate)**np.arange(0, len(values))).sum(axis=0) - - -def _mirr_dispatcher(values, finance_rate, reinvest_rate): - warnings.warn(_depmsg.format(name='mirr'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_mirr_dispatcher) -def mirr(values, finance_rate, reinvest_rate): - """ - Modified internal rate of return. - - .. deprecated:: 1.18 - - `mirr` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - values : array_like - Cash flows (must contain at least one positive and one negative - value) or nan is returned. The first value is considered a sunk - cost at time zero. - finance_rate : scalar - Interest rate paid on the cash flows - reinvest_rate : scalar - Interest rate received on the cash flows upon reinvestment - - Returns - ------- - out : float - Modified internal rate of return - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - """ - values = np.asarray(values) - n = values.size - - # Without this explicit cast the 1/(n - 1) computation below - # becomes a float, which causes TypeError when using Decimal - # values. - if isinstance(finance_rate, Decimal): - n = Decimal(n) - - pos = values > 0 - neg = values < 0 - if not (pos.any() and neg.any()): - return np.nan - numer = np.abs(npv(reinvest_rate, values*pos)) - denom = np.abs(npv(finance_rate, values*neg)) - return (numer/denom)**(1/(n - 1))*(1 + reinvest_rate) - 1 diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 4e6e731c1..ef50fb19d 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -41,10 +41,10 @@ Capabilities - Is straightforward to reverse engineer. Datasets often live longer than the programs that created them. A competent developer should be able to create a solution in their preferred programming language to - read most ``.npy`` files that he has been given without much + read most ``.npy`` files that they have been given without much documentation. -- Allows memory-mapping of the data. See `open_memmep`. +- Allows memory-mapping of the data. See `open_memmap`. - Can be read from a filelike stream object instead of an actual file. @@ -162,7 +162,6 @@ evolved with time and this document is more current. """ import numpy -import io import warnings from numpy.lib.utils import safe_eval from numpy.compat import ( @@ -173,10 +172,13 @@ from numpy.compat import ( __all__ = [] +EXPECTED_KEYS = {'descr', 'fortran_order', 'shape'} MAGIC_PREFIX = b'\x93NUMPY' MAGIC_LEN = len(MAGIC_PREFIX) + 2 ARRAY_ALIGN = 64 # plausible values are powers of 2 between 16 and 4096 BUFFER_SIZE = 2**18 # size of buffer for reading npz files in bytes +# allow growth within the address space of a 64 bit machine along one axis +GROWTH_AXIS_MAX_DIGITS = 21 # = len(str(8*2**64-1)) hypothetical int1 dtype # difference between version 1.0 and 2.0 is a 4 byte (I) header length # instead of 2 bytes (H) allowing storage of large structured arrays @@ -186,6 +188,10 @@ _header_size_info = { (3, 0): ('<I', 'utf8'), } +# Python's literal_eval is not actually safe for large inputs, since parsing +# may become slow or even cause interpreter crashes. +# This is an arbitrary, low limit which should make it safe in practice. +_MAX_HEADER_SIZE = 10000 def _check_version(version): if version not in [(1, 0), (2, 0), (3, 0), None]: @@ -291,7 +297,7 @@ def descr_to_dtype(descr): Parameters ---------- descr : object - The object retreived by dtype.descr. Can be passed to + The object retrieved by dtype.descr. Can be passed to `numpy.dtype()` in order to replicate the input dtype. Returns @@ -370,15 +376,14 @@ def _wrap_header(header, version): import struct assert version is not None fmt, encoding = _header_size_info[version] - if not isinstance(header, bytes): # always true on python 3 - header = header.encode(encoding) + header = header.encode(encoding) hlen = len(header) + 1 padlen = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize(fmt) + hlen) % ARRAY_ALIGN) try: header_prefix = magic(*version) + struct.pack(fmt, hlen + padlen) except struct.error: msg = "Header length {} too big for version={}".format(hlen, version) - raise ValueError(msg) + raise ValueError(msg) from None # Pad the header with spaces and a final newline such that the magic # string, the header-length short and the header are aligned on a @@ -421,10 +426,10 @@ def _write_array_header(fp, d, version=None): d : dict This has the appropriate entries for writing its string representation to the header of the file. - version: tuple or None - None means use oldest that works - explicit version will raise a ValueError if the format does not - allow saving this data. Default: None + version : tuple or None + None means use oldest that works. Providing an explicit version will + raise a ValueError if the format does not allow saving this data. + Default: None """ header = ["{"] for key, value in sorted(d.items()): @@ -432,7 +437,15 @@ def _write_array_header(fp, d, version=None): header.append("'%s': %s, " % (key, repr(value))) header.append("}") header = "".join(header) - header = _filter_header(header) + + # Add some spare space so that the array header can be modified in-place + # when changing the array size, e.g. when growing it by appending data at + # the end. + shape = d['shape'] + header += " " * ((GROWTH_AXIS_MAX_DIGITS - len(repr( + shape[-1 if d['fortran_order'] else 0] + ))) if len(shape) > 0 else 0) + if version is None: header = _wrap_header_guess_version(header) else: @@ -467,7 +480,7 @@ def write_array_header_2_0(fp, d): """ _write_array_header(fp, d, (2, 0)) -def read_array_header_1_0(fp): +def read_array_header_1_0(fp, max_header_size=_MAX_HEADER_SIZE): """ Read an array header from a filelike object using the 1.0 file format version. @@ -489,6 +502,10 @@ def read_array_header_1_0(fp): contiguous before writing it out. dtype : dtype The dtype of the file's data. + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. Raises ------ @@ -496,9 +513,10 @@ def read_array_header_1_0(fp): If the data is invalid. """ - return _read_array_header(fp, version=(1, 0)) + return _read_array_header( + fp, version=(1, 0), max_header_size=max_header_size) -def read_array_header_2_0(fp): +def read_array_header_2_0(fp, max_header_size=_MAX_HEADER_SIZE): """ Read an array header from a filelike object using the 2.0 file format version. @@ -511,6 +529,10 @@ def read_array_header_2_0(fp): ---------- fp : filelike object A file object or something with a `.read()` method like a file. + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. Returns ------- @@ -529,7 +551,8 @@ def read_array_header_2_0(fp): If the data is invalid. """ - return _read_array_header(fp, version=(2, 0)) + return _read_array_header( + fp, version=(2, 0), max_header_size=max_header_size) def _filter_header(s): @@ -567,7 +590,7 @@ def _filter_header(s): return tokenize.untokenize(tokens) -def _read_array_header(fp, version): +def _read_array_header(fp, version, max_header_size=_MAX_HEADER_SIZE): """ see read_array_header_1_0 """ @@ -583,6 +606,14 @@ def _read_array_header(fp, version): header_length = struct.unpack(hlength_type, hlength_str)[0] header = _read_bytes(fp, header_length, "array header") header = header.decode(encoding) + if len(header) > max_header_size: + raise ValueError( + f"Header info length ({len(header)}) is large and may not be safe " + "to load securely.\n" + "To allow loading, adjust `max_header_size` or fully trust " + "the `.npy` file using `allow_pickle=True`.\n" + "For safety against large resource use or crashes, sandboxing " + "may be necessary.") # The header is a pretty-printed string representation of a literal # Python dictionary with trailing newlines padded to a ARRAY_ALIGN byte @@ -590,23 +621,41 @@ def _read_array_header(fp, version): # "shape" : tuple of int # "fortran_order" : bool # "descr" : dtype.descr - header = _filter_header(header) + # Versions (2, 0) and (1, 0) could have been created by a Python 2 + # implementation before header filtering was implemented. + # + # For performance reasons, we try without _filter_header first though try: d = safe_eval(header) except SyntaxError as e: - msg = "Cannot parse header: {!r}\nException: {!r}" - raise ValueError(msg.format(header, e)) + if version <= (2, 0): + header = _filter_header(header) + try: + d = safe_eval(header) + except SyntaxError as e2: + msg = "Cannot parse header: {!r}" + raise ValueError(msg.format(header)) from e2 + else: + warnings.warn( + "Reading `.npy` or `.npz` file required additional " + "header parsing as it was created on Python 2. Save the " + "file again to speed up loading and avoid this warning.", + UserWarning, stacklevel=4) + else: + msg = "Cannot parse header: {!r}" + raise ValueError(msg.format(header)) from e if not isinstance(d, dict): msg = "Header is not a dictionary: {!r}" raise ValueError(msg.format(d)) - keys = sorted(d.keys()) - if keys != ['descr', 'fortran_order', 'shape']: + + if EXPECTED_KEYS != d.keys(): + keys = sorted(d.keys()) msg = "Header does not contain the correct keys: {!r}" raise ValueError(msg.format(keys)) # Sanity-check the values. if (not isinstance(d['shape'], tuple) or - not numpy.all([isinstance(x, int) for x in d['shape']])): + not all(isinstance(x, int) for x in d['shape'])): msg = "shape is not valid: {!r}" raise ValueError(msg.format(d['shape'])) if not isinstance(d['fortran_order'], bool): @@ -614,9 +663,9 @@ def _read_array_header(fp, version): raise ValueError(msg.format(d['fortran_order'])) try: dtype = descr_to_dtype(d['descr']) - except TypeError: + except TypeError as e: msg = "descr is not a valid dtype descriptor: {!r}" - raise ValueError(msg.format(d['descr'])) + raise ValueError(msg.format(d['descr'])) from e return d['shape'], d['fortran_order'], dtype @@ -692,7 +741,8 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): fp.write(chunk.tobytes('C')) -def read_array(fp, allow_pickle=False, pickle_kwargs=None): +def read_array(fp, allow_pickle=False, pickle_kwargs=None, *, + max_header_size=_MAX_HEADER_SIZE): """ Read an array from an NPY file. @@ -711,6 +761,12 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): Additional keyword arguments to pass to pickle.load. These are only useful when loading object arrays saved on Python 2 when using Python 3. + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. + This option is ignored when `allow_pickle` is passed. In that case + the file is by definition trusted and the limit is unnecessary. Returns ------- @@ -724,9 +780,15 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): an object array. """ + if allow_pickle: + # Effectively ignore max_header_size, since `allow_pickle` indicates + # that the input is fully trusted. + max_header_size = 2**64 + version = read_magic(fp) _check_version(version) - shape, fortran_order, dtype = _read_array_header(fp, version) + shape, fortran_order, dtype = _read_array_header( + fp, version, max_header_size=max_header_size) if len(shape) == 0: count = 1 else: @@ -746,7 +808,7 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): # Friendlier error message raise UnicodeError("Unpickling a python object failed: %r\n" "You may need to pass the encoding= option " - "to numpy.load" % (err,)) + "to numpy.load" % (err,)) from err else: if isfileobj(fp): # We can use the fast fromfile() function. @@ -786,7 +848,8 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): def open_memmap(filename, mode='r+', dtype=None, shape=None, - fortran_order=False, version=None): + fortran_order=False, version=None, *, + max_header_size=_MAX_HEADER_SIZE): """ Open a .npy file as a memory-mapped array. @@ -817,6 +880,10 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, If the mode is a "write" mode, then this is the version of the file format used to create the file. None means use the oldest supported version that is able to store the data. Default: None + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. Returns ------- @@ -827,7 +894,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, ------ ValueError If the data or the mode is invalid. - IOError + OSError If the file is not found or cannot be opened correctly. See Also @@ -864,7 +931,8 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, version = read_magic(fp) _check_version(version) - shape, fortran_order, dtype = _read_array_header(fp, version) + shape, fortran_order, dtype = _read_array_header( + fp, version, max_header_size=max_header_size) if dtype.hasobject: msg = "Array can't be memory-mapped: Python objects in dtype." raise ValueError(msg) @@ -905,7 +973,7 @@ def _read_bytes(fp, size, error_template="ran out of data"): data += r if len(r) == 0 or len(data) == size: break - except io.BlockingIOError: + except BlockingIOError: pass if len(data) != size: msg = "EOF: reading %s, expected %d bytes got %d" diff --git a/numpy/lib/format.pyi b/numpy/lib/format.pyi new file mode 100644 index 000000000..a4468f52f --- /dev/null +++ b/numpy/lib/format.pyi @@ -0,0 +1,22 @@ +from typing import Any, Literal, Final + +__all__: list[str] + +EXPECTED_KEYS: Final[set[str]] +MAGIC_PREFIX: Final[bytes] +MAGIC_LEN: Literal[8] +ARRAY_ALIGN: Literal[64] +BUFFER_SIZE: Literal[262144] # 2**18 + +def magic(major, minor): ... +def read_magic(fp): ... +def dtype_to_descr(dtype): ... +def descr_to_dtype(descr): ... +def header_data_from_array_1_0(array): ... +def write_array_header_1_0(fp, d): ... +def write_array_header_2_0(fp, d): ... +def read_array_header_1_0(fp): ... +def read_array_header_2_0(fp): ... +def write_array(fp, array, version=..., allow_pickle=..., pickle_kwargs=...): ... +def read_array(fp, allow_pickle=..., pickle_kwargs=...): ... +def open_memmap(filename, mode=..., dtype=..., shape=..., fortran_order=..., version=...): ... diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 6ea9cc4de..22371a038 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -4,13 +4,13 @@ import re import sys import warnings +from .._utils import set_module import numpy as np import numpy.core.numeric as _nx from numpy.core import transpose from numpy.core.numeric import ( - ones, zeros, arange, concatenate, array, asarray, asanyarray, empty, - ndarray, around, floor, ceil, take, dot, where, intp, - integer, isscalar, absolute + ones, zeros_like, arange, concatenate, array, asarray, asanyarray, empty, + ndarray, take, dot, where, intp, integer, isscalar, absolute ) from numpy.core.umath import ( pi, add, arctan2, frompyfunc, cos, less_equal, sqrt, sin, @@ -20,12 +20,11 @@ from numpy.core.fromnumeric import ( ravel, nonzero, partition, mean, any, sum ) from numpy.core.numerictypes import typecodes -from numpy.core.overrides import set_module from numpy.core import overrides from numpy.core.function_base import add_newdoc from numpy.lib.twodim_base import diag from numpy.core.multiarray import ( - _insert, add_docstring, bincount, normalize_axis_index, _monotonicity, + _place, add_docstring, bincount, normalize_axis_index, _monotonicity, interp as compiled_interp, interp_complex as compiled_interp_complex ) from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc @@ -33,7 +32,7 @@ from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc import builtins # needed in this module for compatibility -from numpy.lib.histograms import histogram, histogramdd +from numpy.lib.histograms import histogram, histogramdd # noqa: F401 array_function_dispatch = functools.partial( @@ -51,6 +50,106 @@ __all__ = [ 'quantile' ] +# _QuantileMethods is a dictionary listing all the supported methods to +# compute quantile/percentile. +# +# Below virtual_index refer to the index of the element where the percentile +# would be found in the sorted sample. +# When the sample contains exactly the percentile wanted, the virtual_index is +# an integer to the index of this element. +# When the percentile wanted is in between two elements, the virtual_index +# is made of a integer part (a.k.a 'i' or 'left') and a fractional part +# (a.k.a 'g' or 'gamma') +# +# Each method in _QuantileMethods has two properties +# get_virtual_index : Callable +# The function used to compute the virtual_index. +# fix_gamma : Callable +# A function used for discret methods to force the index to a specific value. +_QuantileMethods = dict( + # --- HYNDMAN and FAN METHODS + # Discrete methods + inverted_cdf=dict( + get_virtual_index=lambda n, quantiles: _inverted_cdf(n, quantiles), + fix_gamma=lambda gamma, _: gamma, # should never be called + ), + averaged_inverted_cdf=dict( + get_virtual_index=lambda n, quantiles: (n * quantiles) - 1, + fix_gamma=lambda gamma, _: _get_gamma_mask( + shape=gamma.shape, + default_value=1., + conditioned_value=0.5, + where=gamma == 0), + ), + closest_observation=dict( + get_virtual_index=lambda n, quantiles: _closest_observation(n, + quantiles), + fix_gamma=lambda gamma, _: gamma, # should never be called + ), + # Continuous methods + interpolated_inverted_cdf=dict( + get_virtual_index=lambda n, quantiles: + _compute_virtual_index(n, quantiles, 0, 1), + fix_gamma=lambda gamma, _: gamma, + ), + hazen=dict( + get_virtual_index=lambda n, quantiles: + _compute_virtual_index(n, quantiles, 0.5, 0.5), + fix_gamma=lambda gamma, _: gamma, + ), + weibull=dict( + get_virtual_index=lambda n, quantiles: + _compute_virtual_index(n, quantiles, 0, 0), + fix_gamma=lambda gamma, _: gamma, + ), + # Default method. + # To avoid some rounding issues, `(n-1) * quantiles` is preferred to + # `_compute_virtual_index(n, quantiles, 1, 1)`. + # They are mathematically equivalent. + linear=dict( + get_virtual_index=lambda n, quantiles: (n - 1) * quantiles, + fix_gamma=lambda gamma, _: gamma, + ), + median_unbiased=dict( + get_virtual_index=lambda n, quantiles: + _compute_virtual_index(n, quantiles, 1 / 3.0, 1 / 3.0), + fix_gamma=lambda gamma, _: gamma, + ), + normal_unbiased=dict( + get_virtual_index=lambda n, quantiles: + _compute_virtual_index(n, quantiles, 3 / 8.0, 3 / 8.0), + fix_gamma=lambda gamma, _: gamma, + ), + # --- OTHER METHODS + lower=dict( + get_virtual_index=lambda n, quantiles: np.floor( + (n - 1) * quantiles).astype(np.intp), + fix_gamma=lambda gamma, _: gamma, + # should never be called, index dtype is int + ), + higher=dict( + get_virtual_index=lambda n, quantiles: np.ceil( + (n - 1) * quantiles).astype(np.intp), + fix_gamma=lambda gamma, _: gamma, + # should never be called, index dtype is int + ), + midpoint=dict( + get_virtual_index=lambda n, quantiles: 0.5 * ( + np.floor((n - 1) * quantiles) + + np.ceil((n - 1) * quantiles)), + fix_gamma=lambda gamma, index: _get_gamma_mask( + shape=gamma.shape, + default_value=0.5, + conditioned_value=0., + where=index % 1 == 0), + ), + nearest=dict( + get_virtual_index=lambda n, quantiles: np.around( + (n - 1) * quantiles).astype(np.intp), + fix_gamma=lambda gamma, _: gamma, + # should never be called, index dtype is int + )) + def _rot90_dispatcher(m, k=None, axes=None): return (m,) @@ -62,6 +161,8 @@ def rot90(m, k=1, axes=(0, 1)): Rotate an array by 90 degrees in the plane specified by axes. Rotation direction is from the first towards the second axis. + This means for a 2D array with the default `k` and `axes`, the + rotation will be counterclockwise. Parameters ---------- @@ -69,7 +170,7 @@ def rot90(m, k=1, axes=(0, 1)): Array of two or more dimensions. k : integer Number of times the array is rotated by 90 degrees. - axes: (2,) array_like + axes : (2,) array_like The array is rotated in the plane defined by the axes. Axes must be different. @@ -88,8 +189,11 @@ def rot90(m, k=1, axes=(0, 1)): Notes ----- - rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1)) - rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1)) + ``rot90(m, k=1, axes=(1,0))`` is the reverse of + ``rot90(m, k=1, axes=(0,1))`` + + ``rot90(m, k=1, axes=(1,0))`` is equivalent to + ``rot90(m, k=-1, axes=(0,1))`` Examples -------- @@ -265,6 +369,19 @@ def iterable(y): >>> np.iterable(2) False + Notes + ----- + In most cases, the results of ``np.iterable(obj)`` are consistent with + ``isinstance(obj, collections.abc.Iterable)``. One notable exception is + the treatment of 0-dimensional arrays:: + + >>> from collections.abc import Iterable + >>> a = np.array(1.0) # 0-dimensional numpy array + >>> isinstance(a, Iterable) + True + >>> np.iterable(a) + False + """ try: iter(y) @@ -273,12 +390,14 @@ def iterable(y): return True -def _average_dispatcher(a, axis=None, weights=None, returned=None): +def _average_dispatcher(a, axis=None, weights=None, returned=None, *, + keepdims=None): return (a, weights) @array_function_dispatch(_average_dispatcher) -def average(a, axis=None, weights=None, returned=False): +def average(a, axis=None, weights=None, returned=False, *, + keepdims=np._NoValue): """ Compute the weighted average along the specified axis. @@ -313,6 +432,14 @@ def average(a, axis=None, weights=None, returned=False): is returned, otherwise only the average is returned. If `weights=None`, `sum_of_weights` is equivalent to the number of elements over which the average is taken. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the original `a`. + *Note:* `keepdims` will not work with instances of `numpy.matrix` + or other classes whose methods do not support `keepdims`. + + .. versionadded:: 1.23.0 Returns ------- @@ -356,7 +483,7 @@ def average(a, axis=None, weights=None, returned=False): >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1)) 4.0 - >>> data = np.arange(6).reshape((3,2)) + >>> data = np.arange(6).reshape((3, 2)) >>> data array([[0, 1], [2, 3], @@ -373,12 +500,26 @@ def average(a, axis=None, weights=None, returned=False): >>> avg = np.average(a, weights=w) >>> print(avg.dtype) complex256 + + With ``keepdims=True``, the following result has shape (3, 1). + + >>> np.average(data, axis=1, keepdims=True) + array([[0.5], + [2.5], + [4.5]]) """ a = np.asanyarray(a) + if keepdims is np._NoValue: + # Don't pass on the keepdims argument if one wasn't given. + keepdims_kw = {} + else: + keepdims_kw = {'keepdims': keepdims} + if weights is None: - avg = a.mean(axis) - scl = avg.dtype.type(a.size/avg.size) + avg = a.mean(axis, **keepdims_kw) + avg_as_array = np.asanyarray(avg) + scl = avg_as_array.dtype.type(a.size/avg_as_array.size) else: wgt = np.asanyarray(weights) @@ -404,16 +545,17 @@ def average(a, axis=None, weights=None, returned=False): wgt = np.broadcast_to(wgt, (a.ndim-1)*(1,) + wgt.shape) wgt = wgt.swapaxes(-1, axis) - scl = wgt.sum(axis=axis, dtype=result_dtype) + scl = wgt.sum(axis=axis, dtype=result_dtype, **keepdims_kw) if np.any(scl == 0.0): raise ZeroDivisionError( "Weights sum to zero, can't be normalized") - avg = np.multiply(a, wgt, dtype=result_dtype).sum(axis)/scl + avg = avg_as_array = np.multiply(a, wgt, + dtype=result_dtype).sum(axis, **keepdims_kw) / scl if returned: - if scl.shape != avg.shape: - scl = np.broadcast_to(scl, avg.shape).copy() + if scl.shape != avg_as_array.shape: + scl = np.broadcast_to(scl, avg_as_array.shape).copy() return avg, scl else: return avg @@ -433,7 +575,7 @@ def asarray_chkfinite(a, dtype=None, order=None): By default, the data-type is inferred from the input data. order : {'C', 'F', 'A', 'K'}, optional Memory layout. 'A' and 'K' depend on the order of input array a. - 'C' row-major (C-style), + 'C' row-major (C-style), 'F' column-major (Fortran-style) memory representation. 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise 'K' (keep) preserve input order @@ -593,7 +735,7 @@ def piecewise(x, condlist, funclist, *args, **kw): not isinstance(condlist[0], (list, ndarray)) and x.ndim != 0): condlist = [condlist] - condlist = array(condlist, dtype=bool) + condlist = asarray(condlist, dtype=bool) n = len(condlist) if n == n2 - 1: # compute the "otherwise" condition. @@ -606,7 +748,7 @@ def piecewise(x, condlist, funclist, *args, **kw): .format(n, n, n+1) ) - y = zeros(x.shape, x.dtype) + y = zeros_like(x) for cond, func in zip(condlist, funclist): if not isinstance(func, collections.abc.Callable): y[cond] = func @@ -654,11 +796,16 @@ def select(condlist, choicelist, default=0): Examples -------- - >>> x = np.arange(10) - >>> condlist = [x<3, x>5] + >>> x = np.arange(6) + >>> condlist = [x<3, x>3] >>> choicelist = [x, x**2] - >>> np.select(condlist, choicelist) - array([ 0, 1, 2, ..., 49, 64, 81]) + >>> np.select(condlist, choicelist, 42) + array([ 0, 1, 2, 42, 16, 25]) + + >>> condlist = [x<=4, x>3] + >>> choicelist = [x, x**2] + >>> np.select(condlist, choicelist, 55) + array([ 0, 1, 2, 3, 4, 25]) """ # Check the size of condlist and choicelist are the same, or abort. @@ -671,11 +818,22 @@ def select(condlist, choicelist, default=0): raise ValueError("select with an empty condition list is not possible") choicelist = [np.asarray(choice) for choice in choicelist] - choicelist.append(np.asarray(default)) + + try: + intermediate_dtype = np.result_type(*choicelist) + except TypeError as e: + msg = f'Choicelist elements do not have a common dtype: {e}' + raise TypeError(msg) from None + default_array = np.asarray(default) + choicelist.append(default_array) # need to get the result type before broadcasting for correct scalar # behaviour - dtype = np.result_type(*choicelist) + try: + dtype = np.result_type(intermediate_dtype, default_array) + except TypeError as e: + msg = f'Choicelists and default value do not have a common dtype: {e}' + raise TypeError(msg) from None # Convert conditions to arrays and broadcast conditions and choices # as the shape is needed for the result. Doing it separately optimizes @@ -765,6 +923,17 @@ def copy(a, order='K', subok=False): >>> x[0] == z[0] False + Note that, np.copy clears previously set WRITEABLE=False flag. + + >>> a = np.array([1, 2, 3]) + >>> a.flags["WRITEABLE"] = False + >>> b = np.copy(a) + >>> b.flags["WRITEABLE"] + True + >>> b[0] = 3 + >>> b + array([3, 2, 3]) + Note that np.copy is a shallow copy and will not copy object elements within arrays. This is mainly important for arrays containing Python objects. The new array will contain the @@ -846,7 +1015,7 @@ def gradient(f, *varargs, axis=None, edge_order=1): Returns ------- gradient : ndarray or list of ndarray - A set of ndarrays (or a single ndarray if there is only one dimension) + A list of ndarrays (or a single ndarray if there is only one dimension) corresponding to the derivatives of f with respect to each dimension. Each derivative has the same shape as f. @@ -1142,6 +1311,8 @@ def gradient(f, *varargs, axis=None, edge_order=1): if len_axes == 1: return outvals[0] + elif np._using_numpy2_behavior(): + return tuple(outvals) else: return outvals @@ -1290,7 +1461,7 @@ def _interp_dispatcher(x, xp, fp, left=None, right=None, period=None): @array_function_dispatch(_interp_dispatcher) def interp(x, xp, fp, left=None, right=None, period=None): """ - One-dimensional linear interpolation. + One-dimensional linear interpolation for monotonically increasing sample points. Returns the one-dimensional piecewise linear interpolant to a function with given discrete data points (`xp`, `fp`), evaluated at `x`. @@ -1337,8 +1508,8 @@ def interp(x, xp, fp, left=None, right=None, period=None): -------- scipy.interpolate - Notes - ----- + Warnings + -------- The x-coordinate sequence is expected to be increasing, but this is not explicitly enforced. However, if the sequence `xp` is non-increasing, interpolation results are meaningless. @@ -1450,7 +1621,7 @@ def angle(z, deg=False): The counterclockwise angle from the positive real axis on the complex plane in the range ``(-pi, pi]``, with dtype as numpy.float64. - ..versionchanged:: 1.16.0 + .. versionchanged:: 1.16.0 This function works on subclasses of ndarray like `ma.array`. See Also @@ -1485,26 +1656,40 @@ def angle(z, deg=False): return a -def _unwrap_dispatcher(p, discont=None, axis=None): +def _unwrap_dispatcher(p, discont=None, axis=None, *, period=None): return (p,) @array_function_dispatch(_unwrap_dispatcher) -def unwrap(p, discont=pi, axis=-1): - """ - Unwrap by changing deltas between values to 2*pi complement. +def unwrap(p, discont=None, axis=-1, *, period=2*pi): + r""" + Unwrap by taking the complement of large deltas with respect to the period. + + This unwraps a signal `p` by changing elements which have an absolute + difference from their predecessor of more than ``max(discont, period/2)`` + to their `period`-complementary values. - Unwrap radian phase `p` by changing absolute jumps greater than - `discont` to their 2*pi complement along the given axis. + For the default case where `period` is :math:`2\pi` and `discont` is + :math:`\pi`, this unwraps a radian phase `p` such that adjacent differences + are never greater than :math:`\pi` by adding :math:`2k\pi` for some + integer :math:`k`. Parameters ---------- p : array_like Input array. discont : float, optional - Maximum discontinuity between values, default is ``pi``. + Maximum discontinuity between values, default is ``period/2``. + Values below ``period/2`` are treated as if they were ``period/2``. + To have an effect different from the default, `discont` should be + larger than ``period/2``. axis : int, optional Axis along which unwrap will operate, default is the last axis. + period : float, optional + Size of the range over which the input wraps. By default, it is + ``2 pi``. + + .. versionadded:: 1.21.0 Returns ------- @@ -1517,9 +1702,9 @@ def unwrap(p, discont=pi, axis=-1): Notes ----- - If the discontinuity in `p` is smaller than ``pi``, but larger than - `discont`, no unwrapping is done because taking the 2*pi complement - would only make the discontinuity larger. + If the discontinuity in `p` is smaller than ``period/2``, + but larger than `discont`, no unwrapping is done because taking + the complement would only make the discontinuity larger. Examples -------- @@ -1529,19 +1714,44 @@ def unwrap(p, discont=pi, axis=-1): array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) # may vary >>> np.unwrap(phase) array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) # may vary - + >>> np.unwrap([0, 1, 2, -1, 0], period=4) + array([0, 1, 2, 3, 4]) + >>> np.unwrap([ 1, 2, 3, 4, 5, 6, 1, 2, 3], period=6) + array([1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.unwrap([2, 3, 4, 5, 2, 3, 4, 5], period=4) + array([2, 3, 4, 5, 6, 7, 8, 9]) + >>> phase_deg = np.mod(np.linspace(0 ,720, 19), 360) - 180 + >>> np.unwrap(phase_deg, period=360) + array([-180., -140., -100., -60., -20., 20., 60., 100., 140., + 180., 220., 260., 300., 340., 380., 420., 460., 500., + 540.]) """ p = asarray(p) nd = p.ndim dd = diff(p, axis=axis) + if discont is None: + discont = period/2 slice1 = [slice(None, None)]*nd # full slices slice1[axis] = slice(1, None) slice1 = tuple(slice1) - ddmod = mod(dd + pi, 2*pi) - pi - _nx.copyto(ddmod, pi, where=(ddmod == -pi) & (dd > 0)) + dtype = np.result_type(dd, period) + if _nx.issubdtype(dtype, _nx.integer): + interval_high, rem = divmod(period, 2) + boundary_ambiguous = rem == 0 + else: + interval_high = period / 2 + boundary_ambiguous = True + interval_low = -interval_high + ddmod = mod(dd - interval_low, period) + interval_low + if boundary_ambiguous: + # for `mask = (abs(dd) == period/2)`, the above line made + # `ddmod[mask] == -period/2`. correct these such that + # `ddmod[mask] == sign(dd[mask])*period/2`. + _nx.copyto(ddmod, interval_high, + where=(ddmod == interval_low) & (dd > 0)) ph_correct = ddmod - dd _nx.copyto(ph_correct, 0, where=abs(dd) < discont) - up = array(p, copy=True, dtype='d') + up = array(p, copy=True, dtype=dtype) up[slice1] = p[slice1] + ph_correct.cumsum(axis) return up @@ -1625,6 +1835,7 @@ def trim_zeros(filt, trim='fb'): [1, 2] """ + first = 0 trim = trim.upper() if 'F' in trim: @@ -1740,11 +1951,7 @@ def place(arr, mask, vals): [44, 55, 44]]) """ - if not isinstance(arr, np.ndarray): - raise TypeError("argument 1 must be numpy.ndarray, " - "not {name}".format(name=type(arr).__name__)) - - return _insert(arr, mask, vals) + return _place(arr, mask, vals) def disp(mesg, device=None, linefeed=True): @@ -1812,6 +2019,8 @@ def _parse_gufunc_signature(signature): Tuple of input and output core dimensions parsed from the signature, each of the form List[Tuple[str, ...]]. """ + signature = re.sub(r'\s+', '', signature) + if not re.match(_SIGNATURE, signature): raise ValueError( 'not a valid gufunc signature: {}'.format(signature)) @@ -1889,21 +2098,29 @@ def _calculate_shapes(broadcast_shape, dim_sizes, list_of_core_dims): for core_dims in list_of_core_dims] -def _create_arrays(broadcast_shape, dim_sizes, list_of_core_dims, dtypes): +def _create_arrays(broadcast_shape, dim_sizes, list_of_core_dims, dtypes, + results=None): """Helper for creating output arrays in vectorize.""" shapes = _calculate_shapes(broadcast_shape, dim_sizes, list_of_core_dims) - arrays = tuple(np.empty(shape, dtype=dtype) - for shape, dtype in zip(shapes, dtypes)) + if dtypes is None: + dtypes = [None] * len(shapes) + if results is None: + arrays = tuple(np.empty(shape=shape, dtype=dtype) + for shape, dtype in zip(shapes, dtypes)) + else: + arrays = tuple(np.empty_like(result, shape=shape, dtype=dtype) + for result, shape, dtype + in zip(results, shapes, dtypes)) return arrays @set_module('numpy') class vectorize: """ - vectorize(pyfunc, otypes=None, doc=None, excluded=None, cache=False, - signature=None) + vectorize(pyfunc=np._NoValue, otypes=None, doc=None, excluded=None, + cache=False, signature=None) - Generalized function class. + Returns an object that acts like pyfunc, but takes arrays as input. Define a vectorized function which takes a nested sequence of objects or numpy arrays as inputs and returns a single numpy array or a tuple of numpy @@ -1917,8 +2134,9 @@ class vectorize: Parameters ---------- - pyfunc : callable + pyfunc : callable, optional A python function or method. + Can be omitted to produce a decorator with keyword arguments. otypes : str or list of dtypes, optional The output data type. It must be specified as either a string of typecode characters or a list of data type specifiers. There should @@ -1934,8 +2152,8 @@ class vectorize: .. versionadded:: 1.7.0 cache : bool, optional - If `True`, then cache the first function call that determines the number - of outputs if `otypes` is not provided. + If `True`, then cache the first function call that determines the number + of outputs if `otypes` is not provided. .. versionadded:: 1.7.0 @@ -1950,8 +2168,9 @@ class vectorize: Returns ------- - vectorized : callable - Vectorized function. + out : callable + A vectorized function if ``pyfunc`` was provided, + a decorator otherwise. See Also -------- @@ -2048,18 +2267,44 @@ class vectorize: [0., 0., 1., 2., 1., 0.], [0., 0., 0., 1., 2., 1.]]) + Decorator syntax is supported. The decorator can be called as + a function to provide keyword arguments. + >>>@np.vectorize + ...def identity(x): + ... return x + ... + >>>identity([0, 1, 2]) + array([0, 1, 2]) + >>>@np.vectorize(otypes=[float]) + ...def as_float(x): + ... return x + ... + >>>as_float([0, 1, 2]) + array([0., 1., 2.]) """ - def __init__(self, pyfunc, otypes=None, doc=None, excluded=None, - cache=False, signature=None): + def __init__(self, pyfunc=np._NoValue, otypes=None, doc=None, + excluded=None, cache=False, signature=None): + + if (pyfunc != np._NoValue) and (not callable(pyfunc)): + #Splitting the error message to keep + #the length below 79 characters. + part1 = "When used as a decorator, " + part2 = "only accepts keyword arguments." + raise TypeError(part1 + part2) + self.pyfunc = pyfunc self.cache = cache self.signature = signature - self._ufunc = {} # Caching to improve default performance + if pyfunc != np._NoValue and hasattr(pyfunc, '__name__'): + self.__name__ = pyfunc.__name__ - if doc is None: + self._ufunc = {} # Caching to improve default performance + self._doc = None + self.__doc__ = doc + if doc is None and hasattr(pyfunc, '__doc__'): self.__doc__ = pyfunc.__doc__ else: - self.__doc__ = doc + self._doc = doc if isinstance(otypes, str): for char in otypes: @@ -2081,7 +2326,15 @@ class vectorize: else: self._in_and_out_core_dims = None - def __call__(self, *args, **kwargs): + def _init_stage_2(self, pyfunc, *args, **kwargs): + self.__name__ = pyfunc.__name__ + self.pyfunc = pyfunc + if self._doc is None: + self.__doc__ = pyfunc.__doc__ + else: + self.__doc__ = self._doc + + def _call_as_normal(self, *args, **kwargs): """ Return arrays with the results of `pyfunc` broadcast (vectorized) over `args` and `kwargs` not in `excluded`. @@ -2111,6 +2364,13 @@ class vectorize: return self._vectorize_call(func=func, args=vargs) + def __call__(self, *args, **kwargs): + if self.pyfunc is np._NoValue: + self._init_stage_2(*args, **kwargs) + return self + + return self._call_as_normal(*args, **kwargs) + def _get_ufunc_and_otypes(self, func, args): """Return (ufunc, otypes).""" # frompyfunc will fail if args is empty @@ -2190,15 +2450,14 @@ class vectorize: ufunc, otypes = self._get_ufunc_and_otypes(func=func, args=args) # Convert args to object arrays first - inputs = [array(a, copy=False, subok=True, dtype=object) - for a in args] + inputs = [asanyarray(a, dtype=object) for a in args] outputs = ufunc(*inputs) if ufunc.nout == 1: - res = array(outputs, copy=False, subok=True, dtype=otypes[0]) + res = asanyarray(outputs, dtype=otypes[0]) else: - res = tuple([array(x, copy=False, subok=True, dtype=t) + res = tuple([asanyarray(x, dtype=t) for x, t in zip(outputs, otypes)]) return res @@ -2240,11 +2499,8 @@ class vectorize: for result, core_dims in zip(results, output_core_dims): _update_dim_sizes(dim_sizes, result, core_dims) - if otypes is None: - otypes = [asarray(result).dtype for result in results] - outputs = _create_arrays(broadcast_shape, dim_sizes, - output_core_dims, otypes) + output_core_dims, otypes, results) for output, result in zip(outputs, results): output[index] = result @@ -2267,13 +2523,13 @@ class vectorize: def _cov_dispatcher(m, y=None, rowvar=None, bias=None, ddof=None, - fweights=None, aweights=None): + fweights=None, aweights=None, *, dtype=None): return (m, y, fweights, aweights) @array_function_dispatch(_cov_dispatcher) def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, - aweights=None): + aweights=None, *, dtype=None): """ Estimate a covariance matrix, given data and weights. @@ -2324,6 +2580,11 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, weights can be used to assign probabilities to observation vectors. .. versionadded:: 1.10 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2399,13 +2660,16 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, if m.ndim > 2: raise ValueError("m has more than 2 dimensions") - if y is None: - dtype = np.result_type(m, np.float64) - else: + if y is not None: y = np.asarray(y) if y.ndim > 2: raise ValueError("y has more than 2 dimensions") - dtype = np.result_type(m, y, np.float64) + + if dtype is None: + if y is None: + dtype = np.result_type(m, np.float64) + else: + dtype = np.result_type(m, y, np.float64) X = array(m, ndmin=2, dtype=dtype) if not rowvar and X.shape[0] != 1: @@ -2472,7 +2736,7 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, if fact <= 0: warnings.warn("Degrees of freedom <= 0 for slice", - RuntimeWarning, stacklevel=3) + RuntimeWarning, stacklevel=2) fact = 0.0 X -= avg[:, None] @@ -2485,12 +2749,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, return c.squeeze() -def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None): +def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None, *, + dtype=None): return (x, y) @array_function_dispatch(_corrcoef_dispatcher) -def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): +def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue, *, + dtype=None): """ Return Pearson product-moment correlation coefficients. @@ -2498,7 +2764,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): relationship between the correlation coefficient matrix, `R`, and the covariance matrix, `C`, is - .. math:: R_{ij} = \\frac{ C_{ij} } { \\sqrt{ C_{ii} * C_{jj} } } + .. math:: R_{ij} = \\frac{ C_{ij} } { \\sqrt{ C_{ii} C_{jj} } } The values of `R` are between -1 and 1, inclusive. @@ -2524,6 +2790,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): Has no effect, do not use. .. deprecated:: 1.10.0 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2546,11 +2817,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): for backwards compatibility with previous versions of this function. These arguments had no effect on the return values of the function and can be safely ignored in this and previous versions of numpy. - + Examples - -------- + -------- In this example we generate two random arrays, ``xarr`` and ``yarr``, and - compute the row-wise and column-wise Pearson correlation coefficients, + compute the row-wise and column-wise Pearson correlation coefficients, ``R``. Since ``rowvar`` is true by default, we first find the row-wise Pearson correlation coefficients between the variables of ``xarr``. @@ -2566,11 +2837,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): array([[ 1. , 0.99256089, -0.68080986], [ 0.99256089, 1. , -0.76492172], [-0.68080986, -0.76492172, 1. ]]) - - If we add another set of variables and observations ``yarr``, we can + + If we add another set of variables and observations ``yarr``, we can compute the row-wise Pearson correlation coefficients between the variables in ``xarr`` and ``yarr``. - + >>> yarr = rng.random((3, 3)) >>> yarr array([[0.45038594, 0.37079802, 0.92676499], @@ -2592,7 +2863,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): 1. ]]) Finally if we use the option ``rowvar=False``, the columns are now - being treated as the variables and we will find the column-wise Pearson + being treated as the variables and we will find the column-wise Pearson correlation coefficients between variables in ``xarr`` and ``yarr``. >>> R3 = np.corrcoef(xarr, yarr, rowvar=False) @@ -2614,8 +2885,8 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): if bias is not np._NoValue or ddof is not np._NoValue: # 2015-03-15, 1.10 warnings.warn('bias and ddof have no effect and are deprecated', - DeprecationWarning, stacklevel=3) - c = cov(x, y, rowvar) + DeprecationWarning, stacklevel=2) + c = cov(x, y, rowvar, dtype=dtype) try: d = diag(c) except ValueError: @@ -2729,11 +3000,11 @@ def blackman(M): """ if M < 1: - return array([]) + return array([], dtype=np.result_type(M, 0.0)) if M == 1: - return ones(1, float) - n = arange(0, M) - return 0.42 - 0.5*cos(2.0*pi*n/(M-1)) + 0.08*cos(4.0*pi*n/(M-1)) + return ones(1, dtype=np.result_type(M, 0.0)) + n = arange(1-M, M, 2) + return 0.42 + 0.5*cos(pi*n/(M-1)) + 0.08*cos(2.0*pi*n/(M-1)) @set_module('numpy') @@ -2771,15 +3042,14 @@ def bartlett(M): \\frac{M-1}{2} - \\left|n - \\frac{M-1}{2}\\right| \\right) - Most references to the Bartlett window come from the signal - processing literature, where it is used as one of many windowing - functions for smoothing values. Note that convolution with this - window produces linear interpolation. It is also known as an - apodization (which means"removing the foot", i.e. smoothing - discontinuities at the beginning and end of the sampled signal) or - tapering function. The fourier transform of the Bartlett is the product - of two sinc functions. - Note the excellent discussion in Kanasewich. + Most references to the Bartlett window come from the signal processing + literature, where it is used as one of many windowing functions for + smoothing values. Note that convolution with this window produces linear + interpolation. It is also known as an apodization (which means "removing + the foot", i.e. smoothing discontinuities at the beginning and end of the + sampled signal) or tapering function. The Fourier transform of the + Bartlett window is the product of two sinc functions. Note the excellent + discussion in Kanasewich [2]_. References ---------- @@ -2838,11 +3108,11 @@ def bartlett(M): """ if M < 1: - return array([]) + return array([], dtype=np.result_type(M, 0.0)) if M == 1: - return ones(1, float) - n = arange(0, M) - return where(less_equal(n, (M-1)/2.0), 2.0*n/(M-1), 2.0 - 2.0*n/(M-1)) + return ones(1, dtype=np.result_type(M, 0.0)) + n = arange(1-M, M, 2) + return where(less_equal(n, 0), 1 + n/(M-1), 1 - n/(M-1)) @set_module('numpy') @@ -2872,7 +3142,7 @@ def hanning(M): ----- The Hanning window is defined as - .. math:: w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right) + .. math:: w(n) = 0.5 - 0.5\\cos\\left(\\frac{2\\pi{n}}{M-1}\\right) \\qquad 0 \\leq n \\leq M-1 The Hanning was named for Julius von Hann, an Austrian meteorologist. @@ -2942,11 +3212,11 @@ def hanning(M): """ if M < 1: - return array([]) + return array([], dtype=np.result_type(M, 0.0)) if M == 1: - return ones(1, float) - n = arange(0, M) - return 0.5 - 0.5*cos(2.0*pi*n/(M-1)) + return ones(1, dtype=np.result_type(M, 0.0)) + n = arange(1-M, M, 2) + return 0.5 + 0.5*cos(pi*n/(M-1)) @set_module('numpy') @@ -2976,7 +3246,7 @@ def hamming(M): ----- The Hamming window is defined as - .. math:: w(n) = 0.54 - 0.46cos\\left(\\frac{2\\pi{n}}{M-1}\\right) + .. math:: w(n) = 0.54 - 0.46\\cos\\left(\\frac{2\\pi{n}}{M-1}\\right) \\qquad 0 \\leq n \\leq M-1 The Hamming was named for R. W. Hamming, an associate of J. W. Tukey @@ -3042,11 +3312,11 @@ def hamming(M): """ if M < 1: - return array([]) + return array([], dtype=np.result_type(M, 0.0)) if M == 1: - return ones(1, float) - n = arange(0, M) - return 0.54 - 0.46*cos(2.0*pi*n/(M-1)) + return ones(1, dtype=np.result_type(M, 0.0)) + n = arange(1-M, M, 2) + return 0.54 + 0.46*cos(pi*n/(M-1)) ## Code from cephes for i0 @@ -3142,25 +3412,18 @@ def i0(x): """ Modified Bessel function of the first kind, order 0. - Usually denoted :math:`I_0`. This function does broadcast, but will *not* - "up-cast" int dtype arguments unless accompanied by at least one float or - complex dtype argument (see Raises below). + Usually denoted :math:`I_0`. Parameters ---------- - x : array_like, dtype float or complex + x : array_like of float Argument of the Bessel function. Returns ------- - out : ndarray, shape = x.shape, dtype = x.dtype + out : ndarray, shape = x.shape, dtype = float The modified Bessel function evaluated at each of the elements of `x`. - Raises - ------ - TypeError: array cannot be safely cast to required type - If argument consists exclusively of int dtypes. - See Also -------- scipy.special.i0, scipy.special.iv, scipy.special.ive @@ -3184,18 +3447,22 @@ def i0(x): Her Majesty's Stationery Office, 1962. .. [2] M. Abramowitz and I. A. Stegun, *Handbook of Mathematical Functions*, 10th printing, New York: Dover, 1964, pp. 379. - http://www.math.sfu.ca/~cbm/aands/page_379.htm + https://personal.math.ubc.ca/~cbm/aands/page_379.htm .. [3] https://metacpan.org/pod/distribution/Math-Cephes/lib/Math/Cephes.pod#i0:-Modified-Bessel-function-of-order-zero Examples -------- >>> np.i0(0.) - array(1.0) # may vary - >>> np.i0([0., 1. + 2j]) - array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary + array(1.0) + >>> np.i0([0, 1, 2, 3]) + array([1. , 1.26606588, 2.2795853 , 4.88079259]) """ x = np.asanyarray(x) + if x.dtype.kind == 'c': + raise TypeError("i0 not supported for complex values") + if x.dtype.kind != 'f': + x = x.astype(float) x = np.abs(x) return piecewise(x, [x <= 8.0], [_i0_1, _i0_2]) @@ -3324,7 +3591,7 @@ def kaiser(M, beta): """ if M == 1: - return np.array([1.]) + return np.ones(1, dtype=np.result_type(M, 0.0)) n = arange(0, M) alpha = (M-1)/2.0 return i0(beta * sqrt(1-((n-alpha)/alpha)**2.0))/i0(float(beta)) @@ -3339,20 +3606,22 @@ def sinc(x): r""" Return the normalized sinc function. - The sinc function is :math:`\sin(\pi x)/(\pi x)`. + The sinc function is equal to :math:`\sin(\pi x)/(\pi x)` for any argument + :math:`x\ne 0`. ``sinc(0)`` takes the limit value 1, making ``sinc`` not + only everywhere continuous but also infinitely differentiable. .. note:: Note the normalization factor of ``pi`` used in the definition. This is the most commonly used definition in signal processing. Use ``sinc(x / np.pi)`` to obtain the unnormalized sinc function - :math:`\sin(x)/(x)` that is more common in mathematics. + :math:`\sin(x)/x` that is more common in mathematics. Parameters ---------- x : ndarray - Array (possibly multi-dimensional) of values for which to to - calculate ``sinc(x)``. + Array (possibly multi-dimensional) of values for which to calculate + ``sinc(x)``. Returns ------- @@ -3361,8 +3630,6 @@ def sinc(x): Notes ----- - ``sinc(0)`` is the limit value 1. - The name sinc is short for "sine cardinal" or "sinus cardinalis". The sinc function is used in various signal processing applications, @@ -3424,6 +3691,10 @@ def msort(a): """ Return a copy of an array sorted along the first axis. + .. deprecated:: 1.24 + + msort is deprecated, use ``np.sort(a, axis=0)`` instead. + Parameters ---------- a : array_like @@ -3442,13 +3713,26 @@ def msort(a): ----- ``np.msort(a)`` is equivalent to ``np.sort(a, axis=0)``. + Examples + -------- + >>> a = np.array([[1, 4], [3, 1]]) + >>> np.msort(a) # sort along the first axis + array([[1, 1], + [3, 4]]) + """ + # 2022-10-20 1.24 + warnings.warn( + "msort is deprecated, use np.sort(a, axis=0) instead", + DeprecationWarning, + stacklevel=2, + ) b = array(a, subok=True, copy=True) b.sort(0) return b -def _ureduce(a, func, **kwargs): +def _ureduce(a, func, keepdims=False, **kwargs): """ Internal Function. Call `func` with `a` as first argument swapping the axes to use extended @@ -3476,13 +3760,20 @@ def _ureduce(a, func, **kwargs): """ a = np.asanyarray(a) axis = kwargs.get('axis', None) + out = kwargs.get('out', None) + + if keepdims is np._NoValue: + keepdims = False + + nd = a.ndim if axis is not None: - keepdim = list(a.shape) - nd = a.ndim axis = _nx.normalize_axis_tuple(axis, nd) - for ax in axis: - keepdim[ax] = 1 + if keepdims: + if out is not None: + index_out = tuple( + 0 if i in axis else slice(None) for i in range(nd)) + kwargs['out'] = out[(Ellipsis, ) + index_out] if len(axis) == 1: kwargs['axis'] = axis[0] @@ -3495,12 +3786,27 @@ def _ureduce(a, func, **kwargs): # merge reduced axis a = a.reshape(a.shape[:nkeep] + (-1,)) kwargs['axis'] = -1 - keepdim = tuple(keepdim) else: - keepdim = (1,) * a.ndim + if keepdims: + if out is not None: + index_out = (0, ) * nd + kwargs['out'] = out[(Ellipsis, ) + index_out] r = func(a, **kwargs) - return r, keepdim + + if out is not None: + return out + + if keepdims: + if axis is None: + index_r = (np.newaxis, ) * nd + else: + index_r = tuple( + np.newaxis if i in axis else slice(None) + for i in range(nd)) + r = r[(Ellipsis, ) + index_r] + + return r def _median_dispatcher( @@ -3590,12 +3896,8 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False): >>> assert not np.all(a==b) """ - r, k = _ureduce(a, func=_median, axis=axis, out=out, + return _ureduce(a, func=_median, keepdims=keepdims, axis=axis, out=out, overwrite_input=overwrite_input) - if keepdims: - return r.reshape(k) - else: - return r def _median(a, axis=None, out=None, overwrite_input=False): @@ -3642,26 +3944,32 @@ def _median(a, axis=None, out=None, overwrite_input=False): indexer[axis] = slice(index-1, index+1) indexer = tuple(indexer) + # Use mean in both odd and even case to coerce data type, + # using out array if needed. + rout = mean(part[indexer], axis=axis, out=out) # Check if the array contains any nan's if np.issubdtype(a.dtype, np.inexact) and sz > 0: - # warn and return nans like mean would - rout = mean(part[indexer], axis=axis, out=out) - return np.lib.utils._median_nancheck(part, rout, axis, out) - else: - # if there are no nans - # Use mean in odd and even case to coerce data type - # and check, use out array. - return mean(part[indexer], axis=axis, out=out) + # If nans are possible, warn and replace by nans like mean would. + rout = np.lib.utils._median_nancheck(part, rout, axis) + + return rout def _percentile_dispatcher(a, q, axis=None, out=None, overwrite_input=None, - interpolation=None, keepdims=None): + method=None, keepdims=None, *, interpolation=None): return (a, q, out) @array_function_dispatch(_percentile_dispatcher) -def percentile(a, q, axis=None, out=None, - overwrite_input=False, interpolation='linear', keepdims=False): +def percentile(a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=False, + *, + interpolation=None): """ Compute the q-th percentile of the data along the specified axis. @@ -3669,11 +3977,11 @@ def percentile(a, q, axis=None, out=None, Parameters ---------- - a : array_like + a : array_like of real numbers Input array or object that can be converted to an array. q : array_like of float - Percentile or sequence of percentiles to compute, which must be between - 0 and 100 inclusive. + Percentage or sequence of percentages for the percentiles to compute. + Values must be between 0 and 100 inclusive. axis : {int, tuple of int, None}, optional Axis or axes along which the percentiles are computed. The default is to compute the percentile(s) along a flattened @@ -3689,21 +3997,34 @@ def percentile(a, q, axis=None, out=None, If True, then allow the input array `a` to be modified by intermediate calculations, to save memory. In this case, the contents of the input `a` after this function completes is undefined. + method : str, optional + This parameter specifies the method to use for estimating the + percentile. There are many different methods, some unique to NumPy. + See the notes for explanation. The options sorted by their R type + as summarized in the H&F paper [1]_ are: + + 1. 'inverted_cdf' + 2. 'averaged_inverted_cdf' + 3. 'closest_observation' + 4. 'interpolated_inverted_cdf' + 5. 'hazen' + 6. 'weibull' + 7. 'linear' (default) + 8. 'median_unbiased' + 9. 'normal_unbiased' + + The first three methods are discontinuous. NumPy further defines the + following discontinuous variations of the default 'linear' (7.) option: + + * 'lower' + * 'higher', + * 'midpoint' + * 'nearest' + + .. versionchanged:: 1.22.0 + This argument was previously called "interpolation" and only + offered the "linear" default and last four options. - interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'} - This optional parameter specifies the interpolation method to - use when the desired percentile lies between two data points - ``i < j``: - - * 'linear': ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * 'lower': ``i``. - * 'higher': ``j``. - * 'nearest': ``i`` or ``j``, whichever is nearest. - * 'midpoint': ``(i + j) / 2``. - - .. versionadded:: 1.9.0 keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the @@ -3711,6 +4032,11 @@ def percentile(a, q, axis=None, out=None, .. versionadded:: 1.9.0 + interpolation : str, optional + Deprecated name for the method keyword argument. + + .. deprecated:: 1.22.0 + Returns ------- percentile : scalar or ndarray @@ -3728,18 +4054,121 @@ def percentile(a, q, axis=None, out=None, mean median : equivalent to ``percentile(..., 50)`` nanpercentile - quantile : equivalent to percentile, except with q in the range [0, 1]. + quantile : equivalent to percentile, except q in the range [0, 1]. Notes ----- - Given a vector ``V`` of length ``N``, the q-th percentile of - ``V`` is the value ``q/100`` of the way from the minimum to the - maximum in a sorted copy of ``V``. The values and distances of - the two nearest neighbors as well as the `interpolation` parameter - will determine the percentile if the normalized ranking does not - match the location of ``q`` exactly. This function is the same as - the median if ``q=50``, the same as the minimum if ``q=0`` and the - same as the maximum if ``q=100``. + Given a vector ``V`` of length ``n``, the q-th percentile of ``V`` is + the value ``q/100`` of the way from the minimum to the maximum in a + sorted copy of ``V``. The values and distances of the two nearest + neighbors as well as the `method` parameter will determine the + percentile if the normalized ranking does not match the location of + ``q`` exactly. This function is the same as the median if ``q=50``, the + same as the minimum if ``q=0`` and the same as the maximum if + ``q=100``. + + The optional `method` parameter specifies the method to use when the + desired percentile lies between two indexes ``i`` and ``j = i + 1``. + In that case, we first determine ``i + g``, a virtual index that lies + between ``i`` and ``j``, where ``i`` is the floor and ``g`` is the + fractional part of the index. The final result is, then, an interpolation + of ``a[i]`` and ``a[j]`` based on ``g``. During the computation of ``g``, + ``i`` and ``j`` are modified using correction constants ``alpha`` and + ``beta`` whose choices depend on the ``method`` used. Finally, note that + since Python uses 0-based indexing, the code subtracts another 1 from the + index internally. + + The following formula determines the virtual index ``i + g``, the location + of the percentile in the sorted sample: + + .. math:: + i + g = (q / 100) * ( n - alpha - beta + 1 ) + alpha + + The different methods then work as follows + + inverted_cdf: + method 1 of H&F [1]_. + This method gives discontinuous results: + + * if g > 0 ; then take j + * if g = 0 ; then take i + + averaged_inverted_cdf: + method 2 of H&F [1]_. + This method give discontinuous results: + + * if g > 0 ; then take j + * if g = 0 ; then average between bounds + + closest_observation: + method 3 of H&F [1]_. + This method give discontinuous results: + + * if g > 0 ; then take j + * if g = 0 and index is odd ; then take j + * if g = 0 and index is even ; then take i + + interpolated_inverted_cdf: + method 4 of H&F [1]_. + This method give continuous results using: + + * alpha = 0 + * beta = 1 + + hazen: + method 5 of H&F [1]_. + This method give continuous results using: + + * alpha = 1/2 + * beta = 1/2 + + weibull: + method 6 of H&F [1]_. + This method give continuous results using: + + * alpha = 0 + * beta = 0 + + linear: + method 7 of H&F [1]_. + This method give continuous results using: + + * alpha = 1 + * beta = 1 + + median_unbiased: + method 8 of H&F [1]_. + This method is probably the best method if the sample + distribution function is unknown (see reference). + This method give continuous results using: + + * alpha = 1/3 + * beta = 1/3 + + normal_unbiased: + method 9 of H&F [1]_. + This method is probably the best method if the sample + distribution function is known to be normal. + This method give continuous results using: + + * alpha = 3/8 + * beta = 3/8 + + lower: + NumPy method kept for backwards compatibility. + Takes ``i`` as the interpolation point. + + higher: + NumPy method kept for backwards compatibility. + Takes ``j`` as the interpolation point. + + nearest: + NumPy method kept for backwards compatibility. + Takes ``i`` or ``j``, whichever is nearest. + + midpoint: + NumPy method kept for backwards compatibility. + Uses ``(i + j) / 2``. Examples -------- @@ -3769,7 +4198,7 @@ def percentile(a, q, axis=None, out=None, array([7., 2.]) >>> assert not np.all(a == b) - The different types of interpolation can be visualized graphically: + The different methods can be visualized graphically: .. plot:: @@ -3779,41 +4208,68 @@ def percentile(a, q, axis=None, out=None, p = np.linspace(0, 100, 6001) ax = plt.gca() lines = [ - ('linear', None), - ('higher', '--'), - ('lower', '--'), - ('nearest', '-.'), - ('midpoint', '-.'), - ] - for interpolation, style in lines: + ('linear', '-', 'C0'), + ('inverted_cdf', ':', 'C1'), + # Almost the same as `inverted_cdf`: + ('averaged_inverted_cdf', '-.', 'C1'), + ('closest_observation', ':', 'C2'), + ('interpolated_inverted_cdf', '--', 'C1'), + ('hazen', '--', 'C3'), + ('weibull', '-.', 'C4'), + ('median_unbiased', '--', 'C5'), + ('normal_unbiased', '-.', 'C6'), + ] + for method, style, color in lines: ax.plot( - p, np.percentile(a, p, interpolation=interpolation), - label=interpolation, linestyle=style) + p, np.percentile(a, p, method=method), + label=method, linestyle=style, color=color) ax.set( - title='Interpolation methods for list: ' + str(a), + title='Percentiles for different methods and data: ' + str(a), xlabel='Percentile', - ylabel='List item returned', + ylabel='Estimated percentile value', yticks=a) - ax.legend() + ax.legend(bbox_to_anchor=(1.03, 1)) + plt.tight_layout() plt.show() + References + ---------- + .. [1] R. J. Hyndman and Y. Fan, + "Sample quantiles in statistical packages," + The American Statistician, 50(4), pp. 361-365, 1996 + """ + if interpolation is not None: + method = _check_interpolation_as_method( + method, interpolation, "percentile") + + a = np.asanyarray(a) + if a.dtype.kind == "c": + raise TypeError("a must be an array of real numbers") + q = np.true_divide(q, 100) q = asanyarray(q) # undo any decay that the ufunc performed (see gh-13105) if not _quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") return _quantile_unchecked( - a, q, axis, out, overwrite_input, interpolation, keepdims) + a, q, axis, out, overwrite_input, method, keepdims) def _quantile_dispatcher(a, q, axis=None, out=None, overwrite_input=None, - interpolation=None, keepdims=None): + method=None, keepdims=None, *, interpolation=None): return (a, q, out) @array_function_dispatch(_quantile_dispatcher) -def quantile(a, q, axis=None, out=None, - overwrite_input=False, interpolation='linear', keepdims=False): +def quantile(a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=False, + *, + interpolation=None): """ Compute the q-th quantile of the data along the specified axis. @@ -3821,45 +4277,66 @@ def quantile(a, q, axis=None, out=None, Parameters ---------- - a : array_like + a : array_like of real numbers Input array or object that can be converted to an array. q : array_like of float - Quantile or sequence of quantiles to compute, which must be between - 0 and 1 inclusive. + Probability or sequence of probabilities for the quantiles to compute. + Values must be between 0 and 1 inclusive. axis : {int, tuple of int, None}, optional - Axis or axes along which the quantiles are computed. The - default is to compute the quantile(s) along a flattened - version of the array. + Axis or axes along which the quantiles are computed. The default is + to compute the quantile(s) along a flattened version of the array. out : ndarray, optional - Alternative output array in which to place the result. It must - have the same shape and buffer length as the expected output, - but the type (of the output) will be cast if necessary. + Alternative output array in which to place the result. It must have + the same shape and buffer length as the expected output, but the + type (of the output) will be cast if necessary. overwrite_input : bool, optional - If True, then allow the input array `a` to be modified by intermediate - calculations, to save memory. In this case, the contents of the input - `a` after this function completes is undefined. - interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'} - This optional parameter specifies the interpolation method to - use when the desired quantile lies between two data points - ``i < j``: - - * linear: ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * lower: ``i``. - * higher: ``j``. - * nearest: ``i`` or ``j``, whichever is nearest. - * midpoint: ``(i + j) / 2``. + If True, then allow the input array `a` to be modified by + intermediate calculations, to save memory. In this case, the + contents of the input `a` after this function completes is + undefined. + method : str, optional + This parameter specifies the method to use for estimating the + quantile. There are many different methods, some unique to NumPy. + See the notes for explanation. The options sorted by their R type + as summarized in the H&F paper [1]_ are: + + 1. 'inverted_cdf' + 2. 'averaged_inverted_cdf' + 3. 'closest_observation' + 4. 'interpolated_inverted_cdf' + 5. 'hazen' + 6. 'weibull' + 7. 'linear' (default) + 8. 'median_unbiased' + 9. 'normal_unbiased' + + The first three methods are discontinuous. NumPy further defines the + following discontinuous variations of the default 'linear' (7.) option: + + * 'lower' + * 'higher', + * 'midpoint' + * 'nearest' + + .. versionchanged:: 1.22.0 + This argument was previously called "interpolation" and only + offered the "linear" default and last four options. + keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original array `a`. + interpolation : str, optional + Deprecated name for the method keyword argument. + + .. deprecated:: 1.22.0 + Returns ------- quantile : scalar or ndarray - If `q` is a single quantile and `axis=None`, then the result - is a scalar. If multiple quantiles are given, first axis of + If `q` is a single probability and `axis=None`, then the result + is a scalar. If multiple probabilies levels are given, first axis of the result corresponds to the quantiles. The other axes are the axes that remain after the reduction of `a`. If the input contains integers or floats smaller than ``float64``, the output @@ -3876,14 +4353,117 @@ def quantile(a, q, axis=None, out=None, Notes ----- - Given a vector ``V`` of length ``N``, the q-th quantile of - ``V`` is the value ``q`` of the way from the minimum to the - maximum in a sorted copy of ``V``. The values and distances of - the two nearest neighbors as well as the `interpolation` parameter - will determine the quantile if the normalized ranking does not - match the location of ``q`` exactly. This function is the same as - the median if ``q=0.5``, the same as the minimum if ``q=0.0`` and the - same as the maximum if ``q=1.0``. + Given a vector ``V`` of length ``n``, the q-th quantile of ``V`` is + the value ``q`` of the way from the minimum to the maximum in a + sorted copy of ``V``. The values and distances of the two nearest + neighbors as well as the `method` parameter will determine the + quantile if the normalized ranking does not match the location of + ``q`` exactly. This function is the same as the median if ``q=0.5``, the + same as the minimum if ``q=0.0`` and the same as the maximum if + ``q=1.0``. + + The optional `method` parameter specifies the method to use when the + desired quantile lies between two indexes ``i`` and ``j = i + 1``. + In that case, we first determine ``i + g``, a virtual index that lies + between ``i`` and ``j``, where ``i`` is the floor and ``g`` is the + fractional part of the index. The final result is, then, an interpolation + of ``a[i]`` and ``a[j]`` based on ``g``. During the computation of ``g``, + ``i`` and ``j`` are modified using correction constants ``alpha`` and + ``beta`` whose choices depend on the ``method`` used. Finally, note that + since Python uses 0-based indexing, the code subtracts another 1 from the + index internally. + + The following formula determines the virtual index ``i + g``, the location + of the quantile in the sorted sample: + + .. math:: + i + g = q * ( n - alpha - beta + 1 ) + alpha + + The different methods then work as follows + + inverted_cdf: + method 1 of H&F [1]_. + This method gives discontinuous results: + + * if g > 0 ; then take j + * if g = 0 ; then take i + + averaged_inverted_cdf: + method 2 of H&F [1]_. + This method gives discontinuous results: + + * if g > 0 ; then take j + * if g = 0 ; then average between bounds + + closest_observation: + method 3 of H&F [1]_. + This method gives discontinuous results: + + * if g > 0 ; then take j + * if g = 0 and index is odd ; then take j + * if g = 0 and index is even ; then take i + + interpolated_inverted_cdf: + method 4 of H&F [1]_. + This method gives continuous results using: + + * alpha = 0 + * beta = 1 + + hazen: + method 5 of H&F [1]_. + This method gives continuous results using: + + * alpha = 1/2 + * beta = 1/2 + + weibull: + method 6 of H&F [1]_. + This method gives continuous results using: + + * alpha = 0 + * beta = 0 + + linear: + method 7 of H&F [1]_. + This method gives continuous results using: + + * alpha = 1 + * beta = 1 + + median_unbiased: + method 8 of H&F [1]_. + This method is probably the best method if the sample + distribution function is unknown (see reference). + This method gives continuous results using: + + * alpha = 1/3 + * beta = 1/3 + + normal_unbiased: + method 9 of H&F [1]_. + This method is probably the best method if the sample + distribution function is known to be normal. + This method gives continuous results using: + + * alpha = 3/8 + * beta = 3/8 + + lower: + NumPy method kept for backwards compatibility. + Takes ``i`` as the interpolation point. + + higher: + NumPy method kept for backwards compatibility. + Takes ``j`` as the interpolation point. + + nearest: + NumPy method kept for backwards compatibility. + Takes ``i`` or ``j``, whichever is nearest. + + midpoint: + NumPy method kept for backwards compatibility. + Uses ``(i + j) / 2``. Examples -------- @@ -3910,155 +4490,333 @@ def quantile(a, q, axis=None, out=None, >>> np.quantile(b, 0.5, axis=1, overwrite_input=True) array([7., 2.]) >>> assert not np.all(a == b) + + See also `numpy.percentile` for a visualization of most methods. + + References + ---------- + .. [1] R. J. Hyndman and Y. Fan, + "Sample quantiles in statistical packages," + The American Statistician, 50(4), pp. 361-365, 1996 + """ + if interpolation is not None: + method = _check_interpolation_as_method( + method, interpolation, "quantile") + + a = np.asanyarray(a) + if a.dtype.kind == "c": + raise TypeError("a must be an array of real numbers") + q = np.asanyarray(q) if not _quantile_is_valid(q): raise ValueError("Quantiles must be in the range [0, 1]") return _quantile_unchecked( - a, q, axis, out, overwrite_input, interpolation, keepdims) + a, q, axis, out, overwrite_input, method, keepdims) -def _quantile_unchecked(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear', keepdims=False): +def _quantile_unchecked(a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=False): """Assumes that q is in [0, 1], and is an ndarray""" - r, k = _ureduce(a, func=_quantile_ureduce_func, q=q, axis=axis, out=out, + return _ureduce(a, + func=_quantile_ureduce_func, + q=q, + keepdims=keepdims, + axis=axis, + out=out, overwrite_input=overwrite_input, - interpolation=interpolation) - if keepdims: - return r.reshape(q.shape + k) - else: - return r + method=method) def _quantile_is_valid(q): # avoid expensive reductions, relevant for arrays with < O(1000) elements if q.ndim == 1 and q.size < 10: for i in range(q.size): - if q[i] < 0.0 or q[i] > 1.0: + if not (0.0 <= q[i] <= 1.0): return False else: - # faster than any() - if np.count_nonzero(q < 0.0) or np.count_nonzero(q > 1.0): + if not (np.all(0 <= q) and np.all(q <= 1)): return False return True +def _check_interpolation_as_method(method, interpolation, fname): + # Deprecated NumPy 1.22, 2021-11-08 + warnings.warn( + f"the `interpolation=` argument to {fname} was renamed to " + "`method=`, which has additional options.\n" + "Users of the modes 'nearest', 'lower', 'higher', or " + "'midpoint' are encouraged to review the method they used. " + "(Deprecated NumPy 1.22)", + DeprecationWarning, stacklevel=4) + if method != "linear": + # sanity check, we assume this basically never happens + raise TypeError( + "You shall not pass both `method` and `interpolation`!\n" + "(`interpolation` is Deprecated in favor of `method`)") + return interpolation + + +def _compute_virtual_index(n, quantiles, alpha: float, beta: float): + """ + Compute the floating point indexes of an array for the linear + interpolation of quantiles. + n : array_like + The sample sizes. + quantiles : array_like + The quantiles values. + alpha : float + A constant used to correct the index computed. + beta : float + A constant used to correct the index computed. + + alpha and beta values depend on the chosen method + (see quantile documentation) + + Reference: + Hyndman&Fan paper "Sample Quantiles in Statistical Packages", + DOI: 10.1080/00031305.1996.10473566 + """ + return n * quantiles + ( + alpha + quantiles * (1 - alpha - beta) + ) - 1 + + +def _get_gamma(virtual_indexes, previous_indexes, method): + """ + Compute gamma (a.k.a 'm' or 'weight') for the linear interpolation + of quantiles. + + virtual_indexes : array_like + The indexes where the percentile is supposed to be found in the sorted + sample. + previous_indexes : array_like + The floor values of virtual_indexes. + interpolation : dict + The interpolation method chosen, which may have a specific rule + modifying gamma. + + gamma is usually the fractional part of virtual_indexes but can be modified + by the interpolation method. + """ + gamma = np.asanyarray(virtual_indexes - previous_indexes) + gamma = method["fix_gamma"](gamma, virtual_indexes) + return np.asanyarray(gamma) + + def _lerp(a, b, t, out=None): - """ Linearly interpolate from a to b by a factor of t """ + """ + Compute the linear interpolation weighted by gamma on each point of + two same shape array. + + a : array_like + Left bound. + b : array_like + Right bound. + t : array_like + The interpolation weight. + out : array_like + Output array. + """ diff_b_a = subtract(b, a) # asanyarray is a stop-gap until gh-13105 - lerp_interpolation = asanyarray(add(a, diff_b_a*t, out=out)) - subtract(b, diff_b_a * (1 - t), out=lerp_interpolation, where=t>=0.5) + lerp_interpolation = asanyarray(add(a, diff_b_a * t, out=out)) + subtract(b, diff_b_a * (1 - t), out=lerp_interpolation, where=t >= 0.5) if lerp_interpolation.ndim == 0 and out is None: lerp_interpolation = lerp_interpolation[()] # unpack 0d arrays return lerp_interpolation -def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear', keepdims=False): - a = asarray(a) +def _get_gamma_mask(shape, default_value, conditioned_value, where): + out = np.full(shape, default_value) + np.copyto(out, conditioned_value, where=where, casting="unsafe") + return out - # ufuncs cause 0d array results to decay to scalars (see gh-13105), which - # makes them problematic for __setitem__ and attribute access. As a - # workaround, we call this on the result of every ufunc on a possibly-0d - # array. - not_scalar = np.asanyarray - # prepare a for partitioning - if overwrite_input: - if axis is None: - ap = a.ravel() - else: - ap = a - else: - if axis is None: - ap = a.flatten() - else: - ap = a.copy() +def _discret_interpolation_to_boundaries(index, gamma_condition_fun): + previous = np.floor(index) + next = previous + 1 + gamma = index - previous + res = _get_gamma_mask(shape=index.shape, + default_value=next, + conditioned_value=previous, + where=gamma_condition_fun(gamma, index) + ).astype(np.intp) + # Some methods can lead to out-of-bound integers, clip them: + res[res < 0] = 0 + return res - if axis is None: - axis = 0 +def _closest_observation(n, quantiles): + gamma_fun = lambda gamma, index: (gamma == 0) & (np.floor(index) % 2 == 0) + return _discret_interpolation_to_boundaries((n * quantiles) - 1 - 0.5, + gamma_fun) + + +def _inverted_cdf(n, quantiles): + gamma_fun = lambda gamma, _: (gamma == 0) + return _discret_interpolation_to_boundaries((n * quantiles) - 1, + gamma_fun) + + +def _quantile_ureduce_func( + a: np.array, + q: np.array, + axis: int = None, + out=None, + overwrite_input: bool = False, + method="linear", +) -> np.array: if q.ndim > 2: # The code below works fine for nd, but it might not have useful # semantics. For now, keep the supported dimensions the same as it was # before. raise ValueError("q must be a scalar or 1d") - - Nx = ap.shape[axis] - indices = not_scalar(q * (Nx - 1)) - # round fractional indices according to interpolation method - if interpolation == 'lower': - indices = floor(indices).astype(intp) - elif interpolation == 'higher': - indices = ceil(indices).astype(intp) - elif interpolation == 'midpoint': - indices = 0.5 * (floor(indices) + ceil(indices)) - elif interpolation == 'nearest': - indices = around(indices).astype(intp) - elif interpolation == 'linear': - pass # keep index as fraction and interpolate + if overwrite_input: + if axis is None: + axis = 0 + arr = a.ravel() + else: + arr = a else: - raise ValueError( - "interpolation can only be 'linear', 'lower' 'higher', " - "'midpoint', or 'nearest'") + if axis is None: + axis = 0 + arr = a.flatten() + else: + arr = a.copy() + result = _quantile(arr, + quantiles=q, + axis=axis, + method=method, + out=out) + return result - # The dimensions of `q` are prepended to the output shape, so we need the - # axis being sampled from `ap` to be first. - ap = np.moveaxis(ap, axis, 0) - del axis - if np.issubdtype(indices.dtype, np.integer): - # take the points along axis +def _get_indexes(arr, virtual_indexes, valid_values_count): + """ + Get the valid indexes of arr neighbouring virtual_indexes. + Note + This is a companion function to linear interpolation of + Quantiles - if np.issubdtype(a.dtype, np.inexact): + Returns + ------- + (previous_indexes, next_indexes): Tuple + A Tuple of virtual_indexes neighbouring indexes + """ + previous_indexes = np.asanyarray(np.floor(virtual_indexes)) + next_indexes = np.asanyarray(previous_indexes + 1) + indexes_above_bounds = virtual_indexes >= valid_values_count - 1 + # When indexes is above max index, take the max value of the array + if indexes_above_bounds.any(): + previous_indexes[indexes_above_bounds] = -1 + next_indexes[indexes_above_bounds] = -1 + # When indexes is below min index, take the min value of the array + indexes_below_bounds = virtual_indexes < 0 + if indexes_below_bounds.any(): + previous_indexes[indexes_below_bounds] = 0 + next_indexes[indexes_below_bounds] = 0 + if np.issubdtype(arr.dtype, np.inexact): + # After the sort, slices having NaNs will have for last element a NaN + virtual_indexes_nans = np.isnan(virtual_indexes) + if virtual_indexes_nans.any(): + previous_indexes[virtual_indexes_nans] = -1 + next_indexes[virtual_indexes_nans] = -1 + previous_indexes = previous_indexes.astype(np.intp) + next_indexes = next_indexes.astype(np.intp) + return previous_indexes, next_indexes + + +def _quantile( + arr: np.array, + quantiles: np.array, + axis: int = -1, + method="linear", + out=None, +): + """ + Private function that doesn't support extended axis or keepdims. + These methods are extended to this function using _ureduce + See nanpercentile for parameter usage + It computes the quantiles of the array for the given axis. + A linear interpolation is performed based on the `interpolation`. + + By default, the method is "linear" where alpha == beta == 1 which + performs the 7th method of Hyndman&Fan. + With "median_unbiased" we get alpha == beta == 1/3 + thus the 8th method of Hyndman&Fan. + """ + # --- Setup + arr = np.asanyarray(arr) + values_count = arr.shape[axis] + # The dimensions of `q` are prepended to the output shape, so we need the + # axis being sampled from `arr` to be last. + DATA_AXIS = 0 + if axis != DATA_AXIS: # But moveaxis is slow, so only call it if axis!=0. + arr = np.moveaxis(arr, axis, destination=DATA_AXIS) + # --- Computation of indexes + # Index where to find the value in the sorted array. + # Virtual because it is a floating point value, not an valid index. + # The nearest neighbours are used for interpolation + try: + method = _QuantileMethods[method] + except KeyError: + raise ValueError( + f"{method!r} is not a valid method. Use one of: " + f"{_QuantileMethods.keys()}") from None + virtual_indexes = method["get_virtual_index"](values_count, quantiles) + virtual_indexes = np.asanyarray(virtual_indexes) + if np.issubdtype(virtual_indexes.dtype, np.integer): + # No interpolation needed, take the points along axis + if np.issubdtype(arr.dtype, np.inexact): # may contain nan, which would sort to the end - ap.partition(concatenate((indices.ravel(), [-1])), axis=0) - n = np.isnan(ap[-1]) + arr.partition(concatenate((virtual_indexes.ravel(), [-1])), axis=0) + slices_having_nans = np.isnan(arr[-1]) else: # cannot contain nan - ap.partition(indices.ravel(), axis=0) - n = np.array(False, dtype=bool) - - r = take(ap, indices, axis=0, out=out) - + arr.partition(virtual_indexes.ravel(), axis=0) + slices_having_nans = np.array(False, dtype=bool) + result = take(arr, virtual_indexes, axis=0, out=out) else: - # weight the points above and below the indices - - indices_below = not_scalar(floor(indices)).astype(intp) - indices_above = not_scalar(indices_below + 1) - indices_above[indices_above > Nx - 1] = Nx - 1 - - if np.issubdtype(a.dtype, np.inexact): - # may contain nan, which would sort to the end - ap.partition(concatenate(( - indices_below.ravel(), indices_above.ravel(), [-1] - )), axis=0) - n = np.isnan(ap[-1]) + previous_indexes, next_indexes = _get_indexes(arr, + virtual_indexes, + values_count) + # --- Sorting + arr.partition( + np.unique(np.concatenate(([0, -1], + previous_indexes.ravel(), + next_indexes.ravel(), + ))), + axis=DATA_AXIS) + if np.issubdtype(arr.dtype, np.inexact): + slices_having_nans = np.isnan( + take(arr, indices=-1, axis=DATA_AXIS) + ) else: - # cannot contain nan - ap.partition(concatenate(( - indices_below.ravel(), indices_above.ravel() - )), axis=0) - n = np.array(False, dtype=bool) - - weights_shape = indices.shape + (1,) * (ap.ndim - 1) - weights_above = not_scalar(indices - indices_below).reshape(weights_shape) - - x_below = take(ap, indices_below, axis=0) - x_above = take(ap, indices_above, axis=0) - - r = _lerp(x_below, x_above, weights_above, out=out) - - # if any slice contained a nan, then all results on that slice are also nan - if np.any(n): - if r.ndim == 0 and out is None: + slices_having_nans = None + # --- Get values from indexes + previous = np.take(arr, previous_indexes, axis=DATA_AXIS) + next = np.take(arr, next_indexes, axis=DATA_AXIS) + # --- Linear interpolation + gamma = _get_gamma(virtual_indexes, previous_indexes, method) + result_shape = virtual_indexes.shape + (1,) * (arr.ndim - 1) + gamma = gamma.reshape(result_shape) + result = _lerp(previous, + next, + gamma, + out=out) + if np.any(slices_having_nans): + if result.ndim == 0 and out is None: # can't write to a scalar - r = a.dtype.type(np.nan) + result = arr.dtype.type(np.nan) else: - r[..., n] = a.dtype.type(np.nan) - - return r + result[..., slices_having_nans] = np.nan + return result def _trapz_dispatcher(y, x=None, dx=None, axis=None): @@ -4067,10 +4825,17 @@ def _trapz_dispatcher(y, x=None, dx=None, axis=None): @array_function_dispatch(_trapz_dispatcher) def trapz(y, x=None, dx=1.0, axis=-1): - """ + r""" Integrate along the given axis using the composite trapezoidal rule. - Integrate `y` (`x`) along given axis. + If `x` is provided, the integration happens in sequence along its + elements - they are not sorted. + + Integrate `y` (`x`) along each 1d slice on the given axis, compute + :math:`\int y(x) dx`. + When `x` is specified, this integrates along the parametric curve, + computing :math:`\int_t y(t) dt = + \int_t y(t) \left.\frac{dx}{dt}\right|_{x=x(t)} dt`. Parameters ---------- @@ -4087,8 +4852,11 @@ def trapz(y, x=None, dx=1.0, axis=-1): Returns ------- - trapz : float - Definite integral as approximated by trapezoidal rule. + trapz : float or ndarray + Definite integral of `y` = n-dimensional array as approximated along + a single axis by the trapezoidal rule. If `y` is a 1-dimensional array, + then the result is a float. If `n` is greater than 1, then the result + is an `n`-1 dimensional array. See Also -------- @@ -4112,12 +4880,42 @@ def trapz(y, x=None, dx=1.0, axis=-1): Examples -------- - >>> np.trapz([1,2,3]) + Use the trapezoidal rule on evenly spaced points: + + >>> np.trapz([1, 2, 3]) 4.0 - >>> np.trapz([1,2,3], x=[4,6,8]) + + The spacing between sample points can be selected by either the + ``x`` or ``dx`` arguments: + + >>> np.trapz([1, 2, 3], x=[4, 6, 8]) 8.0 - >>> np.trapz([1,2,3], dx=2) + >>> np.trapz([1, 2, 3], dx=2) 8.0 + + Using a decreasing ``x`` corresponds to integrating in reverse: + + >>> np.trapz([1, 2, 3], x=[8, 6, 4]) + -8.0 + + More generally ``x`` is used to integrate along a parametric curve. We can + estimate the integral :math:`\int_0^1 x^2 = 1/3` using: + + >>> x = np.linspace(0, 1, num=50) + >>> y = x**2 + >>> np.trapz(y, x) + 0.33340274885464394 + + Or estimate the area of a circle, noting we repeat the sample which closes + the curve: + + >>> theta = np.linspace(0, 2 * np.pi, num=1000, endpoint=True) + >>> np.trapz(np.cos(theta), x=np.sin(theta)) + 3.141571941375841 + + ``np.trapz`` can be applied along a specified axis to do multiple + computations in one call: + >>> a = np.arange(6).reshape(2, 3) >>> a array([[0, 1, 2], @@ -4126,7 +4924,6 @@ def trapz(y, x=None, dx=1.0, axis=-1): array([1.5, 2.5, 3.5]) >>> np.trapz(a, axis=1) array([2., 8.]) - """ y = asanyarray(y) if x is None: @@ -4156,6 +4953,24 @@ def trapz(y, x=None, dx=1.0, axis=-1): return ret +# __array_function__ has no __code__ or other attributes normal Python funcs we +# wrap everything into a C callable. SciPy however, tries to "clone" `trapz` +# into a new Python function which requires `__code__` and a few other +# attributes. So we create a dummy clone and copy over its attributes allowing +# SciPy <= 1.10 to work: https://github.com/scipy/scipy/issues/17811 +assert not hasattr(trapz, "__code__") + +def _fake_trapz(y, x=None, dx=1.0, axis=-1): + return trapz(y, x=x, dx=dx, axis=axis) + + +trapz.__code__ = _fake_trapz.__code__ +trapz.__globals__ = _fake_trapz.__globals__ +trapz.__defaults__ = _fake_trapz.__defaults__ +trapz.__closure__ = _fake_trapz.__closure__ +trapz.__kwdefaults__ = _fake_trapz.__kwdefaults__ + + def _meshgrid_dispatcher(*xi, copy=None, sparse=None, indexing=None): return xi @@ -4164,7 +4979,7 @@ def _meshgrid_dispatcher(*xi, copy=None, sparse=None, indexing=None): @array_function_dispatch(_meshgrid_dispatcher) def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): """ - Return coordinate matrices from coordinate vectors. + Return a list of coordinate matrices from coordinate vectors. Make N-D coordinate arrays for vectorized evaluations of N-D scalar/vector fields over N-D grids, given @@ -4183,7 +4998,13 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): .. versionadded:: 1.7.0 sparse : bool, optional - If True a sparse grid is returned in order to conserve memory. + If True the shape of the returned coordinate array for dimension *i* + is reduced from ``(N1, ..., Ni, ... Nn)`` to + ``(1, ..., 1, Ni, 1, ..., 1)``. These sparse coordinate grids are + intended to be use with :ref:`basics.broadcasting`. When all + coordinates are used in an expression, broadcasting still leads to a + fully-dimensonal result array. + Default is False. .. versionadded:: 1.7.0 @@ -4199,10 +5020,10 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): Returns ------- - X1, X2,..., XN : ndarray - For vectors `x1`, `x2`,..., 'xn' with lengths ``Ni=len(xi)`` , - return ``(N1, N2, N3,...Nn)`` shaped arrays if indexing='ij' - or ``(N2, N1, N3,...Nn)`` shaped arrays if indexing='xy' + X1, X2,..., XN : list of ndarrays + For vectors `x1`, `x2`,..., `xn` with lengths ``Ni=len(xi)``, + returns ``(N1, N2, N3,..., Nn)`` shaped arrays if indexing='ij' + or ``(N2, N1, N3,..., Nn)`` shaped arrays if indexing='xy' with the elements of `xi` repeated to fill the matrix along the first dimension for `x1`, the second for `x2` and so on. @@ -4217,12 +5038,12 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): 'xy' indexing and (M, N, P) for 'ij' indexing. The difference is illustrated by the following code snippet:: - xv, yv = np.meshgrid(x, y, sparse=False, indexing='ij') + xv, yv = np.meshgrid(x, y, indexing='ij') for i in range(nx): for j in range(ny): # treat xv[i,j], yv[i,j] - xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + xv, yv = np.meshgrid(x, y, indexing='xy') for i in range(nx): for j in range(ny): # treat xv[j,i], yv[j,i] @@ -4231,10 +5052,10 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): See Also -------- - index_tricks.mgrid : Construct a multi-dimensional "meshgrid" - using indexing notation. - index_tricks.ogrid : Construct an open multi-dimensional "meshgrid" - using indexing notation. + mgrid : Construct a multi-dimensional "meshgrid" using indexing notation. + ogrid : Construct an open multi-dimensional "meshgrid" using indexing + notation. + how-to-index Examples -------- @@ -4248,23 +5069,45 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): >>> yv array([[0., 0., 0.], [1., 1., 1.]]) - >>> xv, yv = np.meshgrid(x, y, sparse=True) # make sparse output arrays + + The result of `meshgrid` is a coordinate grid: + + >>> import matplotlib.pyplot as plt + >>> plt.plot(xv, yv, marker='o', color='k', linestyle='none') + >>> plt.show() + + You can create sparse output arrays to save memory and computation time. + + >>> xv, yv = np.meshgrid(x, y, sparse=True) >>> xv array([[0. , 0.5, 1. ]]) >>> yv array([[0.], [1.]]) - `meshgrid` is very useful to evaluate functions on a grid. + `meshgrid` is very useful to evaluate functions on a grid. If the + function depends on all coordinates, both dense and sparse outputs can be + used. + + >>> x = np.linspace(-5, 5, 101) + >>> y = np.linspace(-5, 5, 101) + >>> # full coordinate arrays + >>> xx, yy = np.meshgrid(x, y) + >>> zz = np.sqrt(xx**2 + yy**2) + >>> xx.shape, yy.shape, zz.shape + ((101, 101), (101, 101), (101, 101)) + >>> # sparse coordinate arrays + >>> xs, ys = np.meshgrid(x, y, sparse=True) + >>> zs = np.sqrt(xs**2 + ys**2) + >>> xs.shape, ys.shape, zs.shape + ((1, 101), (101, 1), (101, 101)) + >>> np.array_equal(zz, zs) + True - >>> import matplotlib.pyplot as plt - >>> x = np.arange(-5, 5, 0.1) - >>> y = np.arange(-5, 5, 0.1) - >>> xx, yy = np.meshgrid(x, y, sparse=True) - >>> z = np.sin(xx**2 + yy**2) / (xx**2 + yy**2) - >>> h = plt.contourf(x,y,z) + >>> h = plt.contourf(x, y, zs) + >>> plt.axis('scaled') + >>> plt.colorbar() >>> plt.show() - """ ndim = len(xi) @@ -4338,7 +5181,7 @@ def delete(arr, obj, axis=None): >>> mask[[0,2,4]] = False >>> result = arr[mask,...] - Is equivalent to `np.delete(arr, [0,2,4], axis=0)`, but allows further + Is equivalent to ``np.delete(arr, [0,2,4], axis=0)``, but allows further use of `mask`. Examples @@ -4434,6 +5277,22 @@ def delete(arr, obj, axis=None): return new if isinstance(obj, (int, integer)) and not isinstance(obj, bool): + single_value = True + else: + single_value = False + _obj = obj + obj = np.asarray(obj) + # `size == 0` to allow empty lists similar to indexing, but (as there) + # is really too generic: + if obj.size == 0 and not isinstance(_obj, np.ndarray): + obj = obj.astype(intp) + elif obj.size == 1 and obj.dtype.kind in "ui": + # For a size 1 integer array we can use the single-value path + # (most dtypes, except boolean, should just fail later). + obj = obj.item() + single_value = True + + if single_value: # optimization for a single value if (obj < -N or obj >= N): raise IndexError( @@ -4450,11 +5309,6 @@ def delete(arr, obj, axis=None): slobj2[axis] = slice(obj+1, None) new[tuple(slobj)] = arr[tuple(slobj2)] else: - _obj = obj - obj = np.asarray(obj) - if obj.size == 0 and not isinstance(_obj, np.ndarray): - obj = obj.astype(intp) - if obj.dtype == bool: if obj.shape != (N,): raise ValueError('boolean array argument obj to delete ' @@ -4522,9 +5376,9 @@ def insert(arr, obj, values, axis=None): Notes ----- - Note that for higher dimensional inserts `obj=0` behaves very different - from `obj=[0]` just like `arr[:,0,:] = values` is different from - `arr[:,[0],:] = values`. + Note that for higher dimensional inserts ``obj=0`` behaves very different + from ``obj=[0]`` just like ``arr[:,0,:] = values`` is different from + ``arr[:,[0],:] = values``. Examples -------- @@ -4603,7 +5457,7 @@ def insert(arr, obj, values, axis=None): warnings.warn( "in the future insert will treat boolean arrays and " "array-likes as a boolean index instead of casting it to " - "integer", FutureWarning, stacklevel=3) + "integer", FutureWarning, stacklevel=2) indices = indices.astype(intp) # Code after warning period: #if obj.ndim != 1: @@ -4617,9 +5471,8 @@ def insert(arr, obj, values, axis=None): if indices.size == 1: index = indices.item() if index < -N or index > N: - raise IndexError( - "index %i is out of bounds for axis %i with " - "size %i" % (obj, axis, N)) + raise IndexError(f"index {obj} is out of bounds for axis {axis} " + f"with size {N}") if (index < 0): index += N diff --git a/numpy/lib/function_base.pyi b/numpy/lib/function_base.pyi new file mode 100644 index 000000000..687e4ab17 --- /dev/null +++ b/numpy/lib/function_base.pyi @@ -0,0 +1,697 @@ +import sys +from collections.abc import Sequence, Iterator, Callable, Iterable +from typing import ( + Literal as L, + Any, + TypeVar, + overload, + Protocol, + SupportsIndex, + SupportsInt, +) + +if sys.version_info >= (3, 10): + from typing import TypeGuard +else: + from typing_extensions import TypeGuard + +from numpy import ( + vectorize as vectorize, + ufunc, + generic, + floating, + complexfloating, + intp, + float64, + complex128, + timedelta64, + datetime64, + object_, + _OrderKACF, +) + +from numpy._typing import ( + NDArray, + ArrayLike, + DTypeLike, + _ShapeLike, + _ScalarLike_co, + _DTypeLike, + _ArrayLike, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeTD64_co, + _ArrayLikeDT64_co, + _ArrayLikeObject_co, + _FloatLike_co, + _ComplexLike_co, +) + +from numpy.core.function_base import ( + add_newdoc as add_newdoc, +) + +from numpy.core.multiarray import ( + add_docstring as add_docstring, + bincount as bincount, +) + +from numpy.core.umath import _add_newdoc_ufunc + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_SCT = TypeVar("_SCT", bound=generic) +_ArrayType = TypeVar("_ArrayType", bound=NDArray[Any]) + +_2Tuple = tuple[_T, _T] + +class _TrimZerosSequence(Protocol[_T_co]): + def __len__(self) -> int: ... + def __getitem__(self, key: slice, /) -> _T_co: ... + def __iter__(self) -> Iterator[Any]: ... + +class _SupportsWriteFlush(Protocol): + def write(self, s: str, /) -> object: ... + def flush(self) -> object: ... + +__all__: list[str] + +# NOTE: This is in reality a re-export of `np.core.umath._add_newdoc_ufunc` +def add_newdoc_ufunc(ufunc: ufunc, new_docstring: str, /) -> None: ... + +@overload +def rot90( + m: _ArrayLike[_SCT], + k: int = ..., + axes: tuple[int, int] = ..., +) -> NDArray[_SCT]: ... +@overload +def rot90( + m: ArrayLike, + k: int = ..., + axes: tuple[int, int] = ..., +) -> NDArray[Any]: ... + +@overload +def flip(m: _SCT, axis: None = ...) -> _SCT: ... +@overload +def flip(m: _ScalarLike_co, axis: None = ...) -> Any: ... +@overload +def flip(m: _ArrayLike[_SCT], axis: None | _ShapeLike = ...) -> NDArray[_SCT]: ... +@overload +def flip(m: ArrayLike, axis: None | _ShapeLike = ...) -> NDArray[Any]: ... + +def iterable(y: object) -> TypeGuard[Iterable[Any]]: ... + +@overload +def average( + a: _ArrayLikeFloat_co, + axis: None = ..., + weights: None | _ArrayLikeFloat_co= ..., + returned: L[False] = ..., + keepdims: L[False] = ..., +) -> floating[Any]: ... +@overload +def average( + a: _ArrayLikeComplex_co, + axis: None = ..., + weights: None | _ArrayLikeComplex_co = ..., + returned: L[False] = ..., + keepdims: L[False] = ..., +) -> complexfloating[Any, Any]: ... +@overload +def average( + a: _ArrayLikeObject_co, + axis: None = ..., + weights: None | Any = ..., + returned: L[False] = ..., + keepdims: L[False] = ..., +) -> Any: ... +@overload +def average( + a: _ArrayLikeFloat_co, + axis: None = ..., + weights: None | _ArrayLikeFloat_co= ..., + returned: L[True] = ..., + keepdims: L[False] = ..., +) -> _2Tuple[floating[Any]]: ... +@overload +def average( + a: _ArrayLikeComplex_co, + axis: None = ..., + weights: None | _ArrayLikeComplex_co = ..., + returned: L[True] = ..., + keepdims: L[False] = ..., +) -> _2Tuple[complexfloating[Any, Any]]: ... +@overload +def average( + a: _ArrayLikeObject_co, + axis: None = ..., + weights: None | Any = ..., + returned: L[True] = ..., + keepdims: L[False] = ..., +) -> _2Tuple[Any]: ... +@overload +def average( + a: _ArrayLikeComplex_co | _ArrayLikeObject_co, + axis: None | _ShapeLike = ..., + weights: None | Any = ..., + returned: L[False] = ..., + keepdims: bool = ..., +) -> Any: ... +@overload +def average( + a: _ArrayLikeComplex_co | _ArrayLikeObject_co, + axis: None | _ShapeLike = ..., + weights: None | Any = ..., + returned: L[True] = ..., + keepdims: bool = ..., +) -> _2Tuple[Any]: ... + +@overload +def asarray_chkfinite( + a: _ArrayLike[_SCT], + dtype: None = ..., + order: _OrderKACF = ..., +) -> NDArray[_SCT]: ... +@overload +def asarray_chkfinite( + a: object, + dtype: None = ..., + order: _OrderKACF = ..., +) -> NDArray[Any]: ... +@overload +def asarray_chkfinite( + a: Any, + dtype: _DTypeLike[_SCT], + order: _OrderKACF = ..., +) -> NDArray[_SCT]: ... +@overload +def asarray_chkfinite( + a: Any, + dtype: DTypeLike, + order: _OrderKACF = ..., +) -> NDArray[Any]: ... + +# TODO: Use PEP 612 `ParamSpec` once mypy supports `Concatenate` +# xref python/mypy#8645 +@overload +def piecewise( + x: _ArrayLike[_SCT], + condlist: ArrayLike, + funclist: Sequence[Any | Callable[..., Any]], + *args: Any, + **kw: Any, +) -> NDArray[_SCT]: ... +@overload +def piecewise( + x: ArrayLike, + condlist: ArrayLike, + funclist: Sequence[Any | Callable[..., Any]], + *args: Any, + **kw: Any, +) -> NDArray[Any]: ... + +def select( + condlist: Sequence[ArrayLike], + choicelist: Sequence[ArrayLike], + default: ArrayLike = ..., +) -> NDArray[Any]: ... + +@overload +def copy( + a: _ArrayType, + order: _OrderKACF, + subok: L[True], +) -> _ArrayType: ... +@overload +def copy( + a: _ArrayType, + order: _OrderKACF = ..., + *, + subok: L[True], +) -> _ArrayType: ... +@overload +def copy( + a: _ArrayLike[_SCT], + order: _OrderKACF = ..., + subok: L[False] = ..., +) -> NDArray[_SCT]: ... +@overload +def copy( + a: ArrayLike, + order: _OrderKACF = ..., + subok: L[False] = ..., +) -> NDArray[Any]: ... + +def gradient( + f: ArrayLike, + *varargs: ArrayLike, + axis: None | _ShapeLike = ..., + edge_order: L[1, 2] = ..., +) -> Any: ... + +@overload +def diff( + a: _T, + n: L[0], + axis: SupportsIndex = ..., + prepend: ArrayLike = ..., + append: ArrayLike = ..., +) -> _T: ... +@overload +def diff( + a: ArrayLike, + n: int = ..., + axis: SupportsIndex = ..., + prepend: ArrayLike = ..., + append: ArrayLike = ..., +) -> NDArray[Any]: ... + +@overload +def interp( + x: _ArrayLikeFloat_co, + xp: _ArrayLikeFloat_co, + fp: _ArrayLikeFloat_co, + left: None | _FloatLike_co = ..., + right: None | _FloatLike_co = ..., + period: None | _FloatLike_co = ..., +) -> NDArray[float64]: ... +@overload +def interp( + x: _ArrayLikeFloat_co, + xp: _ArrayLikeFloat_co, + fp: _ArrayLikeComplex_co, + left: None | _ComplexLike_co = ..., + right: None | _ComplexLike_co = ..., + period: None | _FloatLike_co = ..., +) -> NDArray[complex128]: ... + +@overload +def angle(z: _ComplexLike_co, deg: bool = ...) -> floating[Any]: ... +@overload +def angle(z: object_, deg: bool = ...) -> Any: ... +@overload +def angle(z: _ArrayLikeComplex_co, deg: bool = ...) -> NDArray[floating[Any]]: ... +@overload +def angle(z: _ArrayLikeObject_co, deg: bool = ...) -> NDArray[object_]: ... + +@overload +def unwrap( + p: _ArrayLikeFloat_co, + discont: None | float = ..., + axis: int = ..., + *, + period: float = ..., +) -> NDArray[floating[Any]]: ... +@overload +def unwrap( + p: _ArrayLikeObject_co, + discont: None | float = ..., + axis: int = ..., + *, + period: float = ..., +) -> NDArray[object_]: ... + +def sort_complex(a: ArrayLike) -> NDArray[complexfloating[Any, Any]]: ... + +def trim_zeros( + filt: _TrimZerosSequence[_T], + trim: L["f", "b", "fb", "bf"] = ..., +) -> _T: ... + +@overload +def extract(condition: ArrayLike, arr: _ArrayLike[_SCT]) -> NDArray[_SCT]: ... +@overload +def extract(condition: ArrayLike, arr: ArrayLike) -> NDArray[Any]: ... + +def place(arr: NDArray[Any], mask: ArrayLike, vals: Any) -> None: ... + +def disp( + mesg: object, + device: None | _SupportsWriteFlush = ..., + linefeed: bool = ..., +) -> None: ... + +@overload +def cov( + m: _ArrayLikeFloat_co, + y: None | _ArrayLikeFloat_co = ..., + rowvar: bool = ..., + bias: bool = ..., + ddof: None | SupportsIndex | SupportsInt = ..., + fweights: None | ArrayLike = ..., + aweights: None | ArrayLike = ..., + *, + dtype: None = ..., +) -> NDArray[floating[Any]]: ... +@overload +def cov( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + bias: bool = ..., + ddof: None | SupportsIndex | SupportsInt = ..., + fweights: None | ArrayLike = ..., + aweights: None | ArrayLike = ..., + *, + dtype: None = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def cov( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + bias: bool = ..., + ddof: None | SupportsIndex | SupportsInt = ..., + fweights: None | ArrayLike = ..., + aweights: None | ArrayLike = ..., + *, + dtype: _DTypeLike[_SCT], +) -> NDArray[_SCT]: ... +@overload +def cov( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + bias: bool = ..., + ddof: None | SupportsIndex | SupportsInt = ..., + fweights: None | ArrayLike = ..., + aweights: None | ArrayLike = ..., + *, + dtype: DTypeLike, +) -> NDArray[Any]: ... + +# NOTE `bias` and `ddof` have been deprecated +@overload +def corrcoef( + m: _ArrayLikeFloat_co, + y: None | _ArrayLikeFloat_co = ..., + rowvar: bool = ..., + *, + dtype: None = ..., +) -> NDArray[floating[Any]]: ... +@overload +def corrcoef( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + *, + dtype: None = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def corrcoef( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + *, + dtype: _DTypeLike[_SCT], +) -> NDArray[_SCT]: ... +@overload +def corrcoef( + m: _ArrayLikeComplex_co, + y: None | _ArrayLikeComplex_co = ..., + rowvar: bool = ..., + *, + dtype: DTypeLike, +) -> NDArray[Any]: ... + +def blackman(M: _FloatLike_co) -> NDArray[floating[Any]]: ... + +def bartlett(M: _FloatLike_co) -> NDArray[floating[Any]]: ... + +def hanning(M: _FloatLike_co) -> NDArray[floating[Any]]: ... + +def hamming(M: _FloatLike_co) -> NDArray[floating[Any]]: ... + +def i0(x: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... + +def kaiser( + M: _FloatLike_co, + beta: _FloatLike_co, +) -> NDArray[floating[Any]]: ... + +@overload +def sinc(x: _FloatLike_co) -> floating[Any]: ... +@overload +def sinc(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def sinc(x: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... +@overload +def sinc(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +# NOTE: Deprecated +# def msort(a: ArrayLike) -> NDArray[Any]: ... + +@overload +def median( + a: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + keepdims: L[False] = ..., +) -> floating[Any]: ... +@overload +def median( + a: _ArrayLikeComplex_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + keepdims: L[False] = ..., +) -> complexfloating[Any, Any]: ... +@overload +def median( + a: _ArrayLikeTD64_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + keepdims: L[False] = ..., +) -> timedelta64: ... +@overload +def median( + a: _ArrayLikeObject_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + keepdims: L[False] = ..., +) -> Any: ... +@overload +def median( + a: _ArrayLikeFloat_co | _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeObject_co, + axis: None | _ShapeLike = ..., + out: None = ..., + overwrite_input: bool = ..., + keepdims: bool = ..., +) -> Any: ... +@overload +def median( + a: _ArrayLikeFloat_co | _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeObject_co, + axis: None | _ShapeLike = ..., + out: _ArrayType = ..., + overwrite_input: bool = ..., + keepdims: bool = ..., +) -> _ArrayType: ... + +_MethodKind = L[ + "inverted_cdf", + "averaged_inverted_cdf", + "closest_observation", + "interpolated_inverted_cdf", + "hazen", + "weibull", + "linear", + "median_unbiased", + "normal_unbiased", + "lower", + "higher", + "midpoint", + "nearest", +] + +@overload +def percentile( + a: _ArrayLikeFloat_co, + q: _FloatLike_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> floating[Any]: ... +@overload +def percentile( + a: _ArrayLikeComplex_co, + q: _FloatLike_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> complexfloating[Any, Any]: ... +@overload +def percentile( + a: _ArrayLikeTD64_co, + q: _FloatLike_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> timedelta64: ... +@overload +def percentile( + a: _ArrayLikeDT64_co, + q: _FloatLike_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> datetime64: ... +@overload +def percentile( + a: _ArrayLikeObject_co, + q: _FloatLike_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> Any: ... +@overload +def percentile( + a: _ArrayLikeFloat_co, + q: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> NDArray[floating[Any]]: ... +@overload +def percentile( + a: _ArrayLikeComplex_co, + q: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def percentile( + a: _ArrayLikeTD64_co, + q: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> NDArray[timedelta64]: ... +@overload +def percentile( + a: _ArrayLikeDT64_co, + q: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> NDArray[datetime64]: ... +@overload +def percentile( + a: _ArrayLikeObject_co, + q: _ArrayLikeFloat_co, + axis: None = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: L[False] = ..., +) -> NDArray[object_]: ... +@overload +def percentile( + a: _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeTD64_co | _ArrayLikeObject_co, + q: _ArrayLikeFloat_co, + axis: None | _ShapeLike = ..., + out: None = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: bool = ..., +) -> Any: ... +@overload +def percentile( + a: _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeTD64_co | _ArrayLikeObject_co, + q: _ArrayLikeFloat_co, + axis: None | _ShapeLike = ..., + out: _ArrayType = ..., + overwrite_input: bool = ..., + method: _MethodKind = ..., + keepdims: bool = ..., +) -> _ArrayType: ... + +# NOTE: Not an alias, but they do have identical signatures +# (that we can reuse) +quantile = percentile + +# TODO: Returns a scalar for <= 1D array-likes; returns an ndarray otherwise +def trapz( + y: _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeObject_co, + x: None | _ArrayLikeComplex_co | _ArrayLikeTD64_co | _ArrayLikeObject_co = ..., + dx: float = ..., + axis: SupportsIndex = ..., +) -> Any: ... + +def meshgrid( + *xi: ArrayLike, + copy: bool = ..., + sparse: bool = ..., + indexing: L["xy", "ij"] = ..., +) -> list[NDArray[Any]]: ... + +@overload +def delete( + arr: _ArrayLike[_SCT], + obj: slice | _ArrayLikeInt_co, + axis: None | SupportsIndex = ..., +) -> NDArray[_SCT]: ... +@overload +def delete( + arr: ArrayLike, + obj: slice | _ArrayLikeInt_co, + axis: None | SupportsIndex = ..., +) -> NDArray[Any]: ... + +@overload +def insert( + arr: _ArrayLike[_SCT], + obj: slice | _ArrayLikeInt_co, + values: ArrayLike, + axis: None | SupportsIndex = ..., +) -> NDArray[_SCT]: ... +@overload +def insert( + arr: ArrayLike, + obj: slice | _ArrayLikeInt_co, + values: ArrayLike, + axis: None | SupportsIndex = ..., +) -> NDArray[Any]: ... + +def append( + arr: ArrayLike, + values: ArrayLike, + axis: None | SupportsIndex = ..., +) -> NDArray[Any]: ... + +@overload +def digitize( + x: _FloatLike_co, + bins: _ArrayLikeFloat_co, + right: bool = ..., +) -> intp: ... +@overload +def digitize( + x: _ArrayLikeFloat_co, + bins: _ArrayLikeFloat_co, + right: bool = ..., +) -> NDArray[intp]: ... diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 1a9b41ced..35745e6dd 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -506,8 +506,8 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): with non-normal datasets. 'scott' - Less robust estimator that that takes into account data - variability and data size. + Less robust estimator that takes into account data variability + and data size. 'stone' Estimator based on leave-one-out cross-validation estimate of @@ -562,7 +562,8 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): below, :math:`h` is the binwidth and :math:`n_h` is the number of bins. All estimators that compute bin counts are recast to bin width using the `ptp` of the data. The final bin count is obtained from - ``np.round(np.ceil(range / h))``. + ``np.round(np.ceil(range / h))``. The final bin width is often less + than what is returned by the estimators below. 'auto' (maximum of the 'sturges' and 'fd' estimators) A compromise to get a good value. For small datasets the Sturges @@ -580,7 +581,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): datasets. The IQR is very robust to outliers. 'scott' - .. math:: h = \sigma \sqrt[3]{\frac{24 * \sqrt{\pi}}{n}} + .. math:: h = \sigma \sqrt[3]{\frac{24 \sqrt{\pi}}{n}} The binwidth is proportional to the standard deviation of the data and inversely proportional to cube root of ``x.size``. Can @@ -597,7 +598,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): does not take into account data variability. 'sturges' - .. math:: n_h = \log _{2}n+1 + .. math:: n_h = \log _{2}(n) + 1 The number of bins is the base 2 log of ``a.size``. This estimator assumes normality of data and is too conservative for @@ -606,9 +607,9 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): 'doane' .. math:: n_h = 1 + \log_{2}(n) + - \log_{2}(1 + \frac{|g_1|}{\sigma_{g_1}}) + \log_{2}\left(1 + \frac{|g_1|}{\sigma_{g_1}}\right) - g_1 = mean[(\frac{x - \mu}{\sigma})^3] + g_1 = mean\left[\left(\frac{x - \mu}{\sigma}\right)^3\right] \sigma_{g_1} = \sqrt{\frac{6(n - 2)}{(n + 1)(n + 3)}} @@ -670,15 +671,14 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): def _histogram_dispatcher( - a, bins=None, range=None, normed=None, weights=None, density=None): + a, bins=None, range=None, density=None, weights=None): return (a, bins, weights) @array_function_dispatch(_histogram_dispatcher) -def histogram(a, bins=10, range=None, normed=None, weights=None, - density=None): +def histogram(a, bins=10, range=None, density=None, weights=None): r""" - Compute the histogram of a set of data. + Compute the histogram of a dataset. Parameters ---------- @@ -703,16 +703,6 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, computation as well. While bin width is computed to be optimal based on the actual data within `range`, the bin count will fill the entire range including portions containing no data. - normed : bool, optional - - .. deprecated:: 1.6.0 - - This is equivalent to the `density` argument, but produces incorrect - results for unequal bin widths. It should not be used. - - .. versionchanged:: 1.15.0 - DeprecationWarnings are actually emitted. - weights : array_like, optional An array of weights, of the same shape as `a`. Each value in `a` only contributes its associated weight towards the bin count @@ -727,8 +717,6 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, histogram values will not be equal to 1 unless bins of unity width are chosen; it is not a probability *mass* function. - Overrides the ``normed`` keyword if given. - Returns ------- hist : array @@ -890,46 +878,15 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, n = np.diff(cum_n) - # density overrides the normed keyword - if density is not None: - if normed is not None: - # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6) - warnings.warn( - "The normed argument is ignored when density is provided. " - "In future passing both will result in an error.", - DeprecationWarning, stacklevel=3) - normed = None - if density: db = np.array(np.diff(bin_edges), float) return n/db/n.sum(), bin_edges - elif normed: - # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6) - warnings.warn( - "Passing `normed=True` on non-uniform bins has always been " - "broken, and computes neither the probability density " - "function nor the probability mass function. " - "The result is only correct if the bins are uniform, when " - "density=True will produce the same result anyway. " - "The argument will be removed in a future version of " - "numpy.", - np.VisibleDeprecationWarning, stacklevel=3) - - # this normalization is incorrect, but - db = np.array(np.diff(bin_edges), float) - return n/(n*db).sum(), bin_edges - else: - if normed is not None: - # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6) - warnings.warn( - "Passing normed=False is deprecated, and has no effect. " - "Consider passing the density argument instead.", - DeprecationWarning, stacklevel=3) - return n, bin_edges + + return n, bin_edges -def _histogramdd_dispatcher(sample, bins=None, range=None, normed=None, - weights=None, density=None): +def _histogramdd_dispatcher(sample, bins=None, range=None, density=None, + weights=None): if hasattr(sample, 'shape'): # same condition as used in histogramdd yield sample else: @@ -940,14 +897,13 @@ def _histogramdd_dispatcher(sample, bins=None, range=None, normed=None, @array_function_dispatch(_histogramdd_dispatcher) -def histogramdd(sample, bins=10, range=None, normed=None, weights=None, - density=None): +def histogramdd(sample, bins=10, range=None, density=None, weights=None): """ Compute the multidimensional histogram of some data. Parameters ---------- - sample : (N, D) array, or (D, N) array_like + sample : (N, D) array, or (N, D) array_like The data to be histogrammed. Note the unusual interpretation of sample when an array_like: @@ -978,20 +934,16 @@ def histogramdd(sample, bins=10, range=None, normed=None, weights=None, If False, the default, returns the number of samples in each bin. If True, returns the probability *density* function at the bin, ``bin_count / sample_count / bin_volume``. - normed : bool, optional - An alias for the density argument that behaves identically. To avoid - confusion with the broken normed argument to `histogram`, `density` - should be preferred. weights : (N,) array_like, optional An array of values `w_i` weighing each sample `(x_i, y_i, z_i, ...)`. - Weights are normalized to 1 if normed is True. If normed is False, + Weights are normalized to 1 if density is True. If density is False, the values of the returned histogram are equal to the sum of the weights belonging to the samples falling into each bin. Returns ------- H : ndarray - The multidimensional histogram of sample x. See normed and weights + The multidimensional histogram of sample x. See density and weights for the different possible semantics. edges : list A list of D arrays describing the bin edges for each dimension. @@ -1018,7 +970,7 @@ def histogramdd(sample, bins=10, range=None, normed=None, weights=None, sample = np.atleast_2d(sample).T N, D = sample.shape - nbin = np.empty(D, int) + nbin = np.empty(D, np.intp) edges = D*[None] dedges = D*[None] if weights is not None: @@ -1029,7 +981,7 @@ def histogramdd(sample, bins=10, range=None, normed=None, weights=None, if M != D: raise ValueError( 'The dimension of bins must be equal to the dimension of the ' - ' sample x.') + 'sample x.') except TypeError: # bins is an integer bins = D*[bins] @@ -1049,13 +1001,13 @@ def histogramdd(sample, bins=10, range=None, normed=None, weights=None, smin, smax = _get_outer_edges(sample[:,i], range[i]) try: n = operator.index(bins[i]) - + except TypeError as e: raise TypeError( "`bins[{}]` must be an integer, when a scalar".format(i) ) from e - - edges[i] = np.linspace(smin, smax, n + 1) + + edges[i] = np.linspace(smin, smax, n + 1) elif np.ndim(bins[i]) == 1: edges[i] = np.asarray(bins[i]) if np.any(edges[i][:-1] > edges[i][1:]): @@ -1103,16 +1055,6 @@ def histogramdd(sample, bins=10, range=None, normed=None, weights=None, core = D*(slice(1, -1),) hist = hist[core] - # handle the aliasing normed argument - if normed is None: - if density is None: - density = False - elif density is None: - # an explicit normed argument was passed, alias it to the new name - density = normed - else: - raise TypeError("Cannot specify both 'normed' and 'density'") - if density: # calculate the probability density function s = hist.sum() diff --git a/numpy/lib/histograms.pyi b/numpy/lib/histograms.pyi new file mode 100644 index 000000000..ce02718ad --- /dev/null +++ b/numpy/lib/histograms.pyi @@ -0,0 +1,47 @@ +from collections.abc import Sequence +from typing import ( + Literal as L, + Any, + SupportsIndex, +) + +from numpy._typing import ( + NDArray, + ArrayLike, +) + +_BinKind = L[ + "stone", + "auto", + "doane", + "fd", + "rice", + "scott", + "sqrt", + "sturges", +] + +__all__: list[str] + +def histogram_bin_edges( + a: ArrayLike, + bins: _BinKind | SupportsIndex | ArrayLike = ..., + range: None | tuple[float, float] = ..., + weights: None | ArrayLike = ..., +) -> NDArray[Any]: ... + +def histogram( + a: ArrayLike, + bins: _BinKind | SupportsIndex | ArrayLike = ..., + range: None | tuple[float, float] = ..., + density: bool = ..., + weights: None | ArrayLike = ..., +) -> tuple[NDArray[Any], NDArray[Any]]: ... + +def histogramdd( + sample: ArrayLike, + bins: SupportsIndex | ArrayLike = ..., + range: Sequence[tuple[float, float]] = ..., + density: None | bool = ..., + weights: None | ArrayLike = ..., +) -> tuple[NDArray[Any], list[NDArray[Any]]]: ... diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index 6093f7e9d..abf9e1090 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -1,17 +1,17 @@ import functools import sys import math +import warnings +import numpy as np +from .._utils import set_module import numpy.core.numeric as _nx -from numpy.core.numeric import ( - asarray, ScalarType, array, alltrue, cumprod, arange, ndim - ) +from numpy.core.numeric import ScalarType, array from numpy.core.numerictypes import find_common_type, issubdtype import numpy.matrixlib as matrixlib from .function_base import diff from numpy.core.multiarray import ravel_multi_index, unravel_index -from numpy.core.overrides import set_module from numpy.core import overrides, linspace from numpy.lib.stride_tricks import as_strided @@ -24,7 +24,7 @@ __all__ = [ 'ravel_multi_index', 'unravel_index', 'mgrid', 'ogrid', 'r_', 'c_', 's_', 'index_exp', 'ix_', 'ndenumerate', 'ndindex', 'fill_diagonal', 'diag_indices', 'diag_indices_from' - ] +] def _ix__dispatcher(*args): @@ -93,7 +93,7 @@ def ix_(*args): nd = len(args) for k, new in enumerate(args): if not isinstance(new, _nx.ndarray): - new = asarray(new) + new = np.asarray(new) if new.size == 0: # Explicitly type empty arrays to avoid float default new = new.astype(_nx.intp) @@ -105,6 +105,7 @@ def ix_(*args): out.append(new) return tuple(out) + class nd_grid: """ Construct a multi-dimensional "meshgrid". @@ -146,32 +147,33 @@ class nd_grid: def __getitem__(self, key): try: size = [] - typ = int + # Mimic the behavior of `np.arange` and use a data type + # which is at least as large as `np.int_` + num_list = [0] for k in range(len(key)): step = key[k].step start = key[k].start + stop = key[k].stop if start is None: start = 0 if step is None: step = 1 if isinstance(step, (_nx.complexfloating, complex)): - size.append(int(abs(step))) - typ = float + step = abs(step) + size.append(int(step)) else: size.append( - int(math.ceil((key[k].stop - start)/(step*1.0)))) - if (isinstance(step, (_nx.floating, float)) or - isinstance(start, (_nx.floating, float)) or - isinstance(key[k].stop, (_nx.floating, float))): - typ = float + int(math.ceil((stop - start) / (step*1.0)))) + num_list += [start, stop, step] + typ = _nx.result_type(*num_list) if self.sparse: nn = [_nx.arange(_x, dtype=_t) - for _x, _t in zip(size, (typ,)*len(size))] + for _x, _t in zip(size, (typ,)*len(size))] else: nn = _nx.indices(size, typ) - for k in range(len(size)): - step = key[k].step - start = key[k].start + for k, kk in enumerate(key): + step = kk.step + start = kk.start if start is None: start = 0 if step is None: @@ -179,7 +181,7 @@ class nd_grid: if isinstance(step, (_nx.complexfloating, complex)): step = int(abs(step)) if step != 1: - step = (key[k].stop - start)/float(step-1) + step = (kk.stop - start) / float(step - 1) nn[k] = (nn[k]*step+start) if self.sparse: slobj = [_nx.newaxis]*len(size) @@ -195,12 +197,13 @@ class nd_grid: if start is None: start = 0 if isinstance(step, (_nx.complexfloating, complex)): - step = abs(step) - length = int(step) + # Prevent the (potential) creation of integer arrays + step_float = abs(step) + step = length = int(step_float) if step != 1: step = (key.stop-start)/float(step-1) - stop = key.stop + step - return _nx.arange(0, length, 1, float)*step + start + typ = _nx.result_type(start, stop, step_float) + return _nx.arange(0, length, 1, dtype=typ)*step + start else: return _nx.arange(start, stop, step) @@ -226,13 +229,15 @@ class MGridClass(nd_grid): See Also -------- - numpy.lib.index_tricks.nd_grid : class of `ogrid` and `mgrid` objects + lib.index_tricks.nd_grid : class of `ogrid` and `mgrid` objects ogrid : like mgrid but returns open (not fleshed out) mesh grids + meshgrid: return coordinate matrices from coordinate vectors r_ : array concatenator + :ref:`how-to-partition` Examples -------- - >>> np.mgrid[0:5,0:5] + >>> np.mgrid[0:5, 0:5] array([[[0, 0, 0, 0, 0], [1, 1, 1, 1, 1], [2, 2, 2, 2, 2], @@ -247,11 +252,14 @@ class MGridClass(nd_grid): array([-1. , -0.5, 0. , 0.5, 1. ]) """ + def __init__(self): - super(MGridClass, self).__init__(sparse=False) + super().__init__(sparse=False) + mgrid = MGridClass() + class OGridClass(nd_grid): """ `nd_grid` instance which returns an open multi-dimensional "meshgrid". @@ -276,7 +284,9 @@ class OGridClass(nd_grid): -------- np.lib.index_tricks.nd_grid : class of `ogrid` and `mgrid` objects mgrid : like `ogrid` but returns dense (or fleshed out) mesh grids + meshgrid: return coordinate matrices from coordinate vectors r_ : array concatenator + :ref:`how-to-partition` Examples -------- @@ -291,8 +301,10 @@ class OGridClass(nd_grid): [4]]), array([[0, 1, 2, 3, 4]])] """ + def __init__(self): - super(OGridClass, self).__init__(sparse=True) + super().__init__(sparse=True) + ogrid = OGridClass() @@ -356,7 +368,7 @@ class AxisConcatenator: elif isinstance(item, str): if k != 0: raise ValueError("special directives must be the " - "first entry.") + "first entry.") if item in ('r', 'c'): matrix = True col = (item == 'c') @@ -375,15 +387,15 @@ class AxisConcatenator: try: axis = int(item) continue - except (ValueError, TypeError): - raise ValueError("unknown special directive") + except (ValueError, TypeError) as e: + raise ValueError("unknown special directive") from e elif type(item) in ScalarType: newobj = array(item, ndmin=ndmin) scalars.append(len(objs)) scalar = True scalartypes.append(newobj.dtype) else: - item_ndim = ndim(item) + item_ndim = np.ndim(item) newobj = array(item, copy=False, subok=True, ndmin=ndmin) if trans1d != -1 and item_ndim < ndmin: k2 = ndmin - item_ndim @@ -419,6 +431,7 @@ class AxisConcatenator: # etc. because otherwise we couldn't get the doc string to come out right # in help(r_) + class RClass(AxisConcatenator): """ Translates slice objects to concatenation along the first axis. @@ -517,8 +530,10 @@ class RClass(AxisConcatenator): def __init__(self): AxisConcatenator.__init__(self, 0) + r_ = RClass() + class CClass(AxisConcatenator): """ Translates slice objects to concatenation along the second axis. @@ -527,7 +542,7 @@ class CClass(AxisConcatenator): useful because of its common occurrence. In particular, arrays will be stacked along their last axis after being upgraded to at least 2-D with 1's post-pended to the shape (column vectors made out of 1-D arrays). - + See Also -------- column_stack : Stack 1-D arrays as columns into a 2-D array. @@ -580,7 +595,7 @@ class ndenumerate: """ def __init__(self, arr): - self.iter = asarray(arr).flat + self.iter = np.asarray(arr).flat def __next__(self): """ @@ -612,7 +627,7 @@ class ndindex: Parameters ---------- shape : ints, or a single tuple of ints - The size of each dimension of the array can be passed as + The size of each dimension of the array can be passed as individual parameters or as the elements of a tuple. See Also @@ -621,7 +636,8 @@ class ndindex: Examples -------- - # dimensions as individual arguments + Dimensions as individual arguments + >>> for index in np.ndindex(3, 2, 1): ... print(index) (0, 0, 0) @@ -631,7 +647,8 @@ class ndindex: (2, 0, 0) (2, 1, 0) - # same dimensions - but in a tuple (3, 2, 1) + Same dimensions - but in a tuple ``(3, 2, 1)`` + >>> for index in np.ndindex((3, 2, 1)): ... print(index) (0, 0, 0) @@ -659,7 +676,15 @@ class ndindex: Increment the multi-dimensional index by one. This method is for backward compatibility only: do not use. + + .. deprecated:: 1.20.0 + This method has been advised against since numpy 1.8.0, but only + started emitting DeprecationWarning as of this version. """ + # NumPy 1.20.0, 2020-09-08 + warnings.warn( + "`ndindex.ndincr()` is deprecated, use `next(ndindex)` instead", + DeprecationWarning, stacklevel=2) next(self) def __next__(self): @@ -742,6 +767,7 @@ class IndexExpression: else: return item + index_exp = IndexExpression(maketuple=True) s_ = IndexExpression(maketuple=False) @@ -876,15 +902,15 @@ def fill_diagonal(a, val, wrap=False): # Explicit, fast formula for the common case. For 2-d arrays, we # accept rectangular ones. step = a.shape[1] + 1 - #This is needed to don't have tall matrix have the diagonal wrap. + # This is needed to don't have tall matrix have the diagonal wrap. if not wrap: end = a.shape[1] * a.shape[1] else: # For more than d=2, the strided formula is only valid for arrays with # all dimensions equal, so we check first. - if not alltrue(diff(a.shape) == 0): + if not np.all(diff(a.shape) == 0): raise ValueError("All dimensions of input must be of equal length") - step = 1 + (cumprod(a.shape[:-1])).sum() + step = 1 + (np.cumprod(a.shape[:-1])).sum() # Write the value out into the diagonal. a.flat[:end:step] = val @@ -955,7 +981,7 @@ def diag_indices(n, ndim=2): [0, 1]]]) """ - idx = arange(n) + idx = np.arange(n) return (idx,) * ndim @@ -982,13 +1008,39 @@ def diag_indices_from(arr): ----- .. versionadded:: 1.4.0 + Examples + -------- + + Create a 4 by 4 array. + + >>> a = np.arange(16).reshape(4, 4) + >>> a + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]) + + Get the indices of the diagonal elements. + + >>> di = np.diag_indices_from(a) + >>> di + (array([0, 1, 2, 3]), array([0, 1, 2, 3])) + + >>> a[di] + array([ 0, 5, 10, 15]) + + This is simply syntactic sugar for diag_indices. + + >>> np.diag_indices(a.shape[0]) + (array([0, 1, 2, 3]), array([0, 1, 2, 3])) + """ if not arr.ndim >= 2: raise ValueError("input array must be at least 2-d") # For more than d=2, the strided formula is only valid for arrays with # all dimensions equal, so we check first. - if not alltrue(diff(arr.shape) == 0): + if not np.all(diff(arr.shape) == 0): raise ValueError("All dimensions of input must be of equal length") return diag_indices(arr.shape[0], arr.ndim) diff --git a/numpy/lib/index_tricks.pyi b/numpy/lib/index_tricks.pyi new file mode 100644 index 000000000..29a6b9e2b --- /dev/null +++ b/numpy/lib/index_tricks.pyi @@ -0,0 +1,162 @@ +from collections.abc import Sequence +from typing import ( + Any, + TypeVar, + Generic, + overload, + Literal, + SupportsIndex, +) + +from numpy import ( + # Circumvent a naming conflict with `AxisConcatenator.matrix` + matrix as _Matrix, + ndenumerate as ndenumerate, + ndindex as ndindex, + ndarray, + dtype, + integer, + str_, + bytes_, + bool_, + int_, + float_, + complex_, + intp, + _OrderCF, + _ModeKind, +) +from numpy._typing import ( + # Arrays + ArrayLike, + _NestedSequence, + _FiniteNestedSequence, + NDArray, + _ArrayLikeInt, + + # DTypes + DTypeLike, + _SupportsDType, + + # Shapes + _ShapeLike, +) + +from numpy.core.multiarray import ( + unravel_index as unravel_index, + ravel_multi_index as ravel_multi_index, +) + +_T = TypeVar("_T") +_DType = TypeVar("_DType", bound=dtype[Any]) +_BoolType = TypeVar("_BoolType", Literal[True], Literal[False]) +_TupType = TypeVar("_TupType", bound=tuple[Any, ...]) +_ArrayType = TypeVar("_ArrayType", bound=ndarray[Any, Any]) + +__all__: list[str] + +@overload +def ix_(*args: _FiniteNestedSequence[_SupportsDType[_DType]]) -> tuple[ndarray[Any, _DType], ...]: ... +@overload +def ix_(*args: str | _NestedSequence[str]) -> tuple[NDArray[str_], ...]: ... +@overload +def ix_(*args: bytes | _NestedSequence[bytes]) -> tuple[NDArray[bytes_], ...]: ... +@overload +def ix_(*args: bool | _NestedSequence[bool]) -> tuple[NDArray[bool_], ...]: ... +@overload +def ix_(*args: int | _NestedSequence[int]) -> tuple[NDArray[int_], ...]: ... +@overload +def ix_(*args: float | _NestedSequence[float]) -> tuple[NDArray[float_], ...]: ... +@overload +def ix_(*args: complex | _NestedSequence[complex]) -> tuple[NDArray[complex_], ...]: ... + +class nd_grid(Generic[_BoolType]): + sparse: _BoolType + def __init__(self, sparse: _BoolType = ...) -> None: ... + @overload + def __getitem__( + self: nd_grid[Literal[False]], + key: slice | Sequence[slice], + ) -> NDArray[Any]: ... + @overload + def __getitem__( + self: nd_grid[Literal[True]], + key: slice | Sequence[slice], + ) -> list[NDArray[Any]]: ... + +class MGridClass(nd_grid[Literal[False]]): + def __init__(self) -> None: ... + +mgrid: MGridClass + +class OGridClass(nd_grid[Literal[True]]): + def __init__(self) -> None: ... + +ogrid: OGridClass + +class AxisConcatenator: + axis: int + matrix: bool + ndmin: int + trans1d: int + def __init__( + self, + axis: int = ..., + matrix: bool = ..., + ndmin: int = ..., + trans1d: int = ..., + ) -> None: ... + @staticmethod + @overload + def concatenate( # type: ignore[misc] + *a: ArrayLike, axis: SupportsIndex = ..., out: None = ... + ) -> NDArray[Any]: ... + @staticmethod + @overload + def concatenate( + *a: ArrayLike, axis: SupportsIndex = ..., out: _ArrayType = ... + ) -> _ArrayType: ... + @staticmethod + def makemat( + data: ArrayLike, dtype: DTypeLike = ..., copy: bool = ... + ) -> _Matrix[Any, Any]: ... + + # TODO: Sort out this `__getitem__` method + def __getitem__(self, key: Any) -> Any: ... + +class RClass(AxisConcatenator): + axis: Literal[0] + matrix: Literal[False] + ndmin: Literal[1] + trans1d: Literal[-1] + def __init__(self) -> None: ... + +r_: RClass + +class CClass(AxisConcatenator): + axis: Literal[-1] + matrix: Literal[False] + ndmin: Literal[2] + trans1d: Literal[0] + def __init__(self) -> None: ... + +c_: CClass + +class IndexExpression(Generic[_BoolType]): + maketuple: _BoolType + def __init__(self, maketuple: _BoolType) -> None: ... + @overload + def __getitem__(self, item: _TupType) -> _TupType: ... # type: ignore[misc] + @overload + def __getitem__(self: IndexExpression[Literal[True]], item: _T) -> tuple[_T]: ... + @overload + def __getitem__(self: IndexExpression[Literal[False]], item: _T) -> _T: ... + +index_exp: IndexExpression[Literal[True]] +s_: IndexExpression[Literal[False]] + +def fill_diagonal(a: ndarray[Any, Any], val: Any, wrap: bool = ...) -> None: ... +def diag_indices(n: int, ndim: int = ...) -> tuple[NDArray[int_], ...]: ... +def diag_indices_from(arr: ArrayLike) -> tuple[NDArray[int_], ...]: ... + +# NOTE: see `numpy/__init__.pyi` for `ndenumerate` and `ndindex` diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index c81239f6b..117cc7851 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -133,6 +133,7 @@ class NDArrayOperatorsMixin: .. versionadded:: 1.13 """ + __slots__ = () # Like np.ndarray, this mixin class implements "Option 1" from the ufunc # overrides NEP. diff --git a/numpy/lib/mixins.pyi b/numpy/lib/mixins.pyi new file mode 100644 index 000000000..c57442133 --- /dev/null +++ b/numpy/lib/mixins.pyi @@ -0,0 +1,74 @@ +from abc import ABCMeta, abstractmethod +from typing import Literal as L, Any + +from numpy import ufunc + +__all__: list[str] + +# NOTE: `NDArrayOperatorsMixin` is not formally an abstract baseclass, +# even though it's reliant on subclasses implementing `__array_ufunc__` + +# NOTE: The accepted input- and output-types of the various dunders are +# completely dependent on how `__array_ufunc__` is implemented. +# As such, only little type safety can be provided here. + +class NDArrayOperatorsMixin(metaclass=ABCMeta): + @abstractmethod + def __array_ufunc__( + self, + ufunc: ufunc, + method: L["__call__", "reduce", "reduceat", "accumulate", "outer", "inner"], + *inputs: Any, + **kwargs: Any, + ) -> Any: ... + def __lt__(self, other: Any) -> Any: ... + def __le__(self, other: Any) -> Any: ... + def __eq__(self, other: Any) -> Any: ... + def __ne__(self, other: Any) -> Any: ... + def __gt__(self, other: Any) -> Any: ... + def __ge__(self, other: Any) -> Any: ... + def __add__(self, other: Any) -> Any: ... + def __radd__(self, other: Any) -> Any: ... + def __iadd__(self, other: Any) -> Any: ... + def __sub__(self, other: Any) -> Any: ... + def __rsub__(self, other: Any) -> Any: ... + def __isub__(self, other: Any) -> Any: ... + def __mul__(self, other: Any) -> Any: ... + def __rmul__(self, other: Any) -> Any: ... + def __imul__(self, other: Any) -> Any: ... + def __matmul__(self, other: Any) -> Any: ... + def __rmatmul__(self, other: Any) -> Any: ... + def __imatmul__(self, other: Any) -> Any: ... + def __truediv__(self, other: Any) -> Any: ... + def __rtruediv__(self, other: Any) -> Any: ... + def __itruediv__(self, other: Any) -> Any: ... + def __floordiv__(self, other: Any) -> Any: ... + def __rfloordiv__(self, other: Any) -> Any: ... + def __ifloordiv__(self, other: Any) -> Any: ... + def __mod__(self, other: Any) -> Any: ... + def __rmod__(self, other: Any) -> Any: ... + def __imod__(self, other: Any) -> Any: ... + def __divmod__(self, other: Any) -> Any: ... + def __rdivmod__(self, other: Any) -> Any: ... + def __pow__(self, other: Any) -> Any: ... + def __rpow__(self, other: Any) -> Any: ... + def __ipow__(self, other: Any) -> Any: ... + def __lshift__(self, other: Any) -> Any: ... + def __rlshift__(self, other: Any) -> Any: ... + def __ilshift__(self, other: Any) -> Any: ... + def __rshift__(self, other: Any) -> Any: ... + def __rrshift__(self, other: Any) -> Any: ... + def __irshift__(self, other: Any) -> Any: ... + def __and__(self, other: Any) -> Any: ... + def __rand__(self, other: Any) -> Any: ... + def __iand__(self, other: Any) -> Any: ... + def __xor__(self, other: Any) -> Any: ... + def __rxor__(self, other: Any) -> Any: ... + def __ixor__(self, other: Any) -> Any: ... + def __or__(self, other: Any) -> Any: ... + def __ror__(self, other: Any) -> Any: ... + def __ior__(self, other: Any) -> Any: ... + def __neg__(self) -> Any: ... + def __pos__(self) -> Any: ... + def __abs__(self) -> Any: ... + def __invert__(self) -> Any: ... diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index 003550432..b3b570860 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -160,12 +160,16 @@ def _remove_nan_1d(arr1d, overwrite_input=False): True if `res` can be modified in place, given the constraint on the input """ + if arr1d.dtype == object: + # object arrays do not support `isnan` (gh-9009), so make a guess + c = np.not_equal(arr1d, arr1d, dtype=bool) + else: + c = np.isnan(arr1d) - c = np.isnan(arr1d) s = np.nonzero(c)[0] if s.size == arr1d.size: warnings.warn("All-NaN slice encountered", RuntimeWarning, - stacklevel=5) + stacklevel=6) return arr1d[:0], True elif s.size == 0: return arr1d, overwrite_input @@ -184,9 +188,8 @@ def _divide_by_count(a, b, out=None): """ Compute a/b ignoring invalid results. If `a` is an array the division is done in place. If `a` is a scalar, then its type is preserved in the - output. If out is None, then then a is used instead so that the - division is in place. Note that this is only called with `a` an inexact - type. + output. If out is None, then a is used instead so that the division + is in place. Note that this is only called with `a` an inexact type. Parameters ---------- @@ -214,19 +217,25 @@ def _divide_by_count(a, b, out=None): return np.divide(a, b, out=out, casting='unsafe') else: if out is None: - return a.dtype.type(a / b) + # Precaution against reduced object arrays + try: + return a.dtype.type(a / b) + except AttributeError: + return a / b else: # This is questionable, but currently a numpy scalar can # be output to a zero dimensional array. return np.divide(a, b, out=out, casting='unsafe') -def _nanmin_dispatcher(a, axis=None, out=None, keepdims=None): +def _nanmin_dispatcher(a, axis=None, out=None, keepdims=None, + initial=None, where=None): return (a, out) @array_function_dispatch(_nanmin_dispatcher) -def nanmin(a, axis=None, out=None, keepdims=np._NoValue): +def nanmin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, + where=np._NoValue): """ Return minimum of an array or minimum along an axis, ignoring any NaNs. When all-NaN slices are encountered a ``RuntimeWarning`` is raised and @@ -244,7 +253,7 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `ufuncs-output-type` for more details. + :ref:`ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -258,6 +267,16 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): does not implement `keepdims` any exceptions will be raised. .. versionadded:: 1.8.0 + initial : scalar, optional + The maximum value of an output element. Must be present to allow + computation on empty slice. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 + where : array_like of bool, optional + Elements to compare for the minimum. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -313,13 +332,18 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): kwargs = {} if keepdims is not np._NoValue: kwargs['keepdims'] = keepdims + if initial is not np._NoValue: + kwargs['initial'] = initial + if where is not np._NoValue: + kwargs['where'] = where + if type(a) is np.ndarray and a.dtype != np.object_: # Fast, but not safe for subclasses of ndarray, or object arrays, # which do not implement isnan (gh-9009), or fmin correctly (gh-8975) res = np.fmin.reduce(a, axis=axis, out=out, **kwargs) if np.isnan(res).any(): warnings.warn("All-NaN slice encountered", RuntimeWarning, - stacklevel=3) + stacklevel=2) else: # Slow, but safe for subclasses of ndarray a, mask = _replace_nan(a, +np.inf) @@ -328,20 +352,23 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): return res # Check for all-NaN axis + kwargs.pop("initial", None) mask = np.all(mask, axis=axis, **kwargs) if np.any(mask): res = _copyto(res, np.nan, mask) warnings.warn("All-NaN axis encountered", RuntimeWarning, - stacklevel=3) + stacklevel=2) return res -def _nanmax_dispatcher(a, axis=None, out=None, keepdims=None): +def _nanmax_dispatcher(a, axis=None, out=None, keepdims=None, + initial=None, where=None): return (a, out) @array_function_dispatch(_nanmax_dispatcher) -def nanmax(a, axis=None, out=None, keepdims=np._NoValue): +def nanmax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, + where=np._NoValue): """ Return the maximum of an array or maximum along an axis, ignoring any NaNs. When all-NaN slices are encountered a ``RuntimeWarning`` is @@ -359,7 +386,7 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `ufuncs-output-type` for more details. + :ref:`ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -373,6 +400,16 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): does not implement `keepdims` any exceptions will be raised. .. versionadded:: 1.8.0 + initial : scalar, optional + The minimum value of an output element. Must be present to allow + computation on empty slice. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 + where : array_like of bool, optional + Elements to compare for the maximum. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -428,13 +465,18 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): kwargs = {} if keepdims is not np._NoValue: kwargs['keepdims'] = keepdims + if initial is not np._NoValue: + kwargs['initial'] = initial + if where is not np._NoValue: + kwargs['where'] = where + if type(a) is np.ndarray and a.dtype != np.object_: # Fast, but not safe for subclasses of ndarray, or object arrays, # which do not implement isnan (gh-9009), or fmax correctly (gh-8975) res = np.fmax.reduce(a, axis=axis, out=out, **kwargs) if np.isnan(res).any(): warnings.warn("All-NaN slice encountered", RuntimeWarning, - stacklevel=3) + stacklevel=2) else: # Slow, but safe for subclasses of ndarray a, mask = _replace_nan(a, -np.inf) @@ -443,20 +485,21 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): return res # Check for all-NaN axis + kwargs.pop("initial", None) mask = np.all(mask, axis=axis, **kwargs) if np.any(mask): res = _copyto(res, np.nan, mask) warnings.warn("All-NaN axis encountered", RuntimeWarning, - stacklevel=3) + stacklevel=2) return res -def _nanargmin_dispatcher(a, axis=None): +def _nanargmin_dispatcher(a, axis=None, out=None, *, keepdims=None): return (a,) @array_function_dispatch(_nanargmin_dispatcher) -def nanargmin(a, axis=None): +def nanargmin(a, axis=None, out=None, *, keepdims=np._NoValue): """ Return the indices of the minimum values in the specified axis ignoring NaNs. For all-NaN slices ``ValueError`` is raised. Warning: the results @@ -468,6 +511,17 @@ def nanargmin(a, axis=None): Input data. axis : int, optional Axis along which to operate. By default flattened input is used. + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + + .. versionadded:: 1.22.0 + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the array. + + .. versionadded:: 1.22.0 Returns ------- @@ -492,20 +546,20 @@ def nanargmin(a, axis=None): """ a, mask = _replace_nan(a, np.inf) - res = np.argmin(a, axis=axis) if mask is not None: mask = np.all(mask, axis=axis) if np.any(mask): raise ValueError("All-NaN slice encountered") + res = np.argmin(a, axis=axis, out=out, keepdims=keepdims) return res -def _nanargmax_dispatcher(a, axis=None): +def _nanargmax_dispatcher(a, axis=None, out=None, *, keepdims=None): return (a,) @array_function_dispatch(_nanargmax_dispatcher) -def nanargmax(a, axis=None): +def nanargmax(a, axis=None, out=None, *, keepdims=np._NoValue): """ Return the indices of the maximum values in the specified axis ignoring NaNs. For all-NaN slices ``ValueError`` is raised. Warning: the @@ -518,6 +572,17 @@ def nanargmax(a, axis=None): Input data. axis : int, optional Axis along which to operate. By default flattened input is used. + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + + .. versionadded:: 1.22.0 + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the array. + + .. versionadded:: 1.22.0 Returns ------- @@ -542,20 +607,22 @@ def nanargmax(a, axis=None): """ a, mask = _replace_nan(a, -np.inf) - res = np.argmax(a, axis=axis) if mask is not None: mask = np.all(mask, axis=axis) if np.any(mask): raise ValueError("All-NaN slice encountered") + res = np.argmax(a, axis=axis, out=out, keepdims=keepdims) return res -def _nansum_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None): +def _nansum_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None, + initial=None, where=None): return (a, out) @array_function_dispatch(_nansum_dispatcher) -def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): +def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, + initial=np._NoValue, where=np._NoValue): """ Return the sum of array elements over a given axis treating Not a Numbers (NaNs) as zero. @@ -584,7 +651,7 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `ufuncs-output-type` for more details. The casting of NaN to integer + :ref:`ufuncs-output-type` for more details. The casting of NaN to integer can yield unexpected results. .. versionadded:: 1.8.0 @@ -600,6 +667,14 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): does not implement `keepdims` any exceptions will be raised. .. versionadded:: 1.8.0 + initial : scalar, optional + Starting value for the sum. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 + where : array_like of bool, optional + Elements to include in the sum. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -613,7 +688,7 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): -------- numpy.sum : Sum across array propagating NaNs. isnan : Show which elements are NaN. - isfinite: Show which elements are not NaN or +/-inf. + isfinite : Show which elements are not NaN or +/-inf. Notes ----- @@ -645,15 +720,18 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): """ a, mask = _replace_nan(a, 0) - return np.sum(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims) + return np.sum(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims, + initial=initial, where=where) -def _nanprod_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None): +def _nanprod_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None, + initial=None, where=None): return (a, out) @array_function_dispatch(_nanprod_dispatcher) -def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): +def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, + initial=np._NoValue, where=np._NoValue): """ Return the product of array elements over a given axis treating Not a Numbers (NaNs) as ones. @@ -681,12 +759,22 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `ufuncs-output-type` for more details. The casting of NaN to integer + :ref:`ufuncs-output-type` for more details. The casting of NaN to integer can yield unexpected results. keepdims : bool, optional If True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original `arr`. + initial : scalar, optional + The starting value for this product. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.22.0 + where : array_like of bool, optional + Elements to include in the product. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -715,7 +803,8 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): """ a, mask = _replace_nan(a, 1) - return np.prod(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims) + return np.prod(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims, + initial=initial, where=where) def _nancumsum_dispatcher(a, axis=None, dtype=None, out=None): @@ -749,7 +838,7 @@ def nancumsum(a, axis=None, dtype=None, out=None): out : ndarray, optional Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output - but the type will be cast if necessary. See `ufuncs-output-type` for + but the type will be cast if necessary. See :ref:`ufuncs-output-type` for more details. Returns @@ -855,12 +944,14 @@ def nancumprod(a, axis=None, dtype=None, out=None): return np.cumprod(a, axis=axis, dtype=dtype, out=out) -def _nanmean_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None): +def _nanmean_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None, + *, where=None): return (a, out) @array_function_dispatch(_nanmean_dispatcher) -def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): +def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, + *, where=np._NoValue): """ Compute the arithmetic mean along the specified axis, ignoring NaNs. @@ -888,7 +979,7 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `ufuncs-output-type` for more details. + :ref:`ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -898,6 +989,10 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): `keepdims` will be passed through to the `mean` or `sum` methods of sub-classes of `ndarray`. If the sub-classes methods does not implement `keepdims` any exceptions will be raised. + where : array_like of bool, optional + Elements to include in the mean. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -936,7 +1031,8 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): """ arr, mask = _replace_nan(a, 0) if mask is None: - return np.mean(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims) + return np.mean(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims, + where=where) if dtype is not None: dtype = np.dtype(dtype) @@ -945,13 +1041,15 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): if out is not None and not issubclass(out.dtype.type, np.inexact): raise TypeError("If a is inexact, then out must be inexact") - cnt = np.sum(~mask, axis=axis, dtype=np.intp, keepdims=keepdims) - tot = np.sum(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims) + cnt = np.sum(~mask, axis=axis, dtype=np.intp, keepdims=keepdims, + where=where) + tot = np.sum(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims, + where=where) avg = _divide_by_count(tot, cnt, out=out) isbad = (cnt == 0) if isbad.any(): - warnings.warn("Mean of empty slice", RuntimeWarning, stacklevel=3) + warnings.warn("Mean of empty slice", RuntimeWarning, stacklevel=2) # NaN is the only possible bad value, so no further # action is needed to handle bad results. return avg @@ -962,12 +1060,16 @@ def _nanmedian1d(arr1d, overwrite_input=False): Private function for rank 1 arrays. Compute the median ignoring NaNs. See nanmedian for parameter usage """ - arr1d, overwrite_input = _remove_nan_1d(arr1d, - overwrite_input=overwrite_input) - if arr1d.size == 0: - return np.nan + arr1d_parsed, overwrite_input = _remove_nan_1d( + arr1d, overwrite_input=overwrite_input, + ) + + if arr1d_parsed.size == 0: + # Ensure that a nan-esque scalar of the appropriate type (and unit) + # is returned for `timedelta64` and `complexfloating` + return arr1d[-1] - return np.median(arr1d, overwrite_input=overwrite_input) + return np.median(arr1d_parsed, overwrite_input=overwrite_input) def _nanmedian(a, axis=None, out=None, overwrite_input=False): @@ -1007,11 +1109,13 @@ def _nanmedian_small(a, axis=None, out=None, overwrite_input=False): m = np.ma.median(a, axis=axis, overwrite_input=overwrite_input) for i in range(np.count_nonzero(m.mask.ravel())): warnings.warn("All-NaN slice encountered", RuntimeWarning, - stacklevel=4) + stacklevel=5) + + fill_value = np.timedelta64("NaT") if m.dtype.kind == "m" else np.nan if out is not None: - out[...] = m.filled(np.nan) + out[...] = m.filled(fill_value) return out - return m.filled(np.nan) + return m.filled(fill_value) def _nanmedian_dispatcher( @@ -1110,22 +1214,29 @@ def nanmedian(a, axis=None, out=None, overwrite_input=False, keepdims=np._NoValu if a.size == 0: return np.nanmean(a, axis, out=out, keepdims=keepdims) - r, k = function_base._ureduce(a, func=_nanmedian, axis=axis, out=out, + return function_base._ureduce(a, func=_nanmedian, keepdims=keepdims, + axis=axis, out=out, overwrite_input=overwrite_input) - if keepdims and keepdims is not np._NoValue: - return r.reshape(k) - else: - return r -def _nanpercentile_dispatcher(a, q, axis=None, out=None, overwrite_input=None, - interpolation=None, keepdims=None): +def _nanpercentile_dispatcher( + a, q, axis=None, out=None, overwrite_input=None, + method=None, keepdims=None, *, interpolation=None): return (a, q, out) @array_function_dispatch(_nanpercentile_dispatcher) -def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear', keepdims=np._NoValue): +def nanpercentile( + a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=np._NoValue, + *, + interpolation=None, +): """ Compute the qth percentile of the data along the specified axis, while ignoring nan values. @@ -1140,32 +1251,49 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, Input array or object that can be converted to an array, containing nan values to be ignored. q : array_like of float - Percentile or sequence of percentiles to compute, which must be between - 0 and 100 inclusive. + Percentile or sequence of percentiles to compute, which must be + between 0 and 100 inclusive. axis : {int, tuple of int, None}, optional - Axis or axes along which the percentiles are computed. The - default is to compute the percentile(s) along a flattened - version of the array. + Axis or axes along which the percentiles are computed. The default + is to compute the percentile(s) along a flattened version of the + array. out : ndarray, optional - Alternative output array in which to place the result. It must - have the same shape and buffer length as the expected output, - but the type (of the output) will be cast if necessary. + Alternative output array in which to place the result. It must have + the same shape and buffer length as the expected output, but the + type (of the output) will be cast if necessary. overwrite_input : bool, optional - If True, then allow the input array `a` to be modified by intermediate - calculations, to save memory. In this case, the contents of the input - `a` after this function completes is undefined. - interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'} - This optional parameter specifies the interpolation method to - use when the desired percentile lies between two data points - ``i < j``: - - * 'linear': ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * 'lower': ``i``. - * 'higher': ``j``. - * 'nearest': ``i`` or ``j``, whichever is nearest. - * 'midpoint': ``(i + j) / 2``. + If True, then allow the input array `a` to be modified by + intermediate calculations, to save memory. In this case, the + contents of the input `a` after this function completes is + undefined. + method : str, optional + This parameter specifies the method to use for estimating the + percentile. There are many different methods, some unique to NumPy. + See the notes for explanation. The options sorted by their R type + as summarized in the H&F paper [1]_ are: + + 1. 'inverted_cdf' + 2. 'averaged_inverted_cdf' + 3. 'closest_observation' + 4. 'interpolated_inverted_cdf' + 5. 'hazen' + 6. 'weibull' + 7. 'linear' (default) + 8. 'median_unbiased' + 9. 'normal_unbiased' + + The first three methods are discontinuous. NumPy further defines the + following discontinuous variations of the default 'linear' (7.) option: + + * 'lower' + * 'higher', + * 'midpoint' + * 'nearest' + + .. versionchanged:: 1.22.0 + This argument was previously called "interpolation" and only + offered the "linear" default and last four options. + keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the @@ -1177,6 +1305,11 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, a sub-class and `mean` does not have the kwarg `keepdims` this will raise a RuntimeError. + interpolation : str, optional + Deprecated name for the method keyword argument. + + .. deprecated:: 1.22.0 + Returns ------- percentile : scalar or ndarray @@ -1194,18 +1327,11 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, nanmean nanmedian : equivalent to ``nanpercentile(..., 50)`` percentile, median, mean - nanquantile : equivalent to nanpercentile, but with q in the range [0, 1]. + nanquantile : equivalent to nanpercentile, except q in range [0, 1]. Notes ----- - Given a vector ``V`` of length ``N``, the ``q``-th percentile of - ``V`` is the value ``q/100`` of the way from the minimum to the - maximum in a sorted copy of ``V``. The values and distances of - the two nearest neighbors as well as the `interpolation` parameter - will determine the percentile if the normalized ranking does not - match the location of ``q`` exactly. This function is the same as - the median if ``q=50``, the same as the minimum if ``q=0`` and the - same as the maximum if ``q=100``. + For more information please see `numpy.percentile` Examples -------- @@ -1235,28 +1361,52 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, array([7., 2.]) >>> assert not np.all(a==b) + References + ---------- + .. [1] R. J. Hyndman and Y. Fan, + "Sample quantiles in statistical packages," + The American Statistician, 50(4), pp. 361-365, 1996 + """ + if interpolation is not None: + method = function_base._check_interpolation_as_method( + method, interpolation, "nanpercentile") + a = np.asanyarray(a) - q = np.true_divide(q, 100.0) # handles the asarray for us too + if a.dtype.kind == "c": + raise TypeError("a must be an array of real numbers") + + q = np.true_divide(q, 100.0) + # undo any decay that the ufunc performed (see gh-13105) + q = np.asanyarray(q) if not function_base._quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") return _nanquantile_unchecked( - a, q, axis, out, overwrite_input, interpolation, keepdims) + a, q, axis, out, overwrite_input, method, keepdims) def _nanquantile_dispatcher(a, q, axis=None, out=None, overwrite_input=None, - interpolation=None, keepdims=None): + method=None, keepdims=None, *, interpolation=None): return (a, q, out) @array_function_dispatch(_nanquantile_dispatcher) -def nanquantile(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear', keepdims=np._NoValue): +def nanquantile( + a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=np._NoValue, + *, + interpolation=None, +): """ Compute the qth quantile of the data along the specified axis, while ignoring nan values. Returns the qth quantile(s) of the array elements. - + .. versionadded:: 1.15.0 Parameters @@ -1265,8 +1415,8 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, Input array or object that can be converted to an array, containing nan values to be ignored q : array_like of float - Quantile or sequence of quantiles to compute, which must be between - 0 and 1 inclusive. + Probability or sequence of probabilities for the quantiles to compute. + Values must be between 0 and 1 inclusive. axis : {int, tuple of int, None}, optional Axis or axes along which the quantiles are computed. The default is to compute the quantile(s) along a flattened @@ -1279,18 +1429,33 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, If True, then allow the input array `a` to be modified by intermediate calculations, to save memory. In this case, the contents of the input `a` after this function completes is undefined. - interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'} - This optional parameter specifies the interpolation method to - use when the desired quantile lies between two data points - ``i < j``: - - * linear: ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * lower: ``i``. - * higher: ``j``. - * nearest: ``i`` or ``j``, whichever is nearest. - * midpoint: ``(i + j) / 2``. + method : str, optional + This parameter specifies the method to use for estimating the + quantile. There are many different methods, some unique to NumPy. + See the notes for explanation. The options sorted by their R type + as summarized in the H&F paper [1]_ are: + + 1. 'inverted_cdf' + 2. 'averaged_inverted_cdf' + 3. 'closest_observation' + 4. 'interpolated_inverted_cdf' + 5. 'hazen' + 6. 'weibull' + 7. 'linear' (default) + 8. 'median_unbiased' + 9. 'normal_unbiased' + + The first three methods are discontinuous. NumPy further defines the + following discontinuous variations of the default 'linear' (7.) option: + + * 'lower' + * 'higher', + * 'midpoint' + * 'nearest' + + .. versionchanged:: 1.22.0 + This argument was previously called "interpolation" and only + offered the "linear" default and last four options. keepdims : bool, optional If this is set to True, the axes which are reduced are left in @@ -1303,11 +1468,16 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, a sub-class and `mean` does not have the kwarg `keepdims` this will raise a RuntimeError. + interpolation : str, optional + Deprecated name for the method keyword argument. + + .. deprecated:: 1.22.0 + Returns ------- quantile : scalar or ndarray - If `q` is a single percentile and `axis=None`, then the result - is a scalar. If multiple quantiles are given, first axis of + If `q` is a single probability and `axis=None`, then the result + is a scalar. If multiple probability levels are given, first axis of the result corresponds to the quantiles. The other axes are the axes that remain after the reduction of `a`. If the input contains integers or floats smaller than ``float64``, the output @@ -1322,6 +1492,10 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, nanmedian : equivalent to ``nanquantile(..., 0.5)`` nanpercentile : same as nanquantile, but with q in the range [0, 100]. + Notes + ----- + For more information please see `numpy.quantile` + Examples -------- >>> a = np.array([[10., 7., 4.], [3., 2., 1.]]) @@ -1348,35 +1522,56 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, >>> np.nanquantile(b, 0.5, axis=1, overwrite_input=True) array([7., 2.]) >>> assert not np.all(a==b) + + References + ---------- + .. [1] R. J. Hyndman and Y. Fan, + "Sample quantiles in statistical packages," + The American Statistician, 50(4), pp. 361-365, 1996 + """ + + if interpolation is not None: + method = function_base._check_interpolation_as_method( + method, interpolation, "nanquantile") + a = np.asanyarray(a) + if a.dtype.kind == "c": + raise TypeError("a must be an array of real numbers") + q = np.asanyarray(q) if not function_base._quantile_is_valid(q): raise ValueError("Quantiles must be in the range [0, 1]") return _nanquantile_unchecked( - a, q, axis, out, overwrite_input, interpolation, keepdims) - - -def _nanquantile_unchecked(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear', keepdims=np._NoValue): + a, q, axis, out, overwrite_input, method, keepdims) + + +def _nanquantile_unchecked( + a, + q, + axis=None, + out=None, + overwrite_input=False, + method="linear", + keepdims=np._NoValue, +): """Assumes that q is in [0, 1], and is an ndarray""" # apply_along_axis in _nanpercentile doesn't handle empty arrays well, # so deal them upfront if a.size == 0: return np.nanmean(a, axis, out=out, keepdims=keepdims) - - r, k = function_base._ureduce( - a, func=_nanquantile_ureduce_func, q=q, axis=axis, out=out, - overwrite_input=overwrite_input, interpolation=interpolation - ) - if keepdims and keepdims is not np._NoValue: - return r.reshape(q.shape + k) - else: - return r + return function_base._ureduce(a, + func=_nanquantile_ureduce_func, + q=q, + keepdims=keepdims, + axis=axis, + out=out, + overwrite_input=overwrite_input, + method=method) def _nanquantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, - interpolation='linear'): + method="linear"): """ Private function that doesn't support extended axis or keepdims. These methods are extended to this function using _ureduce @@ -1384,10 +1579,10 @@ def _nanquantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, """ if axis is None or a.ndim == 1: part = a.ravel() - result = _nanquantile_1d(part, q, overwrite_input, interpolation) + result = _nanquantile_1d(part, q, overwrite_input, method) else: result = np.apply_along_axis(_nanquantile_1d, axis, a, q, - overwrite_input, interpolation) + overwrite_input, method) # apply_along_axis fills in collapsed axis with results. # Move that axis to the beginning to match percentile's # convention. @@ -1399,7 +1594,7 @@ def _nanquantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, return result -def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'): +def _nanquantile_1d(arr1d, q, overwrite_input=False, method="linear"): """ Private function for rank 1 arrays. Compute quantile ignoring NaNs. See nanpercentile for parameter usage @@ -1407,19 +1602,21 @@ def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'): arr1d, overwrite_input = _remove_nan_1d(arr1d, overwrite_input=overwrite_input) if arr1d.size == 0: - return np.full(q.shape, np.nan)[()] # convert to scalar + # convert to scalar + return np.full(q.shape, np.nan, dtype=arr1d.dtype)[()] return function_base._quantile_unchecked( - arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) + arr1d, q, overwrite_input=overwrite_input, method=method) -def _nanvar_dispatcher( - a, axis=None, dtype=None, out=None, ddof=None, keepdims=None): +def _nanvar_dispatcher(a, axis=None, dtype=None, out=None, ddof=None, + keepdims=None, *, where=None): return (a, out) @array_function_dispatch(_nanvar_dispatcher) -def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): +def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue, + *, where=np._NoValue): """ Compute the variance along the specified axis, while ignoring NaNs. @@ -1456,7 +1653,11 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original `a`. + where : array_like of bool, optional + Elements to include in the variance. See `~numpy.ufunc.reduce` for + details. + .. versionadded:: 1.22.0 Returns ------- @@ -1472,7 +1673,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): mean : Average var : Variance while not ignoring NaNs nanstd, nanmean - ufuncs-output-type + :ref:`ufuncs-output-type` Notes ----- @@ -1512,7 +1713,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): arr, mask = _replace_nan(a, 0) if mask is None: return np.var(arr, axis=axis, dtype=dtype, out=out, ddof=ddof, - keepdims=keepdims) + keepdims=keepdims, where=where) if dtype is not None: dtype = np.dtype(dtype) @@ -1531,21 +1732,29 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): # keepdims=True, however matrix now raises an error in this case, but # the reason that it drops the keepdims kwarg is to force keepdims=True # so this used to work by serendipity. - cnt = np.sum(~mask, axis=axis, dtype=np.intp, keepdims=_keepdims) - avg = np.sum(arr, axis=axis, dtype=dtype, keepdims=_keepdims) + cnt = np.sum(~mask, axis=axis, dtype=np.intp, keepdims=_keepdims, + where=where) + avg = np.sum(arr, axis=axis, dtype=dtype, keepdims=_keepdims, where=where) avg = _divide_by_count(avg, cnt) # Compute squared deviation from mean. - np.subtract(arr, avg, out=arr, casting='unsafe') + np.subtract(arr, avg, out=arr, casting='unsafe', where=where) arr = _copyto(arr, 0, mask) if issubclass(arr.dtype.type, np.complexfloating): - sqr = np.multiply(arr, arr.conj(), out=arr).real + sqr = np.multiply(arr, arr.conj(), out=arr, where=where).real else: - sqr = np.multiply(arr, arr, out=arr) + sqr = np.multiply(arr, arr, out=arr, where=where) # Compute variance. - var = np.sum(sqr, axis=axis, dtype=dtype, out=out, keepdims=keepdims) - if var.ndim < cnt.ndim: + var = np.sum(sqr, axis=axis, dtype=dtype, out=out, keepdims=keepdims, + where=where) + + # Precaution against reduced object arrays + try: + var_ndim = var.ndim + except AttributeError: + var_ndim = np.ndim(var) + if var_ndim < cnt.ndim: # Subclasses of ndarray may ignore keepdims, so check here. cnt = cnt.squeeze(axis) dof = cnt - ddof @@ -1554,20 +1763,21 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): isbad = (dof <= 0) if np.any(isbad): warnings.warn("Degrees of freedom <= 0 for slice.", RuntimeWarning, - stacklevel=3) + stacklevel=2) # NaN, inf, or negative numbers are all possible bad # values, so explicitly replace them with NaN. var = _copyto(var, np.nan, isbad) return var -def _nanstd_dispatcher( - a, axis=None, dtype=None, out=None, ddof=None, keepdims=None): +def _nanstd_dispatcher(a, axis=None, dtype=None, out=None, ddof=None, + keepdims=None, *, where=None): return (a, out) @array_function_dispatch(_nanstd_dispatcher) -def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): +def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue, + *, where=np._NoValue): """ Compute the standard deviation along the specified axis, while ignoring NaNs. @@ -1611,6 +1821,11 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): as-is to the relevant functions of the sub-classes. If these functions do not have a `keepdims` kwarg, a RuntimeError will be raised. + where : array_like of bool, optional + Elements to include in the standard deviation. + See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.22.0 Returns ------- @@ -1624,7 +1839,7 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): -------- var, mean, std nanvar, nanmean - ufuncs-output-type + :ref:`ufuncs-output-type` Notes ----- @@ -1662,9 +1877,11 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): """ var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - keepdims=keepdims) + keepdims=keepdims, where=where) if isinstance(var, np.ndarray): std = np.sqrt(var, out=var) - else: + elif hasattr(var, 'dtype'): std = var.dtype.type(np.sqrt(var)) + else: + std = np.sqrt(var) return std diff --git a/numpy/lib/nanfunctions.pyi b/numpy/lib/nanfunctions.pyi new file mode 100644 index 000000000..8642055fe --- /dev/null +++ b/numpy/lib/nanfunctions.pyi @@ -0,0 +1,38 @@ +from numpy.core.fromnumeric import ( + amin, + amax, + argmin, + argmax, + sum, + prod, + cumsum, + cumprod, + mean, + var, + std +) + +from numpy.lib.function_base import ( + median, + percentile, + quantile, +) + +__all__: list[str] + +# NOTE: In reaility these functions are not aliases but distinct functions +# with identical signatures. +nanmin = amin +nanmax = amax +nanargmin = argmin +nanargmax = argmax +nansum = sum +nanprod = prod +nancumsum = cumsum +nancumprod = cumprod +nanmean = mean +nanvar = var +nanstd = std +nanmedian = median +nanpercentile = percentile +nanquantile = quantile diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 58affc2fc..22fb0eb7d 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -1,4 +1,3 @@ -import sys import os import re import functools @@ -6,7 +5,8 @@ import itertools import warnings import weakref import contextlib -from operator import itemgetter, index as opindex +import operator +from operator import itemgetter, index as opindex, methodcaller from collections.abc import Mapping import numpy as np @@ -14,8 +14,8 @@ from . import format from ._datasource import DataSource from numpy.core import overrides from numpy.core.multiarray import packbits, unpackbits -from numpy.core.overrides import set_module -from numpy.core._internal import recursive +from numpy.core._multiarray_umath import _load_from_filelike +from numpy.core.overrides import set_array_function_like_doc, set_module from ._iotools import ( LineSplitter, NameValidator, StringConverter, ConverterError, ConverterLockError, ConversionWarning, _is_string_like, @@ -23,23 +23,14 @@ from ._iotools import ( ) from numpy.compat import ( - asbytes, asstr, asunicode, bytes, os_fspath, os_PathLike, - pickle, contextlib_nullcontext + asbytes, asstr, asunicode, os_fspath, os_PathLike, + pickle ) -@set_module('numpy') -def loads(*args, **kwargs): - # NumPy 1.15.0, 2017-12-10 - warnings.warn( - "np.loads is deprecated, use pickle.loads instead", - DeprecationWarning, stacklevel=2) - return pickle.loads(*args, **kwargs) - - __all__ = [ - 'savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt', - 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez', + 'savetxt', 'loadtxt', 'genfromtxt', + 'recfromtxt', 'recfromcsv', 'load', 'save', 'savez', 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource' ] @@ -86,7 +77,7 @@ class BagObj: try: return object.__getattribute__(self, '_obj')[key] except KeyError: - raise AttributeError(key) + raise AttributeError(key) from None def __dir__(self): """ @@ -148,6 +139,12 @@ class NpzFile(Mapping): Additional keyword arguments to pass on to pickle.load. These are only useful when loading object arrays saved on Python 2 when using Python 3. + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. + This option is ignored when `allow_pickle` is passed. In that case + the file is by definition trusted and the limit is unnecessary. Parameters ---------- @@ -168,8 +165,10 @@ class NpzFile(Mapping): >>> _ = outfile.seek(0) >>> npz = np.load(outfile) - >>> isinstance(npz, np.lib.io.NpzFile) + >>> isinstance(npz, np.lib.npyio.NpzFile) True + >>> npz + NpzFile 'object' with keys x, y >>> sorted(npz.files) ['x', 'y'] >>> npz['x'] # getitem access @@ -181,15 +180,18 @@ class NpzFile(Mapping): # Make __exit__ safe if zipfile_factory raises an exception zip = None fid = None + _MAX_REPR_ARRAY_COUNT = 5 def __init__(self, fid, own_fid=False, allow_pickle=False, - pickle_kwargs=None): + pickle_kwargs=None, *, + max_header_size=format._MAX_HEADER_SIZE): # Import is postponed to here since zipfile depends on gzip, an # optional component of the so-called standard library. _zip = zipfile_factory(fid) self._files = _zip.namelist() self.files = [] self.allow_pickle = allow_pickle + self.max_header_size = max_header_size self.pickle_kwargs = pickle_kwargs for x in self._files: if x.endswith('.npy'): @@ -253,37 +255,33 @@ class NpzFile(Mapping): bytes = self.zip.open(key) return format.read_array(bytes, allow_pickle=self.allow_pickle, - pickle_kwargs=self.pickle_kwargs) + pickle_kwargs=self.pickle_kwargs, + max_header_size=self.max_header_size) else: return self.zip.read(key) else: raise KeyError("%s is not a file in the archive" % key) + def __contains__(self, key): + return (key in self._files or key in self.files) - # deprecate the python 2 dict apis that we supported by accident in - # python 3. We forgot to implement itervalues() at all in earlier - # versions of numpy, so no need to deprecated it here. - - def iteritems(self): - # Numpy 1.15, 2018-02-20 - warnings.warn( - "NpzFile.iteritems is deprecated in python 3, to match the " - "removal of dict.itertems. Use .items() instead.", - DeprecationWarning, stacklevel=2) - return self.items() + def __repr__(self): + # Get filename or default to `object` + if isinstance(self.fid, str): + filename = self.fid + else: + filename = getattr(self.fid, "name", "object") - def iterkeys(self): - # Numpy 1.15, 2018-02-20 - warnings.warn( - "NpzFile.iterkeys is deprecated in python 3, to match the " - "removal of dict.iterkeys. Use .keys() instead.", - DeprecationWarning, stacklevel=2) - return self.keys() + # Get the name of arrays + array_names = ', '.join(self.files[:self._MAX_REPR_ARRAY_COUNT]) + if len(self.files) > self._MAX_REPR_ARRAY_COUNT: + array_names += "..." + return f"NpzFile {filename!r} with keys: {array_names}" @set_module('numpy') def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, - encoding='ASCII'): + encoding='ASCII', *, max_header_size=format._MAX_HEADER_SIZE): """ Load arrays or pickled objects from ``.npy``, ``.npz`` or pickled files. @@ -297,7 +295,8 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, ---------- file : file-like object, string, or pathlib.Path The file to read. File-like objects must support the - ``seek()`` and ``read()`` methods. Pickled files require that the + ``seek()`` and ``read()`` methods and must always + be opened in binary mode. Pickled files require that the file-like object support the ``readline()`` method as well. mmap_mode : {None, 'r+', 'r', 'w+', 'c'}, optional If not None, then memory-map the file, using the given mode (see @@ -326,6 +325,12 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, npy/npz files containing object arrays. Values other than 'latin1', 'ASCII', and 'bytes' are not allowed, as they can corrupt numerical data. Default: 'ASCII' + max_header_size : int, optional + Maximum allowed size of the header. Large headers may not be safe + to load securely and thus require explicitly passing a larger value. + See :py:func:`ast.literal_eval()` for details. + This option is ignored when `allow_pickle` is passed. In that case + the file is by definition trusted and the limit is unnecessary. Returns ------- @@ -335,10 +340,15 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, Raises ------ - IOError + OSError If the input file does not exist or cannot be read. + UnpicklingError + If ``allow_pickle=True``, but the file cannot be loaded as a pickle. ValueError - The file contains an object array, but allow_pickle=False given. + The file contains an object array, but ``allow_pickle=False`` given. + EOFError + When calling ``np.load`` multiple times on the same file handle, + if all data has already been read See Also -------- @@ -422,6 +432,8 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, _ZIP_SUFFIX = b'PK\x05\x06' # empty zip files start with this N = len(format.MAGIC_PREFIX) magic = fid.read(N) + if not magic: + raise EOFError("No data left in file") # If the file size is less than N, we need to make sure not # to seek past the beginning of the file fid.seek(-min(N, len(magic)), 1) # back-up @@ -430,15 +442,20 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, # Potentially transfer file ownership to NpzFile stack.pop_all() ret = NpzFile(fid, own_fid=own_fid, allow_pickle=allow_pickle, - pickle_kwargs=pickle_kwargs) + pickle_kwargs=pickle_kwargs, + max_header_size=max_header_size) return ret elif magic == format.MAGIC_PREFIX: # .npy file if mmap_mode: - return format.open_memmap(file, mode=mmap_mode) + if allow_pickle: + max_header_size = 2**64 + return format.open_memmap(file, mode=mmap_mode, + max_header_size=max_header_size) else: return format.read_array(fid, allow_pickle=allow_pickle, - pickle_kwargs=pickle_kwargs) + pickle_kwargs=pickle_kwargs, + max_header_size=max_header_size) else: # Try a pickle if not allow_pickle: @@ -446,9 +463,9 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) - except Exception: - raise IOError( - "Failed to interpret file %s as a pickle" % repr(file)) + except Exception as e: + raise pickle.UnpicklingError( + f"Failed to interpret file {file!r} as a pickle") from e def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None): @@ -517,7 +534,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): # [1 2] [1 3] """ if hasattr(file, 'write'): - file_ctx = contextlib_nullcontext(file) + file_ctx = contextlib.nullcontext(file) else: file = os_fspath(file) if not file.endswith('.npy'): @@ -539,10 +556,11 @@ def _savez_dispatcher(file, *args, **kwds): def savez(file, *args, **kwds): """Save several arrays into a single file in uncompressed ``.npz`` format. - If arguments are passed in with no keywords, the corresponding variable - names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword - arguments are given, the corresponding variable names, in the ``.npz`` - file will match the keyword names. + Provide arrays as keyword arguments to store them under the + corresponding name in the output file: ``savez(fn, x=x, y=y)``. + + If arrays are specified as positional arguments, i.e., ``savez(fn, + x, y)``, their names will be `arr_0`, `arr_1`, etc. Parameters ---------- @@ -552,13 +570,12 @@ def savez(file, *args, **kwds): ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional - Arrays to save to the file. Since it is not possible for Python to - know the names of the arrays outside `savez`, the arrays will be saved - with names "arr_0", "arr_1", and so on. These arguments can be any - expression. + Arrays to save to the file. Please use keyword arguments (see + `kwds` below) to assign names to arrays. Arrays specified as + args will be named "arr_0", "arr_1", and so on. kwds : Keyword arguments, optional - Arrays to save to the file. Arrays will be saved in the file with the - keyword names. + Arrays to save to the file. Each array will be saved to the + output file with its corresponding keyword name. Returns ------- @@ -582,9 +599,13 @@ def savez(file, *args, **kwds): its list of arrays (with the ``.files`` attribute), and for the arrays themselves. - When saving dictionaries, the dictionary keys become filenames - inside the ZIP archive. Therefore, keys should be valid filenames. - E.g., avoid keys that begin with ``/`` or contain ``.``. + Keys passed in `kwds` are used as filenames inside the ZIP archive. + Therefore, keys should be valid filenames; e.g., avoid keys that begin with + ``/`` or contain ``.``. + + When naming variables with keyword arguments, it is not possible to name a + variable ``file``, as this would cause the ``file`` argument to be defined + twice in the call to ``savez``. Examples -------- @@ -613,6 +634,7 @@ def savez(file, *args, **kwds): ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + """ _savez(file, args, kwds, False) @@ -627,9 +649,11 @@ def savez_compressed(file, *args, **kwds): """ Save several arrays into a single file in compressed ``.npz`` format. - If keyword arguments are given, then filenames are taken from the keywords. - If arguments are passed in with no keywords, then stored filenames are - arr_0, arr_1, etc. + Provide arrays as keyword arguments to store them under the + corresponding name in the output file: ``savez(fn, x=x, y=y)``. + + If arrays are specified as positional arguments, i.e., ``savez(fn, + x, y)``, their names will be `arr_0`, `arr_1`, etc. Parameters ---------- @@ -639,13 +663,12 @@ def savez_compressed(file, *args, **kwds): ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional - Arrays to save to the file. Since it is not possible for Python to - know the names of the arrays outside `savez`, the arrays will be saved - with names "arr_0", "arr_1", and so on. These arguments can be any - expression. + Arrays to save to the file. Please use keyword arguments (see + `kwds` below) to assign names to arrays. Arrays specified as + args will be named "arr_0", "arr_1", and so on. kwds : Keyword arguments, optional - Arrays to save to the file. Arrays will be saved in the file with the - keyword names. + Arrays to save to the file. Each array will be saved to the + output file with its corresponding keyword name. Returns ------- @@ -712,119 +735,411 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None): zipf = zipfile_factory(file, mode="w", compression=compression) - if sys.version_info >= (3, 6): - # Since Python 3.6 it is possible to write directly to a ZIP file. - for key, val in namedict.items(): - fname = key + '.npy' - val = np.asanyarray(val) - # always force zip64, gh-10776 - with zipf.open(fname, 'w', force_zip64=True) as fid: - format.write_array(fid, val, - allow_pickle=allow_pickle, - pickle_kwargs=pickle_kwargs) - else: - # Stage arrays in a temporary file on disk, before writing to zip. - - # Import deferred for startup time improvement - import tempfile - # Since target file might be big enough to exceed capacity of a global - # temporary directory, create temp file side-by-side with the target file. - file_dir, file_prefix = os.path.split(file) if _is_string_like(file) else (None, 'tmp') - fd, tmpfile = tempfile.mkstemp(prefix=file_prefix, dir=file_dir, suffix='-numpy.npy') - os.close(fd) - try: - for key, val in namedict.items(): - fname = key + '.npy' - fid = open(tmpfile, 'wb') - try: - format.write_array(fid, np.asanyarray(val), - allow_pickle=allow_pickle, - pickle_kwargs=pickle_kwargs) - fid.close() - fid = None - zipf.write(tmpfile, arcname=fname) - except IOError as exc: - raise IOError("Failed to write to %s: %s" % (tmpfile, exc)) - finally: - if fid: - fid.close() - finally: - os.remove(tmpfile) + for key, val in namedict.items(): + fname = key + '.npy' + val = np.asanyarray(val) + # always force zip64, gh-10776 + with zipf.open(fname, 'w', force_zip64=True) as fid: + format.write_array(fid, val, + allow_pickle=allow_pickle, + pickle_kwargs=pickle_kwargs) zipf.close() -def _getconv(dtype): - """ Find the correct dtype converter. Adapted from matplotlib """ - - def floatconv(x): - x.lower() - if '0x' in x: - return float.fromhex(x) - return float(x) - - typ = dtype.type - if issubclass(typ, np.bool_): - return lambda x: bool(int(x)) - if issubclass(typ, np.uint64): - return np.uint64 - if issubclass(typ, np.int64): - return np.int64 - if issubclass(typ, np.integer): - return lambda x: int(float(x)) - elif issubclass(typ, np.longdouble): - return np.longdouble - elif issubclass(typ, np.floating): - return floatconv - elif issubclass(typ, complex): - return lambda x: complex(asstr(x).replace('+-', '-')) - elif issubclass(typ, np.bytes_): - return asbytes - elif issubclass(typ, np.unicode_): - return asunicode - else: - return asstr +def _ensure_ndmin_ndarray_check_param(ndmin): + """Just checks if the param ndmin is supported on + _ensure_ndmin_ndarray. It is intended to be used as + verification before running anything expensive. + e.g. loadtxt, genfromtxt + """ + # Check correctness of the values of `ndmin` + if ndmin not in [0, 1, 2]: + raise ValueError(f"Illegal value of ndmin keyword: {ndmin}") + +def _ensure_ndmin_ndarray(a, *, ndmin: int): + """This is a helper function of loadtxt and genfromtxt to ensure + proper minimum dimension as requested + + ndim : int. Supported values 1, 2, 3 + ^^ whenever this changes, keep in sync with + _ensure_ndmin_ndarray_check_param + """ + # Verify that the array has at least dimensions `ndmin`. + # Tweak the size and shape of the arrays - remove extraneous dimensions + if a.ndim > ndmin: + a = np.squeeze(a) + # and ensure we have the minimum number of dimensions asked for + # - has to be in this order for the odd case ndmin=1, a.squeeze().ndim=0 + if a.ndim < ndmin: + if ndmin == 1: + a = np.atleast_1d(a) + elif ndmin == 2: + a = np.atleast_2d(a).T + + return a # amount of lines loadtxt reads in one chunk, can be overridden for testing _loadtxt_chunksize = 50000 +def _check_nonneg_int(value, name="argument"): + try: + operator.index(value) + except TypeError: + raise TypeError(f"{name} must be an integer") from None + if value < 0: + raise ValueError(f"{name} must be nonnegative") + + +def _preprocess_comments(iterable, comments, encoding): + """ + Generator that consumes a line iterated iterable and strips out the + multiple (or multi-character) comments from lines. + This is a pre-processing step to achieve feature parity with loadtxt + (we assume that this feature is a nieche feature). + """ + for line in iterable: + if isinstance(line, bytes): + # Need to handle conversion here, or the splitting would fail + line = line.decode(encoding) + + for c in comments: + line = line.split(c, 1)[0] + + yield line + + +# The number of rows we read in one go if confronted with a parametric dtype +_loadtxt_chunksize = 50000 + + +def _read(fname, *, delimiter=',', comment='#', quote='"', + imaginary_unit='j', usecols=None, skiplines=0, + max_rows=None, converters=None, ndmin=None, unpack=False, + dtype=np.float64, encoding="bytes"): + r""" + Read a NumPy array from a text file. + + Parameters + ---------- + fname : str or file object + The filename or the file to be read. + delimiter : str, optional + Field delimiter of the fields in line of the file. + Default is a comma, ','. If None any sequence of whitespace is + considered a delimiter. + comment : str or sequence of str or None, optional + Character that begins a comment. All text from the comment + character to the end of the line is ignored. + Multiple comments or multiple-character comment strings are supported, + but may be slower and `quote` must be empty if used. + Use None to disable all use of comments. + quote : str or None, optional + Character that is used to quote string fields. Default is '"' + (a double quote). Use None to disable quote support. + imaginary_unit : str, optional + Character that represent the imaginay unit `sqrt(-1)`. + Default is 'j'. + usecols : array_like, optional + A one-dimensional array of integer column numbers. These are the + columns from the file to be included in the array. If this value + is not given, all the columns are used. + skiplines : int, optional + Number of lines to skip before interpreting the data in the file. + max_rows : int, optional + Maximum number of rows of data to read. Default is to read the + entire file. + converters : dict or callable, optional + A function to parse all columns strings into the desired value, or + a dictionary mapping column number to a parser function. + E.g. if column 0 is a date string: ``converters = {0: datestr2num}``. + Converters can also be used to provide a default value for missing + data, e.g. ``converters = lambda s: float(s.strip() or 0)`` will + convert empty fields to 0. + Default: None + ndmin : int, optional + Minimum dimension of the array returned. + Allowed values are 0, 1 or 2. Default is 0. + unpack : bool, optional + If True, the returned array is transposed, so that arguments may be + unpacked using ``x, y, z = read(...)``. When used with a structured + data-type, arrays are returned for each field. Default is False. + dtype : numpy data type + A NumPy dtype instance, can be a structured dtype to map to the + columns of the file. + encoding : str, optional + Encoding used to decode the inputfile. The special value 'bytes' + (the default) enables backwards-compatible behavior for `converters`, + ensuring that inputs to the converter functions are encoded + bytes objects. The special value 'bytes' has no additional effect if + ``converters=None``. If encoding is ``'bytes'`` or ``None``, the + default system encoding is used. + + Returns + ------- + ndarray + NumPy array. + + Examples + -------- + First we create a file for the example. + + >>> s1 = '1.0,2.0,3.0\n4.0,5.0,6.0\n' + >>> with open('example1.csv', 'w') as f: + ... f.write(s1) + >>> a1 = read_from_filename('example1.csv') + >>> a1 + array([[1., 2., 3.], + [4., 5., 6.]]) + + The second example has columns with different data types, so a + one-dimensional array with a structured data type is returned. + The tab character is used as the field delimiter. + + >>> s2 = '1.0\t10\talpha\n2.3\t25\tbeta\n4.5\t16\tgamma\n' + >>> with open('example2.tsv', 'w') as f: + ... f.write(s2) + >>> a2 = read_from_filename('example2.tsv', delimiter='\t') + >>> a2 + array([(1. , 10, b'alpha'), (2.3, 25, b'beta'), (4.5, 16, b'gamma')], + dtype=[('f0', '<f8'), ('f1', 'u1'), ('f2', 'S5')]) + """ + # Handle special 'bytes' keyword for encoding + byte_converters = False + if encoding == 'bytes': + encoding = None + byte_converters = True + + if dtype is None: + raise TypeError("a dtype must be provided.") + dtype = np.dtype(dtype) + + read_dtype_via_object_chunks = None + if dtype.kind in 'SUM' and ( + dtype == "S0" or dtype == "U0" or dtype == "M8" or dtype == 'm8'): + # This is a legacy "flexible" dtype. We do not truly support + # parametric dtypes currently (no dtype discovery step in the core), + # but have to support these for backward compatibility. + read_dtype_via_object_chunks = dtype + dtype = np.dtype(object) + + if usecols is not None: + # Allow usecols to be a single int or a sequence of ints, the C-code + # handles the rest + try: + usecols = list(usecols) + except TypeError: + usecols = [usecols] + + _ensure_ndmin_ndarray_check_param(ndmin) + + if comment is None: + comments = None + else: + # assume comments are a sequence of strings + if "" in comment: + raise ValueError( + "comments cannot be an empty string. Use comments=None to " + "disable comments." + ) + comments = tuple(comment) + comment = None + if len(comments) == 0: + comments = None # No comments at all + elif len(comments) == 1: + # If there is only one comment, and that comment has one character, + # the normal parsing can deal with it just fine. + if isinstance(comments[0], str) and len(comments[0]) == 1: + comment = comments[0] + comments = None + else: + # Input validation if there are multiple comment characters + if delimiter in comments: + raise TypeError( + f"Comment characters '{comments}' cannot include the " + f"delimiter '{delimiter}'" + ) + + # comment is now either a 1 or 0 character string or a tuple: + if comments is not None: + # Note: An earlier version support two character comments (and could + # have been extended to multiple characters, we assume this is + # rare enough to not optimize for. + if quote is not None: + raise ValueError( + "when multiple comments or a multi-character comment is " + "given, quotes are not supported. In this case quotechar " + "must be set to None.") + + if len(imaginary_unit) != 1: + raise ValueError('len(imaginary_unit) must be 1.') + + _check_nonneg_int(skiplines) + if max_rows is not None: + _check_nonneg_int(max_rows) + else: + # Passing -1 to the C code means "read the entire file". + max_rows = -1 + + fh_closing_ctx = contextlib.nullcontext() + filelike = False + try: + if isinstance(fname, os.PathLike): + fname = os.fspath(fname) + if isinstance(fname, str): + fh = np.lib._datasource.open(fname, 'rt', encoding=encoding) + if encoding is None: + encoding = getattr(fh, 'encoding', 'latin1') + + fh_closing_ctx = contextlib.closing(fh) + data = fh + filelike = True + else: + if encoding is None: + encoding = getattr(fname, 'encoding', 'latin1') + data = iter(fname) + except TypeError as e: + raise ValueError( + f"fname must be a string, filehandle, list of strings,\n" + f"or generator. Got {type(fname)} instead.") from e + + with fh_closing_ctx: + if comments is not None: + if filelike: + data = iter(data) + filelike = False + data = _preprocess_comments(data, comments, encoding) + + if read_dtype_via_object_chunks is None: + arr = _load_from_filelike( + data, delimiter=delimiter, comment=comment, quote=quote, + imaginary_unit=imaginary_unit, + usecols=usecols, skiplines=skiplines, max_rows=max_rows, + converters=converters, dtype=dtype, + encoding=encoding, filelike=filelike, + byte_converters=byte_converters) + + else: + # This branch reads the file into chunks of object arrays and then + # casts them to the desired actual dtype. This ensures correct + # string-length and datetime-unit discovery (like `arr.astype()`). + # Due to chunking, certain error reports are less clear, currently. + if filelike: + data = iter(data) # cannot chunk when reading from file + + c_byte_converters = False + if read_dtype_via_object_chunks == "S": + c_byte_converters = True # Use latin1 rather than ascii + + chunks = [] + while max_rows != 0: + if max_rows < 0: + chunk_size = _loadtxt_chunksize + else: + chunk_size = min(_loadtxt_chunksize, max_rows) + + next_arr = _load_from_filelike( + data, delimiter=delimiter, comment=comment, quote=quote, + imaginary_unit=imaginary_unit, + usecols=usecols, skiplines=skiplines, max_rows=max_rows, + converters=converters, dtype=dtype, + encoding=encoding, filelike=filelike, + byte_converters=byte_converters, + c_byte_converters=c_byte_converters) + # Cast here already. We hope that this is better even for + # large files because the storage is more compact. It could + # be adapted (in principle the concatenate could cast). + chunks.append(next_arr.astype(read_dtype_via_object_chunks)) + + skiprows = 0 # Only have to skip for first chunk + if max_rows >= 0: + max_rows -= chunk_size + if len(next_arr) < chunk_size: + # There was less data than requested, so we are done. + break + + # Need at least one chunk, but if empty, the last one may have + # the wrong shape. + if len(chunks) > 1 and len(chunks[-1]) == 0: + del chunks[-1] + if len(chunks) == 1: + arr = chunks[0] + else: + arr = np.concatenate(chunks, axis=0) + + # NOTE: ndmin works as advertised for structured dtypes, but normally + # these would return a 1D result plus the structured dimension, + # so ndmin=2 adds a third dimension even when no squeezing occurs. + # A `squeeze=False` could be a better solution (pandas uses squeeze). + arr = _ensure_ndmin_ndarray(arr, ndmin=ndmin) + + if arr.shape: + if arr.shape[0] == 0: + warnings.warn( + f'loadtxt: input contained no data: "{fname}"', + category=UserWarning, + stacklevel=3 + ) + + if unpack: + # Unpack structured dtypes if requested: + dt = arr.dtype + if dt.names is not None: + # For structured arrays, return an array for each field. + return [arr[field] for field in dt.names] + else: + return arr.T + else: + return arr + + +@set_array_function_like_doc @set_module('numpy') def loadtxt(fname, dtype=float, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, - ndmin=0, encoding='bytes', max_rows=None): + ndmin=0, encoding='bytes', max_rows=None, *, quotechar=None, + like=None): r""" Load data from a text file. - Each row in the text file must have the same number of values. - Parameters ---------- - fname : file, str, or pathlib.Path - File, filename, or generator to read. If the filename extension is - ``.gz`` or ``.bz2``, the file is first decompressed. Note that - generators should return byte strings. + fname : file, str, pathlib.Path, list of str, generator + File, filename, list, or generator to read. If the filename + extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note + that generators must return bytes or strings. The strings + in a list or produced by a generator are treated as lines. dtype : data-type, optional Data-type of the resulting array; default: float. If this is a structured data-type, the resulting array will be 1-dimensional, and each row will be interpreted as an element of the array. In this case, the number of columns used must match the number of fields in the data-type. - comments : str or sequence of str, optional + comments : str or sequence of str or None, optional The characters or list of characters used to indicate the start of a comment. None implies no comments. For backwards compatibility, byte strings will be decoded as 'latin1'. The default is '#'. delimiter : str, optional - The string used to separate values. For backwards compatibility, byte - strings will be decoded as 'latin1'. The default is whitespace. - converters : dict, optional - A dictionary mapping column number to a function that will parse the - column string into the desired value. E.g., if column 0 is a date - string: ``converters = {0: datestr2num}``. Converters can also be - used to provide a default value for missing data (but see also - `genfromtxt`): ``converters = {3: lambda s: float(s.strip() or 0)}``. + The character used to separate the values. For backwards compatibility, + byte strings will be decoded as 'latin1'. The default is whitespace. + + .. versionchanged:: 1.23.0 + Only single character delimiters are supported. Newline characters + cannot be used as the delimiter. + + converters : dict or callable, optional + Converter functions to customize value parsing. If `converters` is + callable, the function is applied to all columns, else it must be a + dict that maps column number to a parser function. + See examples for further details. Default: None. + + .. versionchanged:: 1.23.0 + The ability to pass a single callable to be applied to all columns + was added. + skiprows : int, optional Skip the first `skiprows` lines, including comments; default: 0. usecols : int or sequence, optional @@ -838,8 +1153,9 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, fourth column the same way as ``usecols = (3,)`` would. unpack : bool, optional If True, the returned array is transposed, so that arguments may be - unpacked using ``x, y, z = loadtxt(...)``. When used with a structured - data-type, arrays are returned for each field. Default is False. + unpacked using ``x, y, z = loadtxt(...)``. When used with a + structured data-type, arrays are returned for each field. + Default is False. ndmin : int, optional The returned array will have at least `ndmin` dimensions. Otherwise mono-dimensional axes will be squeezed. @@ -856,11 +1172,31 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, .. versionadded:: 1.14.0 max_rows : int, optional - Read `max_rows` lines of content after `skiprows` lines. The default - is to read all the lines. + Read `max_rows` rows of content after `skiprows` lines. The default is + to read all the rows. Note that empty rows containing no data such as + empty lines and comment lines are not counted towards `max_rows`, + while such lines are counted in `skiprows`. .. versionadded:: 1.16.0 + .. versionchanged:: 1.23.0 + Lines containing no data, including comment lines (e.g., lines + starting with '#' or as specified via `comments`) are not counted + towards `max_rows`. + quotechar : unicode character or None, optional + The character used to denote the start and end of a quoted item. + Occurrences of the delimiter or comment characters are ignored within + a quoted item. The default value is ``quotechar=None``, which means + quoting support is disabled. + + If two consecutive instances of `quotechar` are found within a quoted + field, the first is treated as an escape character. See examples. + + .. versionadded:: 1.23.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 + Returns ------- out : ndarray @@ -878,6 +1214,11 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, `genfromtxt` function provides more sophisticated handling of, e.g., lines with missing values. + Each row in the input text file must have the same number of values to be + able to read all values. If all rows do not have same number of values, a + subset of up to n columns (where n is the least number of values present + in all rows) can be read by specifying the columns via `usecols`. + .. versionadded:: 1.10.0 The strings produced by the Python float.hex method can be used as @@ -904,6 +1245,29 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, >>> y array([2., 4.]) + The `converters` argument is used to specify functions to preprocess the + text prior to parsing. `converters` can be a dictionary that maps + preprocessing functions to each column: + + >>> s = StringIO("1.618, 2.296\n3.141, 4.669\n") + >>> conv = { + ... 0: lambda x: np.floor(float(x)), # conversion fn for column 0 + ... 1: lambda x: np.ceil(float(x)), # conversion fn for column 1 + ... } + >>> np.loadtxt(s, delimiter=",", converters=conv) + array([[1., 3.], + [3., 5.]]) + + `converters` can be a callable instead of a dictionary, in which case it + is applied to all columns: + + >>> s = StringIO("0xDE 0xAD\n0xC0 0xDE") + >>> import functools + >>> conv = functools.partial(int, base=16) + >>> np.loadtxt(s, converters=conv) + array([[222., 173.], + [192., 222.]]) + This example shows how `converters` can be used to convert a field with a trailing minus sign into a negative number. @@ -911,294 +1275,110 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, >>> def conv(fld): ... return -float(fld[:-1]) if fld.endswith(b'-') else float(fld) ... - >>> np.loadtxt(s, converters={0: conv, 1: conv}) + >>> np.loadtxt(s, converters=conv) array([[ 10.01, -31.25], [ 19.22, 64.31], [-17.57, 63.94]]) - """ - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Nested functions used by loadtxt. - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # not to be confused with the flatten_dtype we import... - @recursive - def flatten_dtype_internal(self, dt): - """Unpack a structured data-type, and produce re-packing info.""" - if dt.names is None: - # If the dtype is flattened, return. - # If the dtype has a shape, the dtype occurs - # in the list more than once. - shape = dt.shape - if len(shape) == 0: - return ([dt.base], None) - else: - packing = [(shape[-1], list)] - if len(shape) > 1: - for dim in dt.shape[-2::-1]: - packing = [(dim*packing[0][0], packing*dim)] - return ([dt.base] * int(np.prod(dt.shape)), packing) - else: - types = [] - packing = [] - for field in dt.names: - tp, bytes = dt.fields[field] - flat_dt, flat_packing = self(tp) - types.extend(flat_dt) - # Avoid extra nesting for subarrays - if tp.ndim > 0: - packing.extend(flat_packing) - else: - packing.append((len(flat_dt), flat_packing)) - return (types, packing) - - @recursive - def pack_items(self, items, packing): - """Pack items into nested lists based on re-packing info.""" - if packing is None: - return items[0] - elif packing is tuple: - return tuple(items) - elif packing is list: - return list(items) - else: - start = 0 - ret = [] - for length, subpacking in packing: - ret.append(self(items[start:start+length], subpacking)) - start += length - return tuple(ret) + Using a callable as the converter can be particularly useful for handling + values with different formatting, e.g. floats with underscores: - def split_line(line): - """Chop off comments, strip, and split at delimiter. """ - line = _decode_line(line, encoding=encoding) + >>> s = StringIO("1 2.7 100_000") + >>> np.loadtxt(s, converters=float) + array([1.e+00, 2.7e+00, 1.e+05]) - if comments is not None: - line = regex_comments.split(line, maxsplit=1)[0] - line = line.strip('\r\n') - if line: - return line.split(delimiter) - else: - return [] + This idea can be extended to automatically handle values specified in + many different formats: - def read_data(chunk_size): - """Parse each line, including the first. + >>> def conv(val): + ... try: + ... return float(val) + ... except ValueError: + ... return float.fromhex(val) + >>> s = StringIO("1, 2.5, 3_000, 0b4, 0x1.4000000000000p+2") + >>> np.loadtxt(s, delimiter=",", converters=conv, encoding=None) + array([1.0e+00, 2.5e+00, 3.0e+03, 1.8e+02, 5.0e+00]) - The file read, `fh`, is a global defined above. + Note that with the default ``encoding="bytes"``, the inputs to the + converter function are latin-1 encoded byte strings. To deactivate the + implicit encoding prior to conversion, use ``encoding=None`` - Parameters - ---------- - chunk_size : int - At most `chunk_size` lines are read at a time, with iteration - until all lines are read. + >>> s = StringIO('10.01 31.25-\n19.22 64.31\n17.57- 63.94') + >>> conv = lambda x: -float(x[:-1]) if x.endswith('-') else float(x) + >>> np.loadtxt(s, converters=conv, encoding=None) + array([[ 10.01, -31.25], + [ 19.22, 64.31], + [-17.57, 63.94]]) - """ - X = [] - line_iter = itertools.chain([first_line], fh) - line_iter = itertools.islice(line_iter, max_rows) - for i, line in enumerate(line_iter): - vals = split_line(line) - if len(vals) == 0: - continue - if usecols: - vals = [vals[j] for j in usecols] - if len(vals) != N: - line_num = i + skiprows + 1 - raise ValueError("Wrong number of columns at line %d" - % line_num) - - # Convert each value according to its column and store - items = [conv(val) for (conv, val) in zip(converters, vals)] - - # Then pack it according to the dtype's nesting - items = pack_items(items, packing) - X.append(items) - if len(X) > chunk_size: - yield X - X = [] - if X: - yield X - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Main body of loadtxt. - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Support for quoted fields is enabled with the `quotechar` parameter. + Comment and delimiter characters are ignored when they appear within a + quoted item delineated by `quotechar`: - # Check correctness of the values of `ndmin` - if ndmin not in [0, 1, 2]: - raise ValueError('Illegal value of ndmin keyword: %s' % ndmin) + >>> s = StringIO('"alpha, #42", 10.0\n"beta, #64", 2.0\n') + >>> dtype = np.dtype([("label", "U12"), ("value", float)]) + >>> np.loadtxt(s, dtype=dtype, delimiter=",", quotechar='"') + array([('alpha, #42', 10.), ('beta, #64', 2.)], + dtype=[('label', '<U12'), ('value', '<f8')]) - # Type conversions for Py3 convenience - if comments is not None: - if isinstance(comments, (str, bytes)): - comments = [comments] - comments = [_decode_line(x) for x in comments] - # Compile regex for comments beforehand - comments = (re.escape(comment) for comment in comments) - regex_comments = re.compile('|'.join(comments)) + Quoted fields can be separated by multiple whitespace characters: - if delimiter is not None: - delimiter = _decode_line(delimiter) + >>> s = StringIO('"alpha, #42" 10.0\n"beta, #64" 2.0\n') + >>> dtype = np.dtype([("label", "U12"), ("value", float)]) + >>> np.loadtxt(s, dtype=dtype, delimiter=None, quotechar='"') + array([('alpha, #42', 10.), ('beta, #64', 2.)], + dtype=[('label', '<U12'), ('value', '<f8')]) - user_converters = converters + Two consecutive quote characters within a quoted field are treated as a + single escaped character: - if encoding == 'bytes': - encoding = None - byte_converters = True - else: - byte_converters = False + >>> s = StringIO('"Hello, my name is ""Monty""!"') + >>> np.loadtxt(s, dtype="U", delimiter=",", quotechar='"') + array('Hello, my name is "Monty"!', dtype='<U26') - if usecols is not None: - # Allow usecols to be a single int or a sequence of ints - try: - usecols_as_list = list(usecols) - except TypeError: - usecols_as_list = [usecols] - for col_idx in usecols_as_list: - try: - opindex(col_idx) - except TypeError as e: - e.args = ( - "usecols must be an int or a sequence of ints but " - "it contains at least one element of type %s" % - type(col_idx), - ) - raise - # Fall back to existing code - usecols = usecols_as_list - - # Make sure we're dealing with a proper dtype - dtype = np.dtype(dtype) - defconv = _getconv(dtype) + Read subset of columns when all rows do not contain equal number of values: - dtype_types, packing = flatten_dtype_internal(dtype) + >>> d = StringIO("1 2\n2 4\n3 9 12\n4 16 20") + >>> np.loadtxt(d, usecols=(0, 1)) + array([[ 1., 2.], + [ 2., 4.], + [ 3., 9.], + [ 4., 16.]]) - fown = False - try: - if isinstance(fname, os_PathLike): - fname = os_fspath(fname) - if _is_string_like(fname): - fh = np.lib._datasource.open(fname, 'rt', encoding=encoding) - fencoding = getattr(fh, 'encoding', 'latin1') - fh = iter(fh) - fown = True - else: - fh = iter(fname) - fencoding = getattr(fname, 'encoding', 'latin1') - except TypeError as e: - raise ValueError( - 'fname must be a string, file handle, or generator' - ) from e + """ - # input may be a python2 io stream - if encoding is not None: - fencoding = encoding - # we must assume local encoding - # TODO emit portability warning? - elif fencoding is None: - import locale - fencoding = locale.getpreferredencoding() + if like is not None: + return _loadtxt_with_like( + like, fname, dtype=dtype, comments=comments, delimiter=delimiter, + converters=converters, skiprows=skiprows, usecols=usecols, + unpack=unpack, ndmin=ndmin, encoding=encoding, + max_rows=max_rows + ) - try: - # Skip the first `skiprows` lines - for i in range(skiprows): - next(fh) + if isinstance(delimiter, bytes): + delimiter.decode("latin1") - # Read until we find a line with some values, and use - # it to estimate the number of columns, N. - first_vals = None - try: - while not first_vals: - first_line = next(fh) - first_vals = split_line(first_line) - except StopIteration: - # End of lines reached - first_line = '' - first_vals = [] - warnings.warn('loadtxt: Empty input file: "%s"' % fname, - stacklevel=2) - N = len(usecols or first_vals) - - # Now that we know N, create the default converters list, and - # set packing, if necessary. - if len(dtype_types) > 1: - # We're dealing with a structured array, each field of - # the dtype matches a column - converters = [_getconv(dt) for dt in dtype_types] - else: - # All fields have the same dtype - converters = [defconv for i in range(N)] - if N > 1: - packing = [(N, tuple)] + if dtype is None: + dtype = np.float64 - # By preference, use the converters specified by the user - for i, conv in (user_converters or {}).items(): - if usecols: - try: - i = usecols.index(i) - except ValueError: - # Unused converter specified - continue - if byte_converters: - # converters may use decode to workaround numpy's old - # behaviour, so encode the string again before passing to - # the user converter - def tobytes_first(x, conv): - if type(x) is bytes: - return conv(x) - return conv(x.encode("latin1")) - converters[i] = functools.partial(tobytes_first, conv=conv) - else: - converters[i] = conv - - converters = [conv if conv is not bytes else - lambda x: x.encode(fencoding) for conv in converters] - - # read data in chunks and fill it into an array via resize - # over-allocating and shrinking the array later may be faster but is - # probably not relevant compared to the cost of actually reading and - # converting the data - X = None - for x in read_data(_loadtxt_chunksize): - if X is None: - X = np.array(x, dtype) - else: - nshape = list(X.shape) - pos = nshape[0] - nshape[0] += len(x) - X.resize(nshape, refcheck=False) - X[pos:, ...] = x - finally: - if fown: - fh.close() + comment = comments + # Control character type conversions for Py3 convenience + if comment is not None: + if isinstance(comment, (str, bytes)): + comment = [comment] + comment = [ + x.decode('latin1') if isinstance(x, bytes) else x for x in comment] + if isinstance(delimiter, bytes): + delimiter = delimiter.decode('latin1') - if X is None: - X = np.array([], dtype) + arr = _read(fname, dtype=dtype, comment=comment, delimiter=delimiter, + converters=converters, skiplines=skiprows, usecols=usecols, + unpack=unpack, ndmin=ndmin, encoding=encoding, + max_rows=max_rows, quote=quotechar) - # Multicolumn data are returned with shape (1, N, M), i.e. - # (1, 1, M) for a single row - remove the singleton dimension there - if X.ndim == 3 and X.shape[:2] == (1, 1): - X.shape = (1, -1) + return arr - # Verify that the array has at least dimensions `ndmin`. - # Tweak the size and shape of the arrays - remove extraneous dimensions - if X.ndim > ndmin: - X = np.squeeze(X) - # and ensure we have the minimum number of dimensions asked for - # - has to be in this order for the odd case ndmin=1, X.squeeze().ndim=0 - if X.ndim < ndmin: - if ndmin == 1: - X = np.atleast_1d(X) - elif ndmin == 2: - X = np.atleast_2d(X).T - if unpack: - if len(dtype_types) > 1: - # For structured arrays, return an array for each field. - return [X[field] for field in dtype.names] - else: - return X.T - else: - return X +_loadtxt_with_like = array_function_dispatch()(loadtxt) def _savetxt_dispatcher(fname, X, fmt=None, delimiter=None, newline=None, @@ -1441,10 +1621,10 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', for row in X: try: v = format % tuple(row) + newline - except TypeError: + except TypeError as e: raise TypeError("Mismatch between array dtype ('%s') and " "format specifier ('%s')" - % (str(X.dtype), format)) + % (str(X.dtype), format)) from e fh.write(v) if len(footer) > 0: @@ -1457,7 +1637,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', @set_module('numpy') def fromregex(file, regexp, dtype, encoding=None): - """ + r""" Construct an array from a text file, using regular expression parsing. The returned array is always a structured array, and is constructed from @@ -1466,13 +1646,16 @@ def fromregex(file, regexp, dtype, encoding=None): Parameters ---------- - file : str or file + file : path or file Filename or file object to read. + + .. versionchanged:: 1.22.0 + Now accepts `os.PathLike` implementations. regexp : str or regexp Regular expression used to parse the file. Groups in the regular expression correspond to fields in the dtype. dtype : dtype or list of dtypes - Dtype for the structured array. + Dtype for the structured array; must be a structured datatype. encoding : str, optional Encoding used to decode the inputfile. Does not apply to input streams. @@ -1497,16 +1680,15 @@ def fromregex(file, regexp, dtype, encoding=None): ----- Dtypes for structured arrays can be specified in several forms, but all forms specify at least the data type and field name. For details see - `doc.structured_arrays`. + `basics.rec`. Examples -------- - >>> f = open('test.dat', 'w') - >>> _ = f.write("1312 foo\\n1534 bar\\n444 qux") - >>> f.close() + >>> from io import StringIO + >>> text = StringIO("1312 foo\n1534 bar\n444 qux") - >>> regexp = r"(\\d+)\\s+(...)" # match [digits, whitespace, anything] - >>> output = np.fromregex('test.dat', regexp, + >>> regexp = r"(\d+)\s+(...)" # match [digits, whitespace, anything] + >>> output = np.fromregex(text, regexp, ... [('num', np.int64), ('key', 'S3')]) >>> output array([(1312, b'foo'), (1534, b'bar'), ( 444, b'qux')], @@ -1517,17 +1699,20 @@ def fromregex(file, regexp, dtype, encoding=None): """ own_fh = False if not hasattr(file, "read"): + file = os.fspath(file) file = np.lib._datasource.open(file, 'rt', encoding=encoding) own_fh = True try: if not isinstance(dtype, np.dtype): dtype = np.dtype(dtype) + if dtype.names is None: + raise TypeError('dtype must be a structured datatype.') content = file.read() - if isinstance(content, bytes) and isinstance(regexp, np.compat.unicode): + if isinstance(content, bytes) and isinstance(regexp, str): regexp = asbytes(regexp) - elif isinstance(content, np.compat.unicode) and isinstance(regexp, bytes): + elif isinstance(content, str) and isinstance(regexp, bytes): regexp = asstr(regexp) if not hasattr(regexp, 'match'): @@ -1554,6 +1739,7 @@ def fromregex(file, regexp, dtype, encoding=None): #####-------------------------------------------------------------------------- +@set_array_function_like_doc @set_module('numpy') def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skip_header=0, skip_footer=0, converters=None, @@ -1562,7 +1748,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, deletechars=''.join(sorted(NameValidator.defaultdeletechars)), replace_space='_', autostrip=False, case_sensitive=True, defaultfmt="f%i", unpack=None, usemask=False, loose=True, - invalid_raise=True, max_rows=None, encoding='bytes'): + invalid_raise=True, max_rows=None, encoding='bytes', + *, ndmin=0, like=None): """ Load data from a text file, with missing values handled as specified. @@ -1573,8 +1760,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, ---------- fname : file, str, pathlib.Path, list of str, generator File, filename, list, or generator to read. If the filename - extension is `.gz` or `.bz2`, the file is first decompressed. Note - that generators must return byte strings. The strings + extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note + that generators must return bytes or strings. The strings in a list or produced by a generator are treated as lines. dtype : dtype, optional Data type of the resulting array. @@ -1582,7 +1769,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, column, individually. comments : str, optional The character used to indicate the start of a comment. - All the characters occurring on a line after a comment are discarded + All the characters occurring on a line after a comment are discarded. delimiter : str, int, or sequence, optional The string used to separate values. By default, any consecutive whitespaces act as delimiter. An integer or sequence of integers @@ -1609,15 +1796,15 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, ``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns. names : {None, True, str, sequence}, optional If `names` is True, the field names are read from the first line after - the first `skip_header` lines. This line can optionally be proceeded + the first `skip_header` lines. This line can optionally be preceded by a comment delimiter. If `names` is a sequence or a single-string of comma-separated names, the names will be used to define the field names in a structured dtype. If `names` is None, the names of the dtype fields will be used, if any. excludelist : sequence, optional A list of names to exclude. This list is appended to the default list - ['return','file','print']. Excluded names are appended an underscore: - for example, `file` would become `file_`. + ['return','file','print']. Excluded names are appended with an + underscore: for example, `file` would become `file_`. deletechars : str, optional A string combining invalid characters that must be deleted from the names. @@ -1626,7 +1813,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, autostrip : bool, optional Whether to automatically strip white spaces from the variables. replace_space : char, optional - Character(s) used in replacement of white spaces in the variables + Character(s) used in replacement of white spaces in the variable names. By default, use a '_'. case_sensitive : {True, False, 'upper', 'lower'}, optional If True, field names are case sensitive. @@ -1634,7 +1821,9 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, If 'lower', field names are converted to lower case. unpack : bool, optional If True, the returned array is transposed, so that arguments may be - unpacked using ``x, y, z = loadtxt(...)`` + unpacked using ``x, y, z = genfromtxt(...)``. When used with a + structured data-type, arrays are returned for each field. + Default is False. usemask : bool, optional If True, return a masked array. If False, return a regular array. @@ -1659,6 +1848,13 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, to None the system default is used. The default value is 'bytes'. .. versionadded:: 1.14.0 + ndmin : int, optional + Same parameter as `loadtxt` + + .. versionadded:: 1.23.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1737,6 +1933,23 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, dtype=[('f0', 'S12'), ('f1', 'S12')]) """ + + if like is not None: + return _genfromtxt_with_like( + like, fname, dtype=dtype, comments=comments, delimiter=delimiter, + skip_header=skip_header, skip_footer=skip_footer, + converters=converters, missing_values=missing_values, + filling_values=filling_values, usecols=usecols, names=names, + excludelist=excludelist, deletechars=deletechars, + replace_space=replace_space, autostrip=autostrip, + case_sensitive=case_sensitive, defaultfmt=defaultfmt, + unpack=unpack, usemask=usemask, loose=loose, + invalid_raise=invalid_raise, max_rows=max_rows, encoding=encoding, + ndmin=ndmin, + ) + + _ensure_ndmin_ndarray_check_param(ndmin) + if max_rows is not None: if skip_footer: raise ValueError( @@ -1761,21 +1974,21 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, byte_converters = False # Initialize the filehandle, the LineSplitter and the NameValidator + if isinstance(fname, os_PathLike): + fname = os_fspath(fname) + if isinstance(fname, str): + fid = np.lib._datasource.open(fname, 'rt', encoding=encoding) + fid_ctx = contextlib.closing(fid) + else: + fid = fname + fid_ctx = contextlib.nullcontext(fid) try: - if isinstance(fname, os_PathLike): - fname = os_fspath(fname) - if isinstance(fname, str): - fid = np.lib._datasource.open(fname, 'rt', encoding=encoding) - fid_ctx = contextlib.closing(fid) - else: - fid = fname - fid_ctx = contextlib_nullcontext(fid) fhd = iter(fid) except TypeError as e: raise TypeError( - "fname must be a string, filehandle, list of strings, " - "or generator. Got %s instead." % type(fname)) from e - + "fname must be a string, a filehandle, a sequence of strings,\n" + f"or an iterator of strings. Got {type(fname)} instead." + ) from e with fid_ctx: split_line = LineSplitter(delimiter=delimiter, comments=comments, autostrip=autostrip, encoding=encoding) @@ -2125,7 +2338,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, column_types = [conv.type for conv in converters] # Find the columns with strings... strcolidx = [i for (i, v) in enumerate(column_types) - if v == np.unicode_] + if v == np.str_] if byte_converters and strcolidx: # convert strings back to bytes for backward compatibility @@ -2245,65 +2458,23 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if usemask: output = output.view(MaskedArray) output._mask = outputmask - if unpack: - return output.squeeze().T - return output.squeeze() + output = _ensure_ndmin_ndarray(output, ndmin=ndmin) -def ndfromtxt(fname, **kwargs): - """ - Load ASCII data stored in a file and return it as a single array. - - .. deprecated:: 1.17 - ndfromtxt` is a deprecated alias of `genfromtxt` which - overwrites the ``usemask`` argument with `False` even when - explicitly called as ``ndfromtxt(..., usemask=True)``. - Use `genfromtxt` instead. - - Parameters - ---------- - fname, kwargs : For a description of input parameters, see `genfromtxt`. - - See Also - -------- - numpy.genfromtxt : generic function. - - """ - kwargs['usemask'] = False - # Numpy 1.17 - warnings.warn( - "np.ndfromtxt is a deprecated alias of np.genfromtxt, " - "prefer the latter.", - DeprecationWarning, stacklevel=2) - return genfromtxt(fname, **kwargs) - - -def mafromtxt(fname, **kwargs): - """ - Load ASCII data stored in a text file and return a masked array. - - .. deprecated:: 1.17 - np.mafromtxt is a deprecated alias of `genfromtxt` which - overwrites the ``usemask`` argument with `True` even when - explicitly called as ``mafromtxt(..., usemask=False)``. - Use `genfromtxt` instead. - - Parameters - ---------- - fname, kwargs : For a description of input parameters, see `genfromtxt`. + if unpack: + if names is None: + return output.T + elif len(names) == 1: + # squeeze single-name dtypes too + return output[names[0]] + else: + # For structured arrays with multiple fields, + # return an array for each field. + return [output[field] for field in names] + return output - See Also - -------- - numpy.genfromtxt : generic function to load ASCII data. - """ - kwargs['usemask'] = True - # Numpy 1.17 - warnings.warn( - "np.mafromtxt is a deprecated alias of np.genfromtxt, " - "prefer the latter.", - DeprecationWarning, stacklevel=2) - return genfromtxt(fname, **kwargs) +_genfromtxt_with_like = array_function_dispatch()(genfromtxt) def recfromtxt(fname, **kwargs): diff --git a/numpy/lib/npyio.pyi b/numpy/lib/npyio.pyi new file mode 100644 index 000000000..cc81e82b7 --- /dev/null +++ b/numpy/lib/npyio.pyi @@ -0,0 +1,330 @@ +import os +import sys +import zipfile +import types +from re import Pattern +from collections.abc import Collection, Mapping, Iterator, Sequence, Callable, Iterable +from typing import ( + Literal as L, + Any, + TypeVar, + Generic, + IO, + overload, + Protocol, +) + +from numpy import ( + DataSource as DataSource, + ndarray, + recarray, + dtype, + generic, + float64, + void, + record, +) + +from numpy.ma.mrecords import MaskedRecords +from numpy._typing import ( + ArrayLike, + DTypeLike, + NDArray, + _DTypeLike, + _SupportsArrayFunc, +) + +from numpy.core.multiarray import ( + packbits as packbits, + unpackbits as unpackbits, +) + +_T = TypeVar("_T") +_T_contra = TypeVar("_T_contra", contravariant=True) +_T_co = TypeVar("_T_co", covariant=True) +_SCT = TypeVar("_SCT", bound=generic) +_CharType_co = TypeVar("_CharType_co", str, bytes, covariant=True) +_CharType_contra = TypeVar("_CharType_contra", str, bytes, contravariant=True) + +class _SupportsGetItem(Protocol[_T_contra, _T_co]): + def __getitem__(self, key: _T_contra, /) -> _T_co: ... + +class _SupportsRead(Protocol[_CharType_co]): + def read(self) -> _CharType_co: ... + +class _SupportsReadSeek(Protocol[_CharType_co]): + def read(self, n: int, /) -> _CharType_co: ... + def seek(self, offset: int, whence: int, /) -> object: ... + +class _SupportsWrite(Protocol[_CharType_contra]): + def write(self, s: _CharType_contra, /) -> object: ... + +__all__: list[str] + +class BagObj(Generic[_T_co]): + def __init__(self, obj: _SupportsGetItem[str, _T_co]) -> None: ... + def __getattribute__(self, key: str) -> _T_co: ... + def __dir__(self) -> list[str]: ... + +class NpzFile(Mapping[str, NDArray[Any]]): + zip: zipfile.ZipFile + fid: None | IO[str] + files: list[str] + allow_pickle: bool + pickle_kwargs: None | Mapping[str, Any] + _MAX_REPR_ARRAY_COUNT: int + # Represent `f` as a mutable property so we can access the type of `self` + @property + def f(self: _T) -> BagObj[_T]: ... + @f.setter + def f(self: _T, value: BagObj[_T]) -> None: ... + def __init__( + self, + fid: IO[str], + own_fid: bool = ..., + allow_pickle: bool = ..., + pickle_kwargs: None | Mapping[str, Any] = ..., + ) -> None: ... + def __enter__(self: _T) -> _T: ... + def __exit__( + self, + exc_type: None | type[BaseException], + exc_value: None | BaseException, + traceback: None | types.TracebackType, + /, + ) -> None: ... + def close(self) -> None: ... + def __del__(self) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + def __getitem__(self, key: str) -> NDArray[Any]: ... + def __contains__(self, key: str) -> bool: ... + def __repr__(self) -> str: ... + +# NOTE: Returns a `NpzFile` if file is a zip file; +# returns an `ndarray`/`memmap` otherwise +def load( + file: str | bytes | os.PathLike[Any] | _SupportsReadSeek[bytes], + mmap_mode: L[None, "r+", "r", "w+", "c"] = ..., + allow_pickle: bool = ..., + fix_imports: bool = ..., + encoding: L["ASCII", "latin1", "bytes"] = ..., +) -> Any: ... + +def save( + file: str | os.PathLike[str] | _SupportsWrite[bytes], + arr: ArrayLike, + allow_pickle: bool = ..., + fix_imports: bool = ..., +) -> None: ... + +def savez( + file: str | os.PathLike[str] | _SupportsWrite[bytes], + *args: ArrayLike, + **kwds: ArrayLike, +) -> None: ... + +def savez_compressed( + file: str | os.PathLike[str] | _SupportsWrite[bytes], + *args: ArrayLike, + **kwds: ArrayLike, +) -> None: ... + +# File-like objects only have to implement `__iter__` and, +# optionally, `encoding` +@overload +def loadtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: None = ..., + comments: None | str | Sequence[str] = ..., + delimiter: None | str = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + skiprows: int = ..., + usecols: int | Sequence[int] = ..., + unpack: bool = ..., + ndmin: L[0, 1, 2] = ..., + encoding: None | str = ..., + max_rows: None | int = ..., + *, + quotechar: None | str = ..., + like: None | _SupportsArrayFunc = ... +) -> NDArray[float64]: ... +@overload +def loadtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: _DTypeLike[_SCT], + comments: None | str | Sequence[str] = ..., + delimiter: None | str = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + skiprows: int = ..., + usecols: int | Sequence[int] = ..., + unpack: bool = ..., + ndmin: L[0, 1, 2] = ..., + encoding: None | str = ..., + max_rows: None | int = ..., + *, + quotechar: None | str = ..., + like: None | _SupportsArrayFunc = ... +) -> NDArray[_SCT]: ... +@overload +def loadtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: DTypeLike, + comments: None | str | Sequence[str] = ..., + delimiter: None | str = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + skiprows: int = ..., + usecols: int | Sequence[int] = ..., + unpack: bool = ..., + ndmin: L[0, 1, 2] = ..., + encoding: None | str = ..., + max_rows: None | int = ..., + *, + quotechar: None | str = ..., + like: None | _SupportsArrayFunc = ... +) -> NDArray[Any]: ... + +def savetxt( + fname: str | os.PathLike[str] | _SupportsWrite[str] | _SupportsWrite[bytes], + X: ArrayLike, + fmt: str | Sequence[str] = ..., + delimiter: str = ..., + newline: str = ..., + header: str = ..., + footer: str = ..., + comments: str = ..., + encoding: None | str = ..., +) -> None: ... + +@overload +def fromregex( + file: str | os.PathLike[str] | _SupportsRead[str] | _SupportsRead[bytes], + regexp: str | bytes | Pattern[Any], + dtype: _DTypeLike[_SCT], + encoding: None | str = ... +) -> NDArray[_SCT]: ... +@overload +def fromregex( + file: str | os.PathLike[str] | _SupportsRead[str] | _SupportsRead[bytes], + regexp: str | bytes | Pattern[Any], + dtype: DTypeLike, + encoding: None | str = ... +) -> NDArray[Any]: ... + +@overload +def genfromtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: None = ..., + comments: str = ..., + delimiter: None | str | int | Iterable[int] = ..., + skip_header: int = ..., + skip_footer: int = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + missing_values: Any = ..., + filling_values: Any = ..., + usecols: None | Sequence[int] = ..., + names: L[None, True] | str | Collection[str] = ..., + excludelist: None | Sequence[str] = ..., + deletechars: str = ..., + replace_space: str = ..., + autostrip: bool = ..., + case_sensitive: bool | L['upper', 'lower'] = ..., + defaultfmt: str = ..., + unpack: None | bool = ..., + usemask: bool = ..., + loose: bool = ..., + invalid_raise: bool = ..., + max_rows: None | int = ..., + encoding: str = ..., + *, + ndmin: L[0, 1, 2] = ..., + like: None | _SupportsArrayFunc = ..., +) -> NDArray[float64]: ... +@overload +def genfromtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: _DTypeLike[_SCT], + comments: str = ..., + delimiter: None | str | int | Iterable[int] = ..., + skip_header: int = ..., + skip_footer: int = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + missing_values: Any = ..., + filling_values: Any = ..., + usecols: None | Sequence[int] = ..., + names: L[None, True] | str | Collection[str] = ..., + excludelist: None | Sequence[str] = ..., + deletechars: str = ..., + replace_space: str = ..., + autostrip: bool = ..., + case_sensitive: bool | L['upper', 'lower'] = ..., + defaultfmt: str = ..., + unpack: None | bool = ..., + usemask: bool = ..., + loose: bool = ..., + invalid_raise: bool = ..., + max_rows: None | int = ..., + encoding: str = ..., + *, + ndmin: L[0, 1, 2] = ..., + like: None | _SupportsArrayFunc = ..., +) -> NDArray[_SCT]: ... +@overload +def genfromtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + dtype: DTypeLike, + comments: str = ..., + delimiter: None | str | int | Iterable[int] = ..., + skip_header: int = ..., + skip_footer: int = ..., + converters: None | Mapping[int | str, Callable[[str], Any]] = ..., + missing_values: Any = ..., + filling_values: Any = ..., + usecols: None | Sequence[int] = ..., + names: L[None, True] | str | Collection[str] = ..., + excludelist: None | Sequence[str] = ..., + deletechars: str = ..., + replace_space: str = ..., + autostrip: bool = ..., + case_sensitive: bool | L['upper', 'lower'] = ..., + defaultfmt: str = ..., + unpack: None | bool = ..., + usemask: bool = ..., + loose: bool = ..., + invalid_raise: bool = ..., + max_rows: None | int = ..., + encoding: str = ..., + *, + ndmin: L[0, 1, 2] = ..., + like: None | _SupportsArrayFunc = ..., +) -> NDArray[Any]: ... + +@overload +def recfromtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + *, + usemask: L[False] = ..., + **kwargs: Any, +) -> recarray[Any, dtype[record]]: ... +@overload +def recfromtxt( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + *, + usemask: L[True], + **kwargs: Any, +) -> MaskedRecords[Any, dtype[void]]: ... + +@overload +def recfromcsv( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + *, + usemask: L[False] = ..., + **kwargs: Any, +) -> recarray[Any, dtype[record]]: ... +@overload +def recfromcsv( + fname: str | os.PathLike[str] | Iterable[str] | Iterable[bytes], + *, + usemask: L[True], + **kwargs: Any, +) -> MaskedRecords[Any, dtype[void]]: ... diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 1c124cc0e..3b8db2a95 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -9,12 +9,13 @@ __all__ = ['poly', 'roots', 'polyint', 'polyder', 'polyadd', import functools import re import warnings + +from .._utils import set_module import numpy.core.numeric as NX from numpy.core import (isscalar, abs, finfo, atleast_1d, hstack, dot, array, ones) from numpy.core import overrides -from numpy.core.overrides import set_module from numpy.lib.twodim_base import diag, vander from numpy.lib.function_base import trim_zeros from numpy.lib.type_check import iscomplex, real, imag, mintypecode @@ -103,7 +104,7 @@ def poly(seq_of_zeros): References ---------- - .. [1] M. Sullivan and M. Sullivan, III, "Algebra and Trignometry, + .. [1] M. Sullivan and M. Sullivan, III, "Algebra and Trigonometry, Enhanced With Graphing Utilities," Prentice-Hall, pg. 318, 1996. .. [2] G. Strang, "Linear Algebra and Its Applications, 2nd Edition," @@ -152,9 +153,8 @@ def poly(seq_of_zeros): return 1.0 dt = seq_of_zeros.dtype a = ones((1,), dtype=dt) - for k in range(len(seq_of_zeros)): - a = NX.convolve(a, array([1, -seq_of_zeros[k]], dtype=dt), - mode='full') + for zero in seq_of_zeros: + a = NX.convolve(a, array([1, -zero], dtype=dt), mode='full') if issubclass(a.dtype.type, NX.complexfloating): # if complex roots are all complex conjugates, the roots are real. @@ -426,7 +426,7 @@ def polyder(p, m=1): >>> np.polyder(p, 3) poly1d([6]) >>> np.polyder(p, 4) - poly1d([0.]) + poly1d([0]) """ m = int(m) @@ -489,16 +489,19 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): default) just the coefficients are returned, when True diagnostic information from the singular value decomposition is also returned. w : array_like, shape (M,), optional - Weights to apply to the y-coordinates of the sample points. For - gaussian uncertainties, use 1/sigma (not 1/sigma**2). + Weights. If not None, the weight ``w[i]`` applies to the unsquared + residual ``y[i] - y_hat[i]`` at ``x[i]``. Ideally the weights are + chosen so that the errors of the products ``w[i]*y[i]`` all have the + same variance. When using inverse-variance weighting, use + ``w[i] = 1/sigma(y[i])``. The default value is None. cov : bool or str, optional If given and not `False`, return not just the estimate but also its covariance matrix. By default, the covariance are scaled by - chi2/dof, where dof = M - (deg + 1), i.e., the weights are presumed - to be unreliable except in a relative sense and everything is scaled - such that the reduced chi2 is unity. This scaling is omitted if - ``cov='unscaled'``, as is relevant for the case that the weights are - 1/sigma**2, with sigma known to be a reliable estimate of the + chi2/dof, where dof = M - (deg + 1), i.e., the weights are presumed + to be unreliable except in a relative sense and everything is scaled + such that the reduced chi2 is unity. This scaling is omitted if + ``cov='unscaled'``, as is relevant for the case that the weights are + w = 1/sigma, with sigma known to be a reliable estimate of the uncertainty. Returns @@ -508,13 +511,19 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): coefficients for `k`-th data set are in ``p[:,k]``. residuals, rank, singular_values, rcond - Present only if `full` = True. Residuals is sum of squared residuals - of the least-squares fit, the effective rank of the scaled Vandermonde - coefficient matrix, its singular values, and the specified value of - `rcond`. For more details, see `linalg.lstsq`. + These values are only returned if ``full == True`` + + - residuals -- sum of squared residuals of the least squares fit + - rank -- the effective rank of the scaled Vandermonde + coefficient matrix + - singular_values -- singular values of the scaled Vandermonde + coefficient matrix + - rcond -- value of `rcond`. + + For more details, see `numpy.linalg.lstsq`. V : ndarray, shape (M,M) or (M,M,K) - Present only if `full` = False and `cov`=True. The covariance + Present only if ``full == False`` and ``cov == True``. The covariance matrix of the polynomial coefficient estimates. The diagonal of this matrix are the variance estimates for each coefficient. If y is a 2-D array, then the covariance matrix for the `k`-th data set @@ -525,7 +534,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): ----- RankWarning The rank of the coefficient matrix in the least-squares fit is - deficient. The warning is only raised if `full` = False. + deficient. The warning is only raised if ``full == False``. The warnings can be turned off by @@ -542,7 +551,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): ----- The solution minimizes the squared error - .. math :: + .. math:: E = \\sum_{j=0}^k |p(x_j) - y_j|^2 in the equations:: @@ -663,7 +672,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): # warn on rank reduction, which indicates an ill conditioned matrix if rank != order and not full: msg = "Polyfit may be poorly conditioned" - warnings.warn(msg, RankWarning, stacklevel=4) + warnings.warn(msg, RankWarning, stacklevel=2) if full: return c, resids, rank, s, rcond @@ -678,7 +687,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): "to scale the covariance matrix") # note, this used to be: fac = resids / (len(x) - order - 2.0) # it was deciced that the "- 2" (originally justified by "Bayesian - # uncertainty analysis") is not was the user expects + # uncertainty analysis") is not what the user expects # (see gh-11196 and gh-11197) fac = resids / (len(x) - order) if y.ndim == 1: @@ -708,8 +717,8 @@ def polyval(p, x): ``p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]`` - If `x` is a sequence, then `p(x)` is returned for each element of `x`. - If `x` is another polynomial then the composite polynomial `p(x(t))` + If `x` is a sequence, then ``p(x)`` is returned for each element of ``x``. + If `x` is another polynomial then the composite polynomial ``p(x(t))`` is returned. Parameters @@ -754,11 +763,11 @@ def polyval(p, x): >>> np.polyval([3,0,1], 5) # 3 * 5**2 + 0 * 5**1 + 1 76 >>> np.polyval([3,0,1], np.poly1d(5)) - poly1d([76.]) + poly1d([76]) >>> np.polyval(np.poly1d([3,0,1]), 5) 76 >>> np.polyval(np.poly1d([3,0,1]), np.poly1d(5)) - poly1d([76.]) + poly1d([76]) """ p = NX.asarray(p) @@ -767,8 +776,8 @@ def polyval(p, x): else: x = NX.asanyarray(x) y = NX.zeros_like(x) - for i in range(len(p)): - y = y * x + p[i] + for pv in p: + y = y * x + pv return y @@ -1017,7 +1026,7 @@ def polydiv(u, v): (array([1.5 , 1.75]), array([0.25])) """ - truepoly = (isinstance(u, poly1d) or isinstance(u, poly1d)) + truepoly = (isinstance(u, poly1d) or isinstance(v, poly1d)) u = atleast_1d(u) + 0.0 v = atleast_1d(v) + 0.0 # w has the common type @@ -1037,7 +1046,7 @@ def polydiv(u, v): return poly1d(q), poly1d(r) return q, r -_poly_mat = re.compile(r"[*][*]([0-9]*)") +_poly_mat = re.compile(r"\*\*([0-9]*)") def _raise_power(astr, wrap=70): n = 0 line1 = '' @@ -1236,7 +1245,7 @@ class poly1d: raise ValueError("Polynomial must be 1d only.") c_or_r = trim_zeros(c_or_r, trim='f') if len(c_or_r) == 0: - c_or_r = NX.array([0.]) + c_or_r = NX.array([0], dtype=c_or_r.dtype) self._coeffs = c_or_r if variable is None: variable = 'x' @@ -1270,14 +1279,14 @@ class poly1d: s = s[:-5] return s - for k in range(len(coeffs)): - if not iscomplex(coeffs[k]): - coefstr = fmt_float(real(coeffs[k])) - elif real(coeffs[k]) == 0: - coefstr = '%sj' % fmt_float(imag(coeffs[k])) + for k, coeff in enumerate(coeffs): + if not iscomplex(coeff): + coefstr = fmt_float(real(coeff)) + elif real(coeff) == 0: + coefstr = '%sj' % fmt_float(imag(coeff)) else: - coefstr = '(%s + %sj)' % (fmt_float(real(coeffs[k])), - fmt_float(imag(coeffs[k]))) + coefstr = '(%s + %sj)' % (fmt_float(real(coeff)), + fmt_float(imag(coeff))) power = (N-k) if power == 0: @@ -1394,9 +1403,9 @@ class poly1d: def __getitem__(self, val): ind = self.order - val if val > self.order: - return 0 + return self.coeffs.dtype.type(0) if val < 0: - return 0 + return self.coeffs.dtype.type(0) return self.coeffs[ind] def __setitem__(self, key, val): diff --git a/numpy/lib/polynomial.pyi b/numpy/lib/polynomial.pyi new file mode 100644 index 000000000..14bbaf39d --- /dev/null +++ b/numpy/lib/polynomial.pyi @@ -0,0 +1,303 @@ +from typing import ( + Literal as L, + overload, + Any, + SupportsInt, + SupportsIndex, + TypeVar, + NoReturn, +) + +from numpy import ( + RankWarning as RankWarning, + poly1d as poly1d, + unsignedinteger, + signedinteger, + floating, + complexfloating, + bool_, + int32, + int64, + float64, + complex128, + object_, +) + +from numpy._typing import ( + NDArray, + ArrayLike, + _ArrayLikeBool_co, + _ArrayLikeUInt_co, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeObject_co, +) + +_T = TypeVar("_T") + +_2Tup = tuple[_T, _T] +_5Tup = tuple[ + _T, + NDArray[float64], + NDArray[int32], + NDArray[float64], + NDArray[float64], +] + +__all__: list[str] + +def poly(seq_of_zeros: ArrayLike) -> NDArray[floating[Any]]: ... + +# Returns either a float or complex array depending on the input values. +# See `np.linalg.eigvals`. +def roots(p: ArrayLike) -> NDArray[complexfloating[Any, Any]] | NDArray[floating[Any]]: ... + +@overload +def polyint( + p: poly1d, + m: SupportsInt | SupportsIndex = ..., + k: None | _ArrayLikeComplex_co | _ArrayLikeObject_co = ..., +) -> poly1d: ... +@overload +def polyint( + p: _ArrayLikeFloat_co, + m: SupportsInt | SupportsIndex = ..., + k: None | _ArrayLikeFloat_co = ..., +) -> NDArray[floating[Any]]: ... +@overload +def polyint( + p: _ArrayLikeComplex_co, + m: SupportsInt | SupportsIndex = ..., + k: None | _ArrayLikeComplex_co = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def polyint( + p: _ArrayLikeObject_co, + m: SupportsInt | SupportsIndex = ..., + k: None | _ArrayLikeObject_co = ..., +) -> NDArray[object_]: ... + +@overload +def polyder( + p: poly1d, + m: SupportsInt | SupportsIndex = ..., +) -> poly1d: ... +@overload +def polyder( + p: _ArrayLikeFloat_co, + m: SupportsInt | SupportsIndex = ..., +) -> NDArray[floating[Any]]: ... +@overload +def polyder( + p: _ArrayLikeComplex_co, + m: SupportsInt | SupportsIndex = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def polyder( + p: _ArrayLikeObject_co, + m: SupportsInt | SupportsIndex = ..., +) -> NDArray[object_]: ... + +@overload +def polyfit( + x: _ArrayLikeFloat_co, + y: _ArrayLikeFloat_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[False] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: L[False] = ..., +) -> NDArray[float64]: ... +@overload +def polyfit( + x: _ArrayLikeComplex_co, + y: _ArrayLikeComplex_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[False] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: L[False] = ..., +) -> NDArray[complex128]: ... +@overload +def polyfit( + x: _ArrayLikeFloat_co, + y: _ArrayLikeFloat_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[False] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: L[True, "unscaled"] = ..., +) -> _2Tup[NDArray[float64]]: ... +@overload +def polyfit( + x: _ArrayLikeComplex_co, + y: _ArrayLikeComplex_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[False] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: L[True, "unscaled"] = ..., +) -> _2Tup[NDArray[complex128]]: ... +@overload +def polyfit( + x: _ArrayLikeFloat_co, + y: _ArrayLikeFloat_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[True] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: bool | L["unscaled"] = ..., +) -> _5Tup[NDArray[float64]]: ... +@overload +def polyfit( + x: _ArrayLikeComplex_co, + y: _ArrayLikeComplex_co, + deg: SupportsIndex | SupportsInt, + rcond: None | float = ..., + full: L[True] = ..., + w: None | _ArrayLikeFloat_co = ..., + cov: bool | L["unscaled"] = ..., +) -> _5Tup[NDArray[complex128]]: ... + +@overload +def polyval( + p: _ArrayLikeBool_co, + x: _ArrayLikeBool_co, +) -> NDArray[int64]: ... +@overload +def polyval( + p: _ArrayLikeUInt_co, + x: _ArrayLikeUInt_co, +) -> NDArray[unsignedinteger[Any]]: ... +@overload +def polyval( + p: _ArrayLikeInt_co, + x: _ArrayLikeInt_co, +) -> NDArray[signedinteger[Any]]: ... +@overload +def polyval( + p: _ArrayLikeFloat_co, + x: _ArrayLikeFloat_co, +) -> NDArray[floating[Any]]: ... +@overload +def polyval( + p: _ArrayLikeComplex_co, + x: _ArrayLikeComplex_co, +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def polyval( + p: _ArrayLikeObject_co, + x: _ArrayLikeObject_co, +) -> NDArray[object_]: ... + +@overload +def polyadd( + a1: poly1d, + a2: _ArrayLikeComplex_co | _ArrayLikeObject_co, +) -> poly1d: ... +@overload +def polyadd( + a1: _ArrayLikeComplex_co | _ArrayLikeObject_co, + a2: poly1d, +) -> poly1d: ... +@overload +def polyadd( + a1: _ArrayLikeBool_co, + a2: _ArrayLikeBool_co, +) -> NDArray[bool_]: ... +@overload +def polyadd( + a1: _ArrayLikeUInt_co, + a2: _ArrayLikeUInt_co, +) -> NDArray[unsignedinteger[Any]]: ... +@overload +def polyadd( + a1: _ArrayLikeInt_co, + a2: _ArrayLikeInt_co, +) -> NDArray[signedinteger[Any]]: ... +@overload +def polyadd( + a1: _ArrayLikeFloat_co, + a2: _ArrayLikeFloat_co, +) -> NDArray[floating[Any]]: ... +@overload +def polyadd( + a1: _ArrayLikeComplex_co, + a2: _ArrayLikeComplex_co, +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def polyadd( + a1: _ArrayLikeObject_co, + a2: _ArrayLikeObject_co, +) -> NDArray[object_]: ... + +@overload +def polysub( + a1: poly1d, + a2: _ArrayLikeComplex_co | _ArrayLikeObject_co, +) -> poly1d: ... +@overload +def polysub( + a1: _ArrayLikeComplex_co | _ArrayLikeObject_co, + a2: poly1d, +) -> poly1d: ... +@overload +def polysub( + a1: _ArrayLikeBool_co, + a2: _ArrayLikeBool_co, +) -> NoReturn: ... +@overload +def polysub( + a1: _ArrayLikeUInt_co, + a2: _ArrayLikeUInt_co, +) -> NDArray[unsignedinteger[Any]]: ... +@overload +def polysub( + a1: _ArrayLikeInt_co, + a2: _ArrayLikeInt_co, +) -> NDArray[signedinteger[Any]]: ... +@overload +def polysub( + a1: _ArrayLikeFloat_co, + a2: _ArrayLikeFloat_co, +) -> NDArray[floating[Any]]: ... +@overload +def polysub( + a1: _ArrayLikeComplex_co, + a2: _ArrayLikeComplex_co, +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def polysub( + a1: _ArrayLikeObject_co, + a2: _ArrayLikeObject_co, +) -> NDArray[object_]: ... + +# NOTE: Not an alias, but they do have the same signature (that we can reuse) +polymul = polyadd + +@overload +def polydiv( + u: poly1d, + v: _ArrayLikeComplex_co | _ArrayLikeObject_co, +) -> _2Tup[poly1d]: ... +@overload +def polydiv( + u: _ArrayLikeComplex_co | _ArrayLikeObject_co, + v: poly1d, +) -> _2Tup[poly1d]: ... +@overload +def polydiv( + u: _ArrayLikeFloat_co, + v: _ArrayLikeFloat_co, +) -> _2Tup[NDArray[floating[Any]]]: ... +@overload +def polydiv( + u: _ArrayLikeComplex_co, + v: _ArrayLikeComplex_co, +) -> _2Tup[NDArray[complexfloating[Any, Any]]]: ... +@overload +def polydiv( + u: _ArrayLikeObject_co, + v: _ArrayLikeObject_co, +) -> _2Tup[NDArray[Any]]: ... diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index cfc5dc9ca..6afcf1b7f 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -13,7 +13,6 @@ from numpy.ma import MaskedArray from numpy.ma.mrecords import MaskedRecords from numpy.core.overrides import array_function_dispatch from numpy.lib._iotools import _is_string_like -from numpy.testing import suppress_warnings _check_fill_value = np.ma.core._check_fill_value @@ -105,7 +104,8 @@ def _get_fieldspec(dtype): def get_names(adtype): """ - Returns the field names of the input datatype as a tuple. + Returns the field names of the input datatype as a tuple. Input datatype + must have fields otherwise error is raised. Parameters ---------- @@ -115,15 +115,10 @@ def get_names(adtype): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> rfn.get_names(np.empty((1,), dtype=int)) - Traceback (most recent call last): - ... - AttributeError: 'numpy.ndarray' object has no attribute 'names' - - >>> rfn.get_names(np.empty((1,), dtype=[('A',int), ('B', float)])) - Traceback (most recent call last): - ... - AttributeError: 'numpy.ndarray' object has no attribute 'names' + >>> rfn.get_names(np.empty((1,), dtype=[('A', int)]).dtype) + ('A',) + >>> rfn.get_names(np.empty((1,), dtype=[('A',int), ('B', float)]).dtype) + ('A', 'B') >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names(adtype) ('a', ('b', ('ba', 'bb'))) @@ -141,8 +136,9 @@ def get_names(adtype): def get_names_flat(adtype): """ - Returns the field names of the input datatype as a tuple. Nested structure - are flattened beforehand. + Returns the field names of the input datatype as a tuple. Input datatype + must have fields otherwise error is raised. + Nested structure are flattened beforehand. Parameters ---------- @@ -152,14 +148,10 @@ def get_names_flat(adtype): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> rfn.get_names_flat(np.empty((1,), dtype=int)) is None - Traceback (most recent call last): - ... - AttributeError: 'numpy.ndarray' object has no attribute 'names' - >>> rfn.get_names_flat(np.empty((1,), dtype=[('A',int), ('B', float)])) - Traceback (most recent call last): - ... - AttributeError: 'numpy.ndarray' object has no attribute 'names' + >>> rfn.get_names_flat(np.empty((1,), dtype=[('A', int)]).dtype) is None + False + >>> rfn.get_names_flat(np.empty((1,), dtype=[('A',int), ('B', str)]).dtype) + ('A', 'B') >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names_flat(adtype) ('a', 'b', 'ba', 'bb') @@ -513,7 +505,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): Nested fields are supported. - ..versionchanged: 1.18.0 + .. versionchanged:: 1.18.0 `drop_fields` returns an array with 0 fields if all fields are dropped, rather than returning ``None`` as it did previously. @@ -784,7 +776,8 @@ def repack_fields(a, align=False, recurse=False): This method removes any overlaps and reorders the fields in memory so they have increasing byte offsets, and adds or removes padding bytes depending - on the `align` option, which behaves like the `align` option to `np.dtype`. + on the `align` option, which behaves like the `align` option to + `numpy.dtype`. If `align=False`, this method produces a "packed" memory layout in which each field starts at the byte the previous field ended, and any padding @@ -819,7 +812,8 @@ def repack_fields(a, align=False, recurse=False): ... >>> dt = np.dtype('u1, <i8, <f8', align=True) >>> dt - dtype({'names':['f0','f1','f2'], 'formats':['u1','<i8','<f8'], 'offsets':[0,8,16], 'itemsize':24}, align=True) + dtype({'names': ['f0', 'f1', 'f2'], 'formats': ['u1', '<i8', '<f8'], \ +'offsets': [0, 8, 16], 'itemsize': 24}, align=True) >>> print_offsets(dt) offsets: [0, 8, 16] itemsize: 24 @@ -916,11 +910,12 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): dtype : dtype, optional The dtype of the output unstructured array. copy : bool, optional - See copy argument to `ndarray.astype`. If true, always return a copy. - If false, and `dtype` requirements are satisfied, a view is returned. + See copy argument to `numpy.ndarray.astype`. If true, always return a + copy. If false, and `dtype` requirements are satisfied, a view is + returned. casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional - See casting argument of `ndarray.astype`. Controls what kind of data - casting may occur. + See casting argument of `numpy.ndarray.astype`. Controls what kind of + data casting may occur. Returns ------- @@ -975,9 +970,7 @@ def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): 'formats': dts, 'offsets': offsets, 'itemsize': arr.dtype.itemsize}) - with suppress_warnings() as sup: # until 1.16 (gh-12447) - sup.filter(FutureWarning, "Numpy has detected") - arr = arr.view(flattened_fields) + arr = arr.view(flattened_fields) # next cast to a packed format with all fields converted to new dtype packed_fields = np.dtype({'names': names, @@ -1019,11 +1012,12 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, align : boolean, optional Whether to create an aligned memory layout. copy : bool, optional - See copy argument to `ndarray.astype`. If true, always return a copy. - If false, and `dtype` requirements are satisfied, a view is returned. + See copy argument to `numpy.ndarray.astype`. If true, always return a + copy. If false, and `dtype` requirements are satisfied, a view is + returned. casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional - See casting argument of `ndarray.astype`. Controls what kind of data - casting may occur. + See casting argument of `numpy.ndarray.astype`. Controls what kind of + data casting may occur. Returns ------- @@ -1063,6 +1057,8 @@ def unstructured_to_structured(arr, dtype=None, names=None, align=False, else: if names is not None: raise ValueError("don't supply both dtype and names") + # if dtype is the args of np.dtype, construct it + dtype = np.dtype(dtype) # sanity check of the input dtype fields = _get_fields_and_offsets(dtype) if len(fields) == 0: diff --git a/numpy/lib/scimath.py b/numpy/lib/scimath.py index 555a3d5a8..b7ef0d710 100644 --- a/numpy/lib/scimath.py +++ b/numpy/lib/scimath.py @@ -7,13 +7,28 @@ For example, for functions like `log` with branch cuts, the versions in this module provide the mathematically valid answers in the complex plane:: >>> import math - >>> from numpy.lib import scimath - >>> scimath.log(-math.exp(1)) == (1+1j*math.pi) + >>> np.emath.log(-math.exp(1)) == (1+1j*math.pi) True Similarly, `sqrt`, other base logarithms, `power` and trig functions are correctly handled. See their respective docstrings for specific examples. +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + sqrt + log + log2 + logn + log10 + power + arccos + arcsin + arctanh + """ import numpy.core.numeric as nx import numpy.core.numerictypes as nt @@ -207,18 +222,27 @@ def sqrt(x): -------- For real, non-negative inputs this works just like `numpy.sqrt`: - >>> np.lib.scimath.sqrt(1) + >>> np.emath.sqrt(1) 1.0 - >>> np.lib.scimath.sqrt([1, 4]) + >>> np.emath.sqrt([1, 4]) array([1., 2.]) But it automatically handles negative inputs: - >>> np.lib.scimath.sqrt(-1) + >>> np.emath.sqrt(-1) 1j - >>> np.lib.scimath.sqrt([-1,4]) + >>> np.emath.sqrt([-1,4]) array([0.+1.j, 2.+0.j]) + Different results are expected because: + floating point 0.0 and -0.0 are distinct. + + For more control, explicitly use complex() as follows: + + >>> np.emath.sqrt(complex(-4.0, 0.0)) + 2j + >>> np.emath.sqrt(complex(-4.0, -0.0)) + -2j """ x = _fix_real_lt_zero(x) return nx.sqrt(x) @@ -351,9 +375,9 @@ def logn(n, x): -------- >>> np.set_printoptions(precision=4) - >>> np.lib.scimath.logn(2, [4, 8]) + >>> np.emath.logn(2, [4, 8]) array([2., 3.]) - >>> np.lib.scimath.logn(2, [-4, -8, 8]) + >>> np.emath.logn(2, [-4, -8, 8]) array([2.+4.5324j, 3.+4.5324j, 3.+0.j ]) """ @@ -446,11 +470,11 @@ def power(x, p): -------- >>> np.set_printoptions(precision=4) - >>> np.lib.scimath.power([2, 4], 2) + >>> np.emath.power([2, 4], 2) array([ 4, 16]) - >>> np.lib.scimath.power([2, 4], -2) + >>> np.emath.power([2, 4], -2) array([0.25 , 0.0625]) - >>> np.lib.scimath.power([-2, 4], 2) + >>> np.emath.power([-2, 4], 2) array([ 4.-0.j, 16.+0.j]) """ @@ -556,10 +580,10 @@ def arctanh(x): Compute the inverse hyperbolic tangent of `x`. Return the "principal value" (for a description of this, see - `numpy.arctanh`) of `arctanh(x)`. For real `x` such that - `abs(x) < 1`, this is a real number. If `abs(x) > 1`, or if `x` is + `numpy.arctanh`) of ``arctanh(x)``. For real `x` such that + ``abs(x) < 1``, this is a real number. If `abs(x) > 1`, or if `x` is complex, the result is complex. Finally, `x = 1` returns``inf`` and - `x=-1` returns ``-inf``. + ``x=-1`` returns ``-inf``. Parameters ---------- @@ -581,7 +605,7 @@ def arctanh(x): ----- For an arctanh() that returns ``NAN`` when real `x` is not in the interval ``(-1,1)``, use `numpy.arctanh` (this latter, however, does - return +/-inf for `x = +/-1`). + return +/-inf for ``x = +/-1``). Examples -------- diff --git a/numpy/lib/scimath.pyi b/numpy/lib/scimath.pyi new file mode 100644 index 000000000..589feb15f --- /dev/null +++ b/numpy/lib/scimath.pyi @@ -0,0 +1,94 @@ +from typing import overload, Any + +from numpy import complexfloating + +from numpy._typing import ( + NDArray, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ComplexLike_co, + _FloatLike_co, +) + +__all__: list[str] + +@overload +def sqrt(x: _FloatLike_co) -> Any: ... +@overload +def sqrt(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def sqrt(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def sqrt(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def log(x: _FloatLike_co) -> Any: ... +@overload +def log(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def log(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def log(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def log10(x: _FloatLike_co) -> Any: ... +@overload +def log10(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def log10(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def log10(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def log2(x: _FloatLike_co) -> Any: ... +@overload +def log2(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def log2(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def log2(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def logn(n: _FloatLike_co, x: _FloatLike_co) -> Any: ... +@overload +def logn(n: _ComplexLike_co, x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def logn(n: _ArrayLikeFloat_co, x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def logn(n: _ArrayLikeComplex_co, x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def power(x: _FloatLike_co, p: _FloatLike_co) -> Any: ... +@overload +def power(x: _ComplexLike_co, p: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def power(x: _ArrayLikeFloat_co, p: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def power(x: _ArrayLikeComplex_co, p: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def arccos(x: _FloatLike_co) -> Any: ... +@overload +def arccos(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def arccos(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def arccos(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def arcsin(x: _FloatLike_co) -> Any: ... +@overload +def arcsin(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def arcsin(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def arcsin(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + +@overload +def arctanh(x: _FloatLike_co) -> Any: ... +@overload +def arctanh(x: _ComplexLike_co) -> complexfloating[Any, Any]: ... +@overload +def arctanh(x: _ArrayLikeFloat_co) -> NDArray[Any]: ... +@overload +def arctanh(x: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... diff --git a/numpy/lib/setup.py b/numpy/lib/setup.py index b3f441f38..7520b72d7 100644 --- a/numpy/lib/setup.py +++ b/numpy/lib/setup.py @@ -4,6 +4,7 @@ def configuration(parent_package='',top_path=None): config = Configuration('lib', parent_package, top_path) config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index bc6718eca..5d8a41bfe 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -1,9 +1,7 @@ import functools import numpy.core.numeric as _nx -from numpy.core.numeric import ( - asarray, zeros, outer, concatenate, array, asanyarray - ) +from numpy.core.numeric import asarray, zeros, array, asanyarray from numpy.core.fromnumeric import reshape, transpose from numpy.core.multiarray import normalize_axis_index from numpy.core import overrides @@ -69,13 +67,13 @@ def take_along_axis(arr, indices, axis): Parameters ---------- - arr: ndarray (Ni..., M, Nk...) + arr : ndarray (Ni..., M, Nk...) Source array - indices: ndarray (Ni..., J, Nk...) + indices : ndarray (Ni..., J, Nk...) Indices to take along each 1d slice of `arr`. This must match the dimension of arr, but dimensions Ni and Nj only need to broadcast against `arr`. - axis: int + axis : int The axis to take 1d slices along. If axis is None, the input array is treated as if it had first been flattened to 1d, for consistency with `sort` and `argsort`. @@ -124,19 +122,21 @@ def take_along_axis(arr, indices, axis): >>> np.sort(a, axis=1) array([[10, 20, 30], [40, 50, 60]]) - >>> ai = np.argsort(a, axis=1); ai + >>> ai = np.argsort(a, axis=1) + >>> ai array([[0, 2, 1], [1, 2, 0]]) >>> np.take_along_axis(a, ai, axis=1) array([[10, 20, 30], [40, 50, 60]]) - The same works for max and min, if you expand the dimensions: + The same works for max and min, if you maintain the trivial dimension + with ``keepdims``: - >>> np.expand_dims(np.max(a, axis=1), axis=1) + >>> np.max(a, axis=1, keepdims=True) array([[30], [60]]) - >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) + >>> ai = np.argmax(a, axis=1, keepdims=True) >>> ai array([[1], [0]]) @@ -147,8 +147,8 @@ def take_along_axis(arr, indices, axis): If we want to get the max and min at the same time, we can stack the indices first - >>> ai_min = np.expand_dims(np.argmin(a, axis=1), axis=1) - >>> ai_max = np.expand_dims(np.argmax(a, axis=1), axis=1) + >>> ai_min = np.argmin(a, axis=1, keepdims=True) + >>> ai_max = np.argmax(a, axis=1, keepdims=True) >>> ai = np.concatenate([ai_min, ai_max], axis=1) >>> ai array([[0, 1], @@ -190,16 +190,16 @@ def put_along_axis(arr, indices, values, axis): Parameters ---------- - arr: ndarray (Ni..., M, Nk...) + arr : ndarray (Ni..., M, Nk...) Destination array. - indices: ndarray (Ni..., J, Nk...) + indices : ndarray (Ni..., J, Nk...) Indices to change along each 1d slice of `arr`. This must match the dimension of arr, but dimensions in Ni and Nj may be 1 to broadcast against `arr`. - values: array_like (Ni..., J, Nk...) + values : array_like (Ni..., J, Nk...) values to insert at those indices. Its shape and dimension are broadcast to match that of `indices`. - axis: int + axis : int The axis to take 1d slices along. If axis is None, the destination array is treated as if a flattened 1d view had been created of it. @@ -237,7 +237,7 @@ def put_along_axis(arr, indices, values, axis): We can replace the maximum values with: - >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) + >>> ai = np.argmax(a, axis=1, keepdims=True) >>> ai array([[1], [0]]) @@ -372,7 +372,7 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): # invoke the function on the first item try: ind0 = next(inds) - except StopIteration as e: + except StopIteration: raise ValueError( 'Cannot apply_along_axis when any iteration dimensions are 0' ) from None @@ -643,13 +643,9 @@ def column_stack(tup): [3, 4]]) """ - if not overrides.ARRAY_FUNCTION_ENABLED: - # raise warning if necessary - _arrays_for_stack_dispatcher(tup, stacklevel=2) - arrays = [] for v in tup: - arr = array(v, copy=False, subok=True) + arr = asanyarray(v) if arr.ndim < 2: arr = array(arr, copy=False, subok=True, ndmin=2).T arrays.append(arr) @@ -713,10 +709,6 @@ def dstack(tup): [[3, 4]]]) """ - if not overrides.ARRAY_FUNCTION_ENABLED: - # raise warning if necessary - _arrays_for_stack_dispatcher(tup, stacklevel=2) - arrs = atleast_3d(*tup) if not isinstance(arrs, list): arrs = [arrs] @@ -756,11 +748,11 @@ def array_split(ary, indices_or_sections, axis=0): -------- >>> x = np.arange(8.0) >>> np.array_split(x, 3) - [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7.])] + [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7.])] - >>> x = np.arange(7.0) - >>> np.array_split(x, 3) - [array([0., 1., 2.]), array([3., 4.]), array([5., 6.])] + >>> x = np.arange(9) + >>> np.array_split(x, 4) + [array([0, 1, 2]), array([3, 4]), array([5, 6]), array([7, 8])] """ try: @@ -775,7 +767,7 @@ def array_split(ary, indices_or_sections, axis=0): # indices_or_sections is a scalar, not an array. Nsections = int(indices_or_sections) if Nsections <= 0: - raise ValueError('number sections must be larger than 0.') + raise ValueError('number sections must be larger than 0.') from None Neach_section, extras = divmod(Ntotal, Nsections) section_sizes = ([0] + extras * [Neach_section+1] + @@ -870,7 +862,7 @@ def split(ary, indices_or_sections, axis=0): N = ary.shape[axis] if N % sections: raise ValueError( - 'array split does not result in an equal division') + 'array split does not result in an equal division') from None return array_split(ary, indices_or_sections, axis) @@ -885,7 +877,7 @@ def hsplit(ary, indices_or_sections): Please refer to the `split` documentation. `hsplit` is equivalent to `split` with ``axis=1``, the array is always split along the second - axis regardless of the array dimension. + axis except for 1-D arrays, where it is split at ``axis=0``. See Also -------- @@ -933,6 +925,12 @@ def hsplit(ary, indices_or_sections): array([[[2., 3.]], [[6., 7.]]])] + With a 1-D array, the split is along axis 0. + + >>> x = np.array([0, 1, 2, 3, 4, 5]) + >>> np.hsplit(x, 2) + [array([0, 1, 2]), array([3, 4, 5])] + """ if _nx.ndim(ary) == 0: raise ValueError('hsplit only works on arrays of 1 or more dimensions') @@ -1035,6 +1033,7 @@ def dsplit(ary, indices_or_sections): raise ValueError('dsplit only works on arrays of 3 or more dimensions') return split(ary, indices_or_sections, 2) + def get_array_prepare(*args): """Find the wrapper for the array with the highest priority. @@ -1047,6 +1046,7 @@ def get_array_prepare(*args): return wrappers[-1][-1] return None + def get_array_wrap(*args): """Find the wrapper for the array with the highest priority. @@ -1088,8 +1088,8 @@ def kron(a, b): ----- The function assumes that the number of dimensions of `a` and `b` are the same, if necessary prepending the smallest with ones. - If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`, - the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`. + If ``a.shape = (r0,r1,..,rN)`` and ``b.shape = (s0,s1,...,sN)``, + the Kronecker product has shape ``(r0*s0, r1*s1, ..., rN*SN)``. The elements are products of elements from `a` and `b`, organized explicitly by:: @@ -1133,35 +1133,49 @@ def kron(a, b): True """ + # Working: + # 1. Equalise the shapes by prepending smaller array with 1s + # 2. Expand shapes of both the arrays by adding new axes at + # odd positions for 1st array and even positions for 2nd + # 3. Compute the product of the modified array + # 4. The inner most array elements now contain the rows of + # the Kronecker product + # 5. Reshape the result to kron's shape, which is same as + # product of shapes of the two arrays. b = asanyarray(b) a = array(a, copy=False, subok=True, ndmin=b.ndim) + is_any_mat = isinstance(a, matrix) or isinstance(b, matrix) ndb, nda = b.ndim, a.ndim + nd = max(ndb, nda) + if (nda == 0 or ndb == 0): return _nx.multiply(a, b) + as_ = a.shape bs = b.shape if not a.flags.contiguous: a = reshape(a, as_) if not b.flags.contiguous: b = reshape(b, bs) - nd = ndb - if (ndb != nda): - if (ndb > nda): - as_ = (1,)*(ndb-nda) + as_ - else: - bs = (1,)*(nda-ndb) + bs - nd = nda - result = outer(a, b).reshape(as_+bs) - axis = nd-1 - for _ in range(nd): - result = concatenate(result, axis=axis) - wrapper = get_array_prepare(a, b) - if wrapper is not None: - result = wrapper(result) - wrapper = get_array_wrap(a, b) - if wrapper is not None: - result = wrapper(result) - return result + + # Equalise the shapes by prepending smaller one with 1s + as_ = (1,)*max(0, ndb-nda) + as_ + bs = (1,)*max(0, nda-ndb) + bs + + # Insert empty dimensions + a_arr = expand_dims(a, axis=tuple(range(ndb-nda))) + b_arr = expand_dims(b, axis=tuple(range(nda-ndb))) + + # Compute the product + a_arr = expand_dims(a_arr, axis=tuple(range(1, nd*2, 2))) + b_arr = expand_dims(b_arr, axis=tuple(range(0, nd*2, 2))) + # In case of `mat`, convert result to `array` + result = _nx.multiply(a_arr, b_arr, subok=(not is_any_mat)) + + # Reshape back + result = result.reshape(_nx.multiply(as_, bs)) + + return result if not is_any_mat else matrix(result, copy=False) def _tile_dispatcher(A, reps): diff --git a/numpy/lib/shape_base.pyi b/numpy/lib/shape_base.pyi new file mode 100644 index 000000000..1b718da22 --- /dev/null +++ b/numpy/lib/shape_base.pyi @@ -0,0 +1,215 @@ +from collections.abc import Callable, Sequence +from typing import TypeVar, Any, overload, SupportsIndex, Protocol + +from numpy import ( + generic, + integer, + ufunc, + bool_, + unsignedinteger, + signedinteger, + floating, + complexfloating, + object_, +) + +from numpy._typing import ( + ArrayLike, + NDArray, + _ShapeLike, + _ArrayLike, + _ArrayLikeBool_co, + _ArrayLikeUInt_co, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeObject_co, +) + +from numpy.core.shape_base import vstack + +_SCT = TypeVar("_SCT", bound=generic) + +# The signatures of `__array_wrap__` and `__array_prepare__` are the same; +# give them unique names for the sake of clarity +class _ArrayWrap(Protocol): + def __call__( + self, + array: NDArray[Any], + context: None | tuple[ufunc, tuple[Any, ...], int] = ..., + /, + ) -> Any: ... + +class _ArrayPrepare(Protocol): + def __call__( + self, + array: NDArray[Any], + context: None | tuple[ufunc, tuple[Any, ...], int] = ..., + /, + ) -> Any: ... + +class _SupportsArrayWrap(Protocol): + @property + def __array_wrap__(self) -> _ArrayWrap: ... + +class _SupportsArrayPrepare(Protocol): + @property + def __array_prepare__(self) -> _ArrayPrepare: ... + +__all__: list[str] + +row_stack = vstack + +def take_along_axis( + arr: _SCT | NDArray[_SCT], + indices: NDArray[integer[Any]], + axis: None | int, +) -> NDArray[_SCT]: ... + +def put_along_axis( + arr: NDArray[_SCT], + indices: NDArray[integer[Any]], + values: ArrayLike, + axis: None | int, +) -> None: ... + +# TODO: Use PEP 612 `ParamSpec` once mypy supports `Concatenate` +# xref python/mypy#8645 +@overload +def apply_along_axis( + func1d: Callable[..., _ArrayLike[_SCT]], + axis: SupportsIndex, + arr: ArrayLike, + *args: Any, + **kwargs: Any, +) -> NDArray[_SCT]: ... +@overload +def apply_along_axis( + func1d: Callable[..., ArrayLike], + axis: SupportsIndex, + arr: ArrayLike, + *args: Any, + **kwargs: Any, +) -> NDArray[Any]: ... + +def apply_over_axes( + func: Callable[[NDArray[Any], int], NDArray[_SCT]], + a: ArrayLike, + axes: int | Sequence[int], +) -> NDArray[_SCT]: ... + +@overload +def expand_dims( + a: _ArrayLike[_SCT], + axis: _ShapeLike, +) -> NDArray[_SCT]: ... +@overload +def expand_dims( + a: ArrayLike, + axis: _ShapeLike, +) -> NDArray[Any]: ... + +@overload +def column_stack(tup: Sequence[_ArrayLike[_SCT]]) -> NDArray[_SCT]: ... +@overload +def column_stack(tup: Sequence[ArrayLike]) -> NDArray[Any]: ... + +@overload +def dstack(tup: Sequence[_ArrayLike[_SCT]]) -> NDArray[_SCT]: ... +@overload +def dstack(tup: Sequence[ArrayLike]) -> NDArray[Any]: ... + +@overload +def array_split( + ary: _ArrayLike[_SCT], + indices_or_sections: _ShapeLike, + axis: SupportsIndex = ..., +) -> list[NDArray[_SCT]]: ... +@overload +def array_split( + ary: ArrayLike, + indices_or_sections: _ShapeLike, + axis: SupportsIndex = ..., +) -> list[NDArray[Any]]: ... + +@overload +def split( + ary: _ArrayLike[_SCT], + indices_or_sections: _ShapeLike, + axis: SupportsIndex = ..., +) -> list[NDArray[_SCT]]: ... +@overload +def split( + ary: ArrayLike, + indices_or_sections: _ShapeLike, + axis: SupportsIndex = ..., +) -> list[NDArray[Any]]: ... + +@overload +def hsplit( + ary: _ArrayLike[_SCT], + indices_or_sections: _ShapeLike, +) -> list[NDArray[_SCT]]: ... +@overload +def hsplit( + ary: ArrayLike, + indices_or_sections: _ShapeLike, +) -> list[NDArray[Any]]: ... + +@overload +def vsplit( + ary: _ArrayLike[_SCT], + indices_or_sections: _ShapeLike, +) -> list[NDArray[_SCT]]: ... +@overload +def vsplit( + ary: ArrayLike, + indices_or_sections: _ShapeLike, +) -> list[NDArray[Any]]: ... + +@overload +def dsplit( + ary: _ArrayLike[_SCT], + indices_or_sections: _ShapeLike, +) -> list[NDArray[_SCT]]: ... +@overload +def dsplit( + ary: ArrayLike, + indices_or_sections: _ShapeLike, +) -> list[NDArray[Any]]: ... + +@overload +def get_array_prepare(*args: _SupportsArrayPrepare) -> _ArrayPrepare: ... +@overload +def get_array_prepare(*args: object) -> None | _ArrayPrepare: ... + +@overload +def get_array_wrap(*args: _SupportsArrayWrap) -> _ArrayWrap: ... +@overload +def get_array_wrap(*args: object) -> None | _ArrayWrap: ... + +@overload +def kron(a: _ArrayLikeBool_co, b: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] +@overload +def kron(a: _ArrayLikeUInt_co, b: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] +@overload +def kron(a: _ArrayLikeInt_co, b: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] +@overload +def kron(a: _ArrayLikeFloat_co, b: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] +@overload +def kron(a: _ArrayLikeComplex_co, b: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def kron(a: _ArrayLikeObject_co, b: Any) -> NDArray[object_]: ... +@overload +def kron(a: Any, b: _ArrayLikeObject_co) -> NDArray[object_]: ... + +@overload +def tile( + A: _ArrayLike[_SCT], + reps: int | Sequence[int], +) -> NDArray[_SCT]: ... +@overload +def tile( + A: ArrayLike, + reps: int | Sequence[int], +) -> NDArray[Any]: ... diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index 502235bdf..6794ad557 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -6,9 +6,10 @@ NumPy reference guide. """ import numpy as np -from numpy.core.overrides import array_function_dispatch +from numpy.core.numeric import normalize_axis_tuple +from numpy.core.overrides import array_function_dispatch, set_module -__all__ = ['broadcast_to', 'broadcast_arrays'] +__all__ = ['broadcast_to', 'broadcast_arrays', 'broadcast_shapes'] class DummyArray: @@ -65,8 +66,10 @@ def as_strided(x, shape=None, strides=None, subok=False, writeable=True): See also -------- - broadcast_to: broadcast an array to a given shape. + broadcast_to : broadcast an array to a given shape. reshape : reshape an array. + lib.stride_tricks.sliding_window_view : + userfriendly and safe function for the creation of sliding window views. Notes ----- @@ -83,6 +86,7 @@ def as_strided(x, shape=None, strides=None, subok=False, writeable=True): Vectorized write operations on such arrays will typically be unpredictable. They may even give different results for small, large, or transposed arrays. + Since writing to these arrays has to be tested and done with great care, you may want to use ``writeable=False`` to avoid accidental write operations. @@ -111,6 +115,228 @@ def as_strided(x, shape=None, strides=None, subok=False, writeable=True): return view +def _sliding_window_view_dispatcher(x, window_shape, axis=None, *, + subok=None, writeable=None): + return (x,) + + +@array_function_dispatch(_sliding_window_view_dispatcher) +def sliding_window_view(x, window_shape, axis=None, *, + subok=False, writeable=False): + """ + Create a sliding window view into the array with the given window shape. + + Also known as rolling or moving window, the window slides across all + dimensions of the array and extracts subsets of the array at all window + positions. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + x : array_like + Array to create the sliding window view from. + window_shape : int or tuple of int + Size of window over each axis that takes part in the sliding window. + If `axis` is not present, must have same length as the number of input + array dimensions. Single integers `i` are treated as if they were the + tuple `(i,)`. + axis : int or tuple of int, optional + Axis or axes along which the sliding window is applied. + By default, the sliding window is applied to all axes and + `window_shape[i]` will refer to axis `i` of `x`. + If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to + the axis `axis[i]` of `x`. + Single integers `i` are treated as if they were the tuple `(i,)`. + subok : bool, optional + If True, sub-classes will be passed-through, otherwise the returned + array will be forced to be a base-class array (default). + writeable : bool, optional + When true, allow writing to the returned view. The default is false, + as this should be used with caution: the returned view contains the + same memory location multiple times, so writing to one location will + cause others to change. + + Returns + ------- + view : ndarray + Sliding window view of the array. The sliding window dimensions are + inserted at the end, and the original dimensions are trimmed as + required by the size of the sliding window. + That is, ``view.shape = x_shape_trimmed + window_shape``, where + ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less + than the corresponding window size. + + See Also + -------- + lib.stride_tricks.as_strided: A lower-level and less safe routine for + creating arbitrary views from custom shape and strides. + broadcast_to: broadcast an array to a given shape. + + Notes + ----- + For many applications using a sliding window view can be convenient, but + potentially very slow. Often specialized solutions exist, for example: + + - `scipy.signal.fftconvolve` + + - filtering functions in `scipy.ndimage` + + - moving window functions provided by + `bottleneck <https://github.com/pydata/bottleneck>`_. + + As a rough estimate, a sliding window approach with an input size of `N` + and a window size of `W` will scale as `O(N*W)` where frequently a special + algorithm can achieve `O(N)`. That means that the sliding window variant + for a window size of 100 can be a 100 times slower than a more specialized + version. + + Nevertheless, for small window sizes, when no custom algorithm exists, or + as a prototyping and developing tool, this function can be a good solution. + + Examples + -------- + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + + This also works in more dimensions, e.g. + + >>> i, j = np.ogrid[:3, :4] + >>> x = 10*i + j + >>> x.shape + (3, 4) + >>> x + array([[ 0, 1, 2, 3], + [10, 11, 12, 13], + [20, 21, 22, 23]]) + >>> shape = (2,2) + >>> v = sliding_window_view(x, shape) + >>> v.shape + (2, 3, 2, 2) + >>> v + array([[[[ 0, 1], + [10, 11]], + [[ 1, 2], + [11, 12]], + [[ 2, 3], + [12, 13]]], + [[[10, 11], + [20, 21]], + [[11, 12], + [21, 22]], + [[12, 13], + [22, 23]]]]) + + The axis can be specified explicitly: + + >>> v = sliding_window_view(x, 3, 0) + >>> v.shape + (1, 4, 3) + >>> v + array([[[ 0, 10, 20], + [ 1, 11, 21], + [ 2, 12, 22], + [ 3, 13, 23]]]) + + The same axis can be used several times. In that case, every use reduces + the corresponding original dimension: + + >>> v = sliding_window_view(x, (2, 3), (1, 1)) + >>> v.shape + (3, 1, 2, 3) + >>> v + array([[[[ 0, 1, 2], + [ 1, 2, 3]]], + [[[10, 11, 12], + [11, 12, 13]]], + [[[20, 21, 22], + [21, 22, 23]]]]) + + Combining with stepped slicing (`::step`), this can be used to take sliding + views which skip elements: + + >>> x = np.arange(7) + >>> sliding_window_view(x, 5)[:, ::2] + array([[0, 2, 4], + [1, 3, 5], + [2, 4, 6]]) + + or views which move by multiple elements + + >>> x = np.arange(7) + >>> sliding_window_view(x, 3)[::2, :] + array([[0, 1, 2], + [2, 3, 4], + [4, 5, 6]]) + + A common application of `sliding_window_view` is the calculation of running + statistics. The simplest example is the + `moving average <https://en.wikipedia.org/wiki/Moving_average>`_: + + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + >>> moving_average = v.mean(axis=-1) + >>> moving_average + array([1., 2., 3., 4.]) + + Note that a sliding window approach is often **not** optimal (see Notes). + """ + window_shape = (tuple(window_shape) + if np.iterable(window_shape) + else (window_shape,)) + # first convert input to array, possibly keeping subclass + x = np.array(x, copy=False, subok=subok) + + window_shape_array = np.array(window_shape) + if np.any(window_shape_array < 0): + raise ValueError('`window_shape` cannot contain negative values') + + if axis is None: + axis = tuple(range(x.ndim)) + if len(window_shape) != len(axis): + raise ValueError(f'Since axis is `None`, must provide ' + f'window_shape for all dimensions of `x`; ' + f'got {len(window_shape)} window_shape elements ' + f'and `x.ndim` is {x.ndim}.') + else: + axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True) + if len(window_shape) != len(axis): + raise ValueError(f'Must provide matching length window_shape and ' + f'axis; got {len(window_shape)} window_shape ' + f'elements and {len(axis)} axes elements.') + + out_strides = x.strides + tuple(x.strides[ax] for ax in axis) + + # note: same axis can be windowed repeatedly + x_shape_trimmed = list(x.shape) + for ax, dim in zip(axis, window_shape): + if x_shape_trimmed[ax] < dim: + raise ValueError( + 'window shape cannot be larger than input array shape') + x_shape_trimmed[ax] -= dim - 1 + out_shape = tuple(x_shape_trimmed) + window_shape + return as_strided(x, strides=out_strides, shape=out_shape, + subok=subok, writeable=writeable) + + def _broadcast_to(array, shape, subok, readonly): shape = tuple(shape) if np.iterable(shape) else (shape,) array = np.array(array, copy=False, subok=subok) @@ -146,8 +372,9 @@ def broadcast_to(array, shape, subok=False): ---------- array : array_like The array to broadcast. - shape : tuple - The shape of the desired array. + shape : tuple or int + The shape of the desired array. A single integer ``i`` is interpreted + as ``(i,)``. subok : bool, optional If True, then sub-classes will be passed-through, otherwise the returned array will be forced to be a base-class array (default). @@ -165,6 +392,12 @@ def broadcast_to(array, shape, subok=False): If the array is not compatible with the new shape according to NumPy's broadcasting rules. + See Also + -------- + broadcast + broadcast_arrays + broadcast_shapes + Notes ----- .. versionadded:: 1.10.0 @@ -197,6 +430,49 @@ def _broadcast_shape(*args): return b.shape +@set_module('numpy') +def broadcast_shapes(*args): + """ + Broadcast the input shapes into a single shape. + + :ref:`Learn more about broadcasting here <basics.broadcasting>`. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + `*args` : tuples of ints, or ints + The shapes to be broadcast against each other. + + Returns + ------- + tuple + Broadcasted shape. + + Raises + ------ + ValueError + If the shapes are not compatible and cannot be broadcast according + to NumPy's broadcasting rules. + + See Also + -------- + broadcast + broadcast_arrays + broadcast_to + + Examples + -------- + >>> np.broadcast_shapes((1, 2), (3, 1), (3, 2)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) + """ + arrays = [np.empty(x, dtype=[]) for x in args] + return _broadcast_shape(*arrays) + + def _broadcast_arrays_dispatcher(*args, subok=None): return args @@ -230,6 +506,12 @@ def broadcast_arrays(*args, subok=False): warning will be emitted. A future version will set the ``writable`` flag False so writing to it will raise an error. + See Also + -------- + broadcast + broadcast_to + broadcast_shapes + Examples -------- >>> x = np.array([[1,2,3]]) diff --git a/numpy/lib/stride_tricks.pyi b/numpy/lib/stride_tricks.pyi new file mode 100644 index 000000000..4c9a98e85 --- /dev/null +++ b/numpy/lib/stride_tricks.pyi @@ -0,0 +1,80 @@ +from collections.abc import Iterable +from typing import Any, TypeVar, overload, SupportsIndex + +from numpy import generic +from numpy._typing import ( + NDArray, + ArrayLike, + _ShapeLike, + _Shape, + _ArrayLike +) + +_SCT = TypeVar("_SCT", bound=generic) + +__all__: list[str] + +class DummyArray: + __array_interface__: dict[str, Any] + base: None | NDArray[Any] + def __init__( + self, + interface: dict[str, Any], + base: None | NDArray[Any] = ..., + ) -> None: ... + +@overload +def as_strided( + x: _ArrayLike[_SCT], + shape: None | Iterable[int] = ..., + strides: None | Iterable[int] = ..., + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def as_strided( + x: ArrayLike, + shape: None | Iterable[int] = ..., + strides: None | Iterable[int] = ..., + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[Any]: ... + +@overload +def sliding_window_view( + x: _ArrayLike[_SCT], + window_shape: int | Iterable[int], + axis: None | SupportsIndex = ..., + *, + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def sliding_window_view( + x: ArrayLike, + window_shape: int | Iterable[int], + axis: None | SupportsIndex = ..., + *, + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[Any]: ... + +@overload +def broadcast_to( + array: _ArrayLike[_SCT], + shape: int | Iterable[int], + subok: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def broadcast_to( + array: ArrayLike, + shape: int | Iterable[int], + subok: bool = ..., +) -> NDArray[Any]: ... + +def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... + +def broadcast_arrays( + *args: ArrayLike, + subok: bool = ..., +) -> list[NDArray[Any]]: ... diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 1ed7815d9..c8149abc3 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -87,11 +87,11 @@ def invalid_httpfile(): class TestDataSourceOpen: - def setup(self): + def setup_method(self): self.tmpdir = mkdtemp() self.ds = datasource.DataSource(self.tmpdir) - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) del self.ds @@ -102,10 +102,10 @@ class TestDataSourceOpen: def test_InvalidHTTP(self): url = invalid_httpurl() - assert_raises(IOError, self.ds.open, url) + assert_raises(OSError, self.ds.open, url) try: self.ds.open(url) - except IOError as e: + except OSError as e: # Regression test for bug fixed in r4342. assert_(e.errno is None) @@ -120,7 +120,7 @@ class TestDataSourceOpen: def test_InvalidFile(self): invalid_file = invalid_textfile(self.tmpdir) - assert_raises(IOError, self.ds.open, invalid_file) + assert_raises(OSError, self.ds.open, invalid_file) def test_ValidGzipFile(self): try: @@ -156,11 +156,11 @@ class TestDataSourceOpen: class TestDataSourceExists: - def setup(self): + def setup_method(self): self.tmpdir = mkdtemp() self.ds = datasource.DataSource(self.tmpdir) - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) del self.ds @@ -186,11 +186,11 @@ class TestDataSourceExists: class TestDataSourceAbspath: - def setup(self): + def setup_method(self): self.tmpdir = os.path.abspath(mkdtemp()) self.ds = datasource.DataSource(self.tmpdir) - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) del self.ds @@ -251,11 +251,11 @@ class TestDataSourceAbspath: class TestRepositoryAbspath: - def setup(self): + def setup_method(self): self.tmpdir = os.path.abspath(mkdtemp()) self.repos = datasource.Repository(valid_baseurl(), self.tmpdir) - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) del self.repos @@ -284,11 +284,11 @@ class TestRepositoryAbspath: class TestRepositoryExists: - def setup(self): + def setup_method(self): self.tmpdir = mkdtemp() self.repos = datasource.Repository(valid_baseurl(), self.tmpdir) - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) del self.repos @@ -317,10 +317,10 @@ class TestRepositoryExists: class TestOpenFunc: - def setup(self): + def setup_method(self): self.tmpdir = mkdtemp() - def teardown(self): + def teardown_method(self): rmtree(self.tmpdir) def test_DataSourceOpen(self): diff --git a/numpy/lib/tests/test__version.py b/numpy/lib/tests/test__version.py index 182504631..e6d41ad93 100644 --- a/numpy/lib/tests/test__version.py +++ b/numpy/lib/tests/test__version.py @@ -7,7 +7,7 @@ from numpy.lib import NumpyVersion def test_main_versions(): assert_(NumpyVersion('1.8.0') == '1.8.0') - for ver in ['1.9.0', '2.0.0', '1.8.1']: + for ver in ['1.9.0', '2.0.0', '1.8.1', '10.0.1']: assert_(NumpyVersion('1.8.0') < ver) for ver in ['1.7.0', '1.7.1', '0.9.9']: diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index 75db5928b..0bebe3693 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -474,8 +474,7 @@ class TestStatistic: @pytest.mark.filterwarnings("ignore:Mean of empty slice:RuntimeWarning") @pytest.mark.filterwarnings( - "ignore:invalid value encountered in (true_divide|double_scalars):" - "RuntimeWarning" + "ignore:invalid value encountered in( scalar)? divide:RuntimeWarning" ) @pytest.mark.parametrize("mode", ["mean", "median"]) def test_zero_stat_length_valid(self, mode): @@ -1140,6 +1139,23 @@ class TestWrap: a = np.arange(5) b = np.pad(a, (0, 12), mode="wrap") assert_array_equal(np.r_[a, a, a, a][:-3], b) + + def test_repeated_wrapping_multiple_origin(self): + """ + Assert that 'wrap' pads only with multiples of the original area if + the pad width is larger than the original array. + """ + a = np.arange(4).reshape(2, 2) + a = np.pad(a, [(1, 3), (3, 1)], mode='wrap') + b = np.array( + [[3, 2, 3, 2, 3, 2], + [1, 0, 1, 0, 1, 0], + [3, 2, 3, 2, 3, 2], + [1, 0, 1, 0, 1, 0], + [3, 2, 3, 2, 3, 2], + [1, 0, 1, 0, 1, 0]] + ) + assert_array_equal(a, b) class TestEdge: @@ -1215,7 +1231,7 @@ def test_legacy_vector_functionality(): def test_unicode_mode(): - a = np.pad([1], 2, mode=u'constant') + a = np.pad([1], 2, mode='constant') b = np.array([0, 0, 1, 0, 0]) assert_array_equal(a, b) diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 81ba789e3..a180accbe 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -125,32 +125,36 @@ class TestSetOps: assert_array_equal([7, 1], ediff1d(two_elem, to_begin=7)) assert_array_equal([5, 6, 1], ediff1d(two_elem, to_begin=[5, 6])) - @pytest.mark.parametrize("ary, prepend, append", [ + @pytest.mark.parametrize("ary, prepend, append, expected", [ # should fail because trying to cast # np.nan standard floating point value # into an integer array: (np.array([1, 2, 3], dtype=np.int64), None, - np.nan), + np.nan, + 'to_end'), # should fail because attempting # to downcast to int type: (np.array([1, 2, 3], dtype=np.int64), np.array([5, 7, 2], dtype=np.float32), - None), + None, + 'to_begin'), # should fail because attempting to cast # two special floating point values - # to integers (on both sides of ary): + # to integers (on both sides of ary), + # `to_begin` is in the error message as the impl checks this first: (np.array([1., 3., 9.], dtype=np.int8), np.nan, - np.nan), + np.nan, + 'to_begin'), ]) - def test_ediff1d_forbidden_type_casts(self, ary, prepend, append): + def test_ediff1d_forbidden_type_casts(self, ary, prepend, append, expected): # verify resolution of gh-11490 # specifically, raise an appropriate # Exception when attempting to append or # prepend with an incompatible type - msg = 'must be compatible' + msg = 'dtype of `{}` must be compatible'.format(expected) with assert_raises_regex(TypeError, msg): ediff1d(ary=ary, to_end=append, @@ -191,7 +195,8 @@ class TestSetOps: assert_equal(actual, expected) assert actual.dtype == expected.dtype - def test_isin(self): + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_isin(self, kind): # the tests for in1d cover most of isin's behavior # if in1d is removed, would need to change those tests to test # isin instead. @@ -201,7 +206,7 @@ class TestSetOps: isin_slow = np.vectorize(_isin_slow, otypes=[bool], excluded={1}) def assert_isin_equal(a, b): - x = isin(a, b) + x = isin(a, b, kind=kind) y = isin_slow(a, b) assert_array_equal(x, y) @@ -227,12 +232,32 @@ class TestSetOps: assert_isin_equal(5, 6) # empty array-like: - x = [] - assert_isin_equal(x, b) - assert_isin_equal(a, x) - assert_isin_equal(x, x) - - def test_in1d(self): + if kind != "table": + # An empty list will become float64, + # which is invalid for kind="table" + x = [] + assert_isin_equal(x, b) + assert_isin_equal(a, x) + assert_isin_equal(x, x) + + # empty array with various types: + for dtype in [bool, np.int64, np.float64]: + if kind == "table" and dtype == np.float64: + continue + + if dtype in {np.int64, np.float64}: + ar = np.array([10, 20, 30], dtype=dtype) + elif dtype in {bool}: + ar = np.array([True, False, False]) + + empty_array = np.array([], dtype=dtype) + + assert_isin_equal(empty_array, ar) + assert_isin_equal(ar, empty_array) + assert_isin_equal(empty_array, empty_array) + + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d(self, kind): # we use two different sizes for the b array here to test the # two different paths in in1d(). for mult in (1, 10): @@ -240,57 +265,58 @@ class TestSetOps: a = [5, 7, 1, 2] b = [2, 4, 3, 1, 5] * mult ec = np.array([True, False, True, True]) - c = in1d(a, b, assume_unique=True) + c = in1d(a, b, assume_unique=True, kind=kind) assert_array_equal(c, ec) a[0] = 8 ec = np.array([False, False, True, True]) - c = in1d(a, b, assume_unique=True) + c = in1d(a, b, assume_unique=True, kind=kind) assert_array_equal(c, ec) a[0], a[3] = 4, 8 ec = np.array([True, False, True, False]) - c = in1d(a, b, assume_unique=True) + c = in1d(a, b, assume_unique=True, kind=kind) assert_array_equal(c, ec) a = np.array([5, 4, 5, 3, 4, 4, 3, 4, 3, 5, 2, 1, 5, 5]) b = [2, 3, 4] * mult ec = [False, True, False, True, True, True, True, True, True, False, True, False, False, False] - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) b = b + [5, 5, 4] * mult ec = [True, True, True, True, True, True, True, True, True, True, True, False, True, True] - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) a = np.array([5, 7, 1, 2]) b = np.array([2, 4, 3, 1, 5] * mult) ec = np.array([True, False, True, True]) - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) a = np.array([5, 7, 1, 1, 2]) b = np.array([2, 4, 3, 3, 1, 5] * mult) ec = np.array([True, False, True, True, True]) - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) a = np.array([5, 5]) b = np.array([2, 2] * mult) ec = np.array([False, False]) - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) a = np.array([5]) b = np.array([2]) ec = np.array([False]) - c = in1d(a, b) + c = in1d(a, b, kind=kind) assert_array_equal(c, ec) - assert_array_equal(in1d([], []), []) + if kind in {None, "sort"}: + assert_array_equal(in1d([], [], kind=kind), []) def test_in1d_char_array(self): a = np.array(['a', 'b', 'c', 'd', 'e', 'c', 'e', 'b']) @@ -301,16 +327,29 @@ class TestSetOps: assert_array_equal(c, ec) - def test_in1d_invert(self): + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d_invert(self, kind): "Test in1d's invert parameter" # We use two different sizes for the b array here to test the # two different paths in in1d(). for mult in (1, 10): a = np.array([5, 4, 5, 3, 4, 4, 3, 4, 3, 5, 2, 1, 5, 5]) b = [2, 3, 4] * mult - assert_array_equal(np.invert(in1d(a, b)), in1d(a, b, invert=True)) - - def test_in1d_ravel(self): + assert_array_equal(np.invert(in1d(a, b, kind=kind)), + in1d(a, b, invert=True, kind=kind)) + + # float: + if kind in {None, "sort"}: + for mult in (1, 10): + a = np.array([5, 4, 5, 3, 4, 4, 3, 4, 3, 5, 2, 1, 5, 5], + dtype=np.float32) + b = [2, 3, 4] * mult + b = np.array(b, dtype=np.float32) + assert_array_equal(np.invert(in1d(a, b, kind=kind)), + in1d(a, b, invert=True, kind=kind)) + + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d_ravel(self, kind): # Test that in1d ravels its input arrays. This is not documented # behavior however. The test is to ensure consistentency. a = np.arange(6).reshape(2, 3) @@ -318,10 +357,110 @@ class TestSetOps: long_b = np.arange(3, 63).reshape(30, 2) ec = np.array([False, False, False, True, True, True]) - assert_array_equal(in1d(a, b, assume_unique=True), ec) - assert_array_equal(in1d(a, b, assume_unique=False), ec) - assert_array_equal(in1d(a, long_b, assume_unique=True), ec) - assert_array_equal(in1d(a, long_b, assume_unique=False), ec) + assert_array_equal(in1d(a, b, assume_unique=True, kind=kind), + ec) + assert_array_equal(in1d(a, b, assume_unique=False, + kind=kind), + ec) + assert_array_equal(in1d(a, long_b, assume_unique=True, + kind=kind), + ec) + assert_array_equal(in1d(a, long_b, assume_unique=False, + kind=kind), + ec) + + def test_in1d_hit_alternate_algorithm(self): + """Hit the standard isin code with integers""" + # Need extreme range to hit standard code + # This hits it without the use of kind='table' + a = np.array([5, 4, 5, 3, 4, 4, 1e9], dtype=np.int64) + b = np.array([2, 3, 4, 1e9], dtype=np.int64) + expected = np.array([0, 1, 0, 1, 1, 1, 1], dtype=bool) + assert_array_equal(expected, in1d(a, b)) + assert_array_equal(np.invert(expected), in1d(a, b, invert=True)) + + a = np.array([5, 7, 1, 2], dtype=np.int64) + b = np.array([2, 4, 3, 1, 5, 1e9], dtype=np.int64) + ec = np.array([True, False, True, True]) + c = in1d(a, b, assume_unique=True) + assert_array_equal(c, ec) + + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d_boolean(self, kind): + """Test that in1d works for boolean input""" + a = np.array([True, False]) + b = np.array([False, False, False]) + expected = np.array([False, True]) + assert_array_equal(expected, + in1d(a, b, kind=kind)) + assert_array_equal(np.invert(expected), + in1d(a, b, invert=True, kind=kind)) + + @pytest.mark.parametrize("kind", [None, "sort"]) + def test_in1d_timedelta(self, kind): + """Test that in1d works for timedelta input""" + rstate = np.random.RandomState(0) + a = rstate.randint(0, 100, size=10) + b = rstate.randint(0, 100, size=10) + truth = in1d(a, b) + a_timedelta = a.astype("timedelta64[s]") + b_timedelta = b.astype("timedelta64[s]") + assert_array_equal(truth, in1d(a_timedelta, b_timedelta, kind=kind)) + + def test_in1d_table_timedelta_fails(self): + a = np.array([0, 1, 2], dtype="timedelta64[s]") + b = a + # Make sure it raises a value error: + with pytest.raises(ValueError): + in1d(a, b, kind="table") + + @pytest.mark.parametrize( + "dtype1,dtype2", + [ + (np.int8, np.int16), + (np.int16, np.int8), + (np.uint8, np.uint16), + (np.uint16, np.uint8), + (np.uint8, np.int16), + (np.int16, np.uint8), + ] + ) + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d_mixed_dtype(self, dtype1, dtype2, kind): + """Test that in1d works as expected for mixed dtype input.""" + is_dtype2_signed = np.issubdtype(dtype2, np.signedinteger) + ar1 = np.array([0, 0, 1, 1], dtype=dtype1) + + if is_dtype2_signed: + ar2 = np.array([-128, 0, 127], dtype=dtype2) + else: + ar2 = np.array([127, 0, 255], dtype=dtype2) + + expected = np.array([True, True, False, False]) + + expect_failure = kind == "table" and any(( + dtype1 == np.int8 and dtype2 == np.int16, + dtype1 == np.int16 and dtype2 == np.int8 + )) + + if expect_failure: + with pytest.raises(RuntimeError, match="exceed the maximum"): + in1d(ar1, ar2, kind=kind) + else: + assert_array_equal(in1d(ar1, ar2, kind=kind), expected) + + @pytest.mark.parametrize("kind", [None, "sort", "table"]) + def test_in1d_mixed_boolean(self, kind): + """Test that in1d works as expected for bool/int input.""" + for dtype in np.typecodes["AllInteger"]: + a = np.array([True, False, False], dtype=bool) + b = np.array([0, 0, 0, 0], dtype=dtype) + expected = np.array([False, True, True], dtype=bool) + assert_array_equal(in1d(a, b, kind=kind), expected) + + a, b = b, a + expected = np.array([True, True, True, True], dtype=bool) + assert_array_equal(in1d(a, b, kind=kind), expected) def test_in1d_first_array_is_object(self): ar1 = [None] @@ -354,6 +493,73 @@ class TestSetOps: result = np.in1d(ar1, ar2) assert_array_equal(result, expected) + def test_in1d_with_arrays_containing_tuples(self): + ar1 = np.array([(1,), 2], dtype=object) + ar2 = np.array([(1,), 2], dtype=object) + expected = np.array([True, True]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + + # An integer is added at the end of the array to make sure + # that the array builder will create the array with tuples + # and after it's created the integer is removed. + # There's a bug in the array constructor that doesn't handle + # tuples properly and adding the integer fixes that. + ar1 = np.array([(1,), (2, 1), 1], dtype=object) + ar1 = ar1[:-1] + ar2 = np.array([(1,), (2, 1), 1], dtype=object) + ar2 = ar2[:-1] + expected = np.array([True, True]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + + ar1 = np.array([(1,), (2, 3), 1], dtype=object) + ar1 = ar1[:-1] + ar2 = np.array([(1,), 2], dtype=object) + expected = np.array([True, False]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + + def test_in1d_errors(self): + """Test that in1d raises expected errors.""" + + # Error 1: `kind` is not one of 'sort' 'table' or None. + ar1 = np.array([1, 2, 3, 4, 5]) + ar2 = np.array([2, 4, 6, 8, 10]) + assert_raises(ValueError, in1d, ar1, ar2, kind='quicksort') + + # Error 2: `kind="table"` does not work for non-integral arrays. + obj_ar1 = np.array([1, 'a', 3, 'b', 5], dtype=object) + obj_ar2 = np.array([1, 'a', 3, 'b', 5], dtype=object) + assert_raises(ValueError, in1d, obj_ar1, obj_ar2, kind='table') + + for dtype in [np.int32, np.int64]: + ar1 = np.array([-1, 2, 3, 4, 5], dtype=dtype) + # The range of this array will overflow: + overflow_ar2 = np.array([-1, np.iinfo(dtype).max], dtype=dtype) + + # Error 3: `kind="table"` will trigger a runtime error + # if there is an integer overflow expected when computing the + # range of ar2 + assert_raises( + RuntimeError, + in1d, ar1, overflow_ar2, kind='table' + ) + + # Non-error: `kind=None` will *not* trigger a runtime error + # if there is an integer overflow, it will switch to + # the `sort` algorithm. + result = np.in1d(ar1, overflow_ar2, kind=None) + assert_array_equal(result, [True] + [False] * 4) + result = np.in1d(ar1, overflow_ar2, kind='sort') + assert_array_equal(result, [True] + [False] * 4) + def test_union1d(self): a = np.array([5, 4, 7, 1, 2]) b = np.array([2, 4, 3, 3, 2, 1, 5]) @@ -527,6 +733,63 @@ class TestUnique: assert_equal(a3_idx.dtype, np.intp) assert_equal(a3_inv.dtype, np.intp) + # test for ticket 2111 - float + a = [2.0, np.nan, 1.0, np.nan] + ua = [1.0, 2.0, np.nan] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - complex + a = [2.0-1j, np.nan, 1.0+1j, complex(0.0, np.nan), complex(1.0, np.nan)] + ua = [1.0+1j, 2.0-1j, complex(0.0, np.nan)] + ua_idx = [2, 0, 3] + ua_inv = [1, 2, 0, 2, 2] + ua_cnt = [1, 1, 3] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - datetime64 + nat = np.datetime64('nat') + a = [np.datetime64('2020-12-26'), nat, np.datetime64('2020-12-24'), nat] + ua = [np.datetime64('2020-12-24'), np.datetime64('2020-12-26'), nat] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - timedelta + nat = np.timedelta64('nat') + a = [np.timedelta64(1, 'D'), nat, np.timedelta64(1, 'h'), nat] + ua = [np.timedelta64(1, 'h'), np.timedelta64(1, 'D'), nat] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for gh-19300 + all_nans = [np.nan] * 4 + ua = [np.nan] + ua_idx = [0] + ua_inv = [0, 0, 0, 0] + ua_cnt = [4] + assert_equal(np.unique(all_nans), ua) + assert_equal(np.unique(all_nans, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(all_nans, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(all_nans, return_counts=True), (ua, ua_cnt)) + def test_unique_axis_errors(self): assert_raises(TypeError, self._run_axis_tests, object) assert_raises(TypeError, self._run_axis_tests, @@ -671,3 +934,11 @@ class TestUnique: assert_array_equal(uniq[:, inv], data) msg = "Unique's return_counts=True failed with axis=1" assert_array_equal(cnt, np.array([2, 1, 1]), msg) + + def test_unique_nanequals(self): + # issue 20326 + a = np.array([1, 1, np.nan, np.nan, np.nan]) + unq = np.unique(a) + not_unq = np.unique(a, equal_nan=False) + assert_array_equal(unq, np.array([1, np.nan])) + assert_array_equal(not_unq, np.array([1, np.nan, np.nan, np.nan])) diff --git a/numpy/lib/tests/test_financial.py b/numpy/lib/tests/test_financial.py deleted file mode 100644 index 26e79bc06..000000000 --- a/numpy/lib/tests/test_financial.py +++ /dev/null @@ -1,380 +0,0 @@ -import warnings -from decimal import Decimal - -import numpy as np -from numpy.testing import ( - assert_, assert_almost_equal, assert_allclose, assert_equal, assert_raises - ) - - -def filter_deprecation(func): - def newfunc(*args, **kwargs): - with warnings.catch_warnings(record=True) as ws: - warnings.filterwarnings('always', category=DeprecationWarning) - func(*args, **kwargs) - assert_(all(w.category is DeprecationWarning for w in ws)) - return newfunc - - -class TestFinancial: - @filter_deprecation - def test_npv_irr_congruence(self): - # IRR is defined as the rate required for the present value of a - # a series of cashflows to be zero i.e. NPV(IRR(x), x) = 0 - cashflows = np.array([-40000, 5000, 8000, 12000, 30000]) - assert_allclose(np.npv(np.irr(cashflows), cashflows), 0, atol=1e-10, rtol=0) - - @filter_deprecation - def test_rate(self): - assert_almost_equal( - np.rate(10, 0, -3500, 10000), - 0.1107, 4) - - @filter_deprecation - def test_rate_decimal(self): - rate = np.rate(Decimal('10'), Decimal('0'), Decimal('-3500'), Decimal('10000')) - assert_equal(Decimal('0.1106908537142689284704528100'), rate) - - @filter_deprecation - def test_irr(self): - v = [-150000, 15000, 25000, 35000, 45000, 60000] - assert_almost_equal(np.irr(v), 0.0524, 2) - v = [-100, 0, 0, 74] - assert_almost_equal(np.irr(v), -0.0955, 2) - v = [-100, 39, 59, 55, 20] - assert_almost_equal(np.irr(v), 0.28095, 2) - v = [-100, 100, 0, -7] - assert_almost_equal(np.irr(v), -0.0833, 2) - v = [-100, 100, 0, 7] - assert_almost_equal(np.irr(v), 0.06206, 2) - v = [-5, 10.5, 1, -8, 1] - assert_almost_equal(np.irr(v), 0.0886, 2) - - # Test that if there is no solution then np.irr returns nan - # Fixes gh-6744 - v = [-1, -2, -3] - assert_equal(np.irr(v), np.nan) - - @filter_deprecation - def test_pv(self): - assert_almost_equal(np.pv(0.07, 20, 12000, 0), -127128.17, 2) - - @filter_deprecation - def test_pv_decimal(self): - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), - Decimal('-127128.1709461939327295222005')) - - @filter_deprecation - def test_fv(self): - assert_equal(np.fv(0.075, 20, -2000, 0, 0), 86609.362673042924) - - @filter_deprecation - def test_fv_decimal(self): - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), 0, 0), - Decimal('86609.36267304300040536731624')) - - @filter_deprecation - def test_pmt(self): - res = np.pmt(0.08 / 12, 5 * 12, 15000) - tgt = -304.145914 - assert_allclose(res, tgt) - # Test the edge case where rate == 0.0 - res = np.pmt(0.0, 5 * 12, 15000) - tgt = -250.0 - assert_allclose(res, tgt) - # Test the case where we use broadcast and - # the arguments passed in are arrays. - res = np.pmt([[0.0, 0.8], [0.3, 0.8]], [12, 3], [2000, 20000]) - tgt = np.array([[-166.66667, -19311.258], [-626.90814, -19311.258]]) - assert_allclose(res, tgt) - - @filter_deprecation - def test_pmt_decimal(self): - res = np.pmt(Decimal('0.08') / Decimal('12'), 5 * 12, 15000) - tgt = Decimal('-304.1459143262052370338701494') - assert_equal(res, tgt) - # Test the edge case where rate == 0.0 - res = np.pmt(Decimal('0'), Decimal('60'), Decimal('15000')) - tgt = -250 - assert_equal(res, tgt) - # Test the case where we use broadcast and - # the arguments passed in are arrays. - res = np.pmt([[Decimal('0'), Decimal('0.8')], [Decimal('0.3'), Decimal('0.8')]], - [Decimal('12'), Decimal('3')], [Decimal('2000'), Decimal('20000')]) - tgt = np.array([[Decimal('-166.6666666666666666666666667'), Decimal('-19311.25827814569536423841060')], - [Decimal('-626.9081401700757748402586600'), Decimal('-19311.25827814569536423841060')]]) - - # Cannot use the `assert_allclose` because it uses isfinite under the covers - # which does not support the Decimal type - # See issue: https://github.com/numpy/numpy/issues/9954 - assert_equal(res[0][0], tgt[0][0]) - assert_equal(res[0][1], tgt[0][1]) - assert_equal(res[1][0], tgt[1][0]) - assert_equal(res[1][1], tgt[1][1]) - - @filter_deprecation - def test_ppmt(self): - assert_equal(np.round(np.ppmt(0.1 / 12, 1, 60, 55000), 2), -710.25) - - @filter_deprecation - def test_ppmt_decimal(self): - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000')), - Decimal('-710.2541257864217612489830917')) - - # Two tests showing how Decimal is actually getting at a more exact result - # .23 / 12 does not come out nicely as a float but does as a decimal - @filter_deprecation - def test_ppmt_special_rate(self): - assert_equal(np.round(np.ppmt(0.23 / 12, 1, 60, 10000000000), 8), -90238044.232277036) - - @filter_deprecation - def test_ppmt_special_rate_decimal(self): - # When rounded out to 8 decimal places like the float based test, this should not equal the same value - # as the float, substituted for the decimal - def raise_error_because_not_equal(): - assert_equal( - round(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), 8), - Decimal('-90238044.232277036')) - - assert_raises(AssertionError, raise_error_because_not_equal) - assert_equal(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), - Decimal('-90238044.2322778884413969909')) - - @filter_deprecation - def test_ipmt(self): - assert_almost_equal(np.round(np.ipmt(0.1 / 12, 1, 24, 2000), 2), -16.67) - - @filter_deprecation - def test_ipmt_decimal(self): - result = np.ipmt(Decimal('0.1') / Decimal('12'), 1, 24, 2000) - assert_equal(result.flat[0], Decimal('-16.66666666666666666666666667')) - - @filter_deprecation - def test_nper(self): - assert_almost_equal(np.nper(0.075, -2000, 0, 100000.), - 21.54, 2) - - @filter_deprecation - def test_nper2(self): - assert_almost_equal(np.nper(0.0, -2000, 0, 100000.), - 50.0, 1) - - @filter_deprecation - def test_npv(self): - assert_almost_equal( - np.npv(0.05, [-15000, 1500, 2500, 3500, 4500, 6000]), - 122.89, 2) - - @filter_deprecation - def test_npv_decimal(self): - assert_equal( - np.npv(Decimal('0.05'), [-15000, 1500, 2500, 3500, 4500, 6000]), - Decimal('122.894854950942692161628715')) - - @filter_deprecation - def test_mirr(self): - val = [-4500, -800, 800, 800, 600, 600, 800, 800, 700, 3000] - assert_almost_equal(np.mirr(val, 0.08, 0.055), 0.0666, 4) - - val = [-120000, 39000, 30000, 21000, 37000, 46000] - assert_almost_equal(np.mirr(val, 0.10, 0.12), 0.126094, 6) - - val = [100, 200, -50, 300, -200] - assert_almost_equal(np.mirr(val, 0.05, 0.06), 0.3428, 4) - - val = [39000, 30000, 21000, 37000, 46000] - assert_(np.isnan(np.mirr(val, 0.10, 0.12))) - - @filter_deprecation - def test_mirr_decimal(self): - val = [Decimal('-4500'), Decimal('-800'), Decimal('800'), Decimal('800'), - Decimal('600'), Decimal('600'), Decimal('800'), Decimal('800'), - Decimal('700'), Decimal('3000')] - assert_equal(np.mirr(val, Decimal('0.08'), Decimal('0.055')), - Decimal('0.066597175031553548874239618')) - - val = [Decimal('-120000'), Decimal('39000'), Decimal('30000'), - Decimal('21000'), Decimal('37000'), Decimal('46000')] - assert_equal(np.mirr(val, Decimal('0.10'), Decimal('0.12')), Decimal('0.126094130365905145828421880')) - - val = [Decimal('100'), Decimal('200'), Decimal('-50'), - Decimal('300'), Decimal('-200')] - assert_equal(np.mirr(val, Decimal('0.05'), Decimal('0.06')), Decimal('0.342823387842176663647819868')) - - val = [Decimal('39000'), Decimal('30000'), Decimal('21000'), Decimal('37000'), Decimal('46000')] - assert_(np.isnan(np.mirr(val, Decimal('0.10'), Decimal('0.12')))) - - @filter_deprecation - def test_when(self): - # begin - assert_equal(np.rate(10, 20, -3500, 10000, 1), - np.rate(10, 20, -3500, 10000, 'begin')) - # end - assert_equal(np.rate(10, 20, -3500, 10000), - np.rate(10, 20, -3500, 10000, 'end')) - assert_equal(np.rate(10, 20, -3500, 10000, 0), - np.rate(10, 20, -3500, 10000, 'end')) - - # begin - assert_equal(np.pv(0.07, 20, 12000, 0, 1), - np.pv(0.07, 20, 12000, 0, 'begin')) - # end - assert_equal(np.pv(0.07, 20, 12000, 0), - np.pv(0.07, 20, 12000, 0, 'end')) - assert_equal(np.pv(0.07, 20, 12000, 0, 0), - np.pv(0.07, 20, 12000, 0, 'end')) - - # begin - assert_equal(np.fv(0.075, 20, -2000, 0, 1), - np.fv(0.075, 20, -2000, 0, 'begin')) - # end - assert_equal(np.fv(0.075, 20, -2000, 0), - np.fv(0.075, 20, -2000, 0, 'end')) - assert_equal(np.fv(0.075, 20, -2000, 0, 0), - np.fv(0.075, 20, -2000, 0, 'end')) - - # begin - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0, 1), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'begin')) - # end - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'end')) - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0, 0), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'end')) - - # begin - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0, 1), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'begin')) - # end - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'end')) - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0, 0), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'end')) - - # begin - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0, 1), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'begin')) - # end - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'end')) - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0, 0), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'end')) - - # begin - assert_equal(np.nper(0.075, -2000, 0, 100000., 1), - np.nper(0.075, -2000, 0, 100000., 'begin')) - # end - assert_equal(np.nper(0.075, -2000, 0, 100000.), - np.nper(0.075, -2000, 0, 100000., 'end')) - assert_equal(np.nper(0.075, -2000, 0, 100000., 0), - np.nper(0.075, -2000, 0, 100000., 'end')) - - @filter_deprecation - def test_decimal_with_when(self): - """Test that decimals are still supported if the when argument is passed""" - # begin - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), Decimal('1')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'begin')) - # end - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'end')) - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), Decimal('0')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'end')) - - # begin - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), Decimal('1')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'begin')) - # end - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'end')) - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), Decimal('0')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'end')) - - # begin - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), Decimal('1')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'begin')) - # end - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'end')) - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), Decimal('0')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'end')) - - # begin - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), Decimal('1')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'begin')) - # end - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'end')) - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), Decimal('0')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'end')) - - # begin - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), Decimal('1')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'begin')) - # end - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'end')) - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), Decimal('0')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'end')) - - # begin - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), Decimal('1')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'begin').flat[0]) - # end - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'end').flat[0]) - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), Decimal('0')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'end').flat[0]) - - @filter_deprecation - def test_broadcast(self): - assert_almost_equal(np.nper(0.075, -2000, 0, 100000., [0, 1]), - [21.5449442, 20.76156441], 4) - - assert_almost_equal(np.ipmt(0.1 / 12, list(range(5)), 24, 2000), - [-17.29165168, -16.66666667, -16.03647345, - -15.40102862, -14.76028842], 4) - - assert_almost_equal(np.ppmt(0.1 / 12, list(range(5)), 24, 2000), - [-74.998201, -75.62318601, -76.25337923, - -76.88882405, -77.52956425], 4) - - assert_almost_equal(np.ppmt(0.1 / 12, list(range(5)), 24, 2000, 0, - [0, 0, 1, 'end', 'begin']), - [-74.998201, -75.62318601, -75.62318601, - -76.88882405, -76.88882405], 4) - - @filter_deprecation - def test_broadcast_decimal(self): - # Use almost equal because precision is tested in the explicit tests, this test is to ensure - # broadcast with Decimal is not broken. - assert_almost_equal(np.ipmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000')), - [Decimal('-17.29165168'), Decimal('-16.66666667'), Decimal('-16.03647345'), - Decimal('-15.40102862'), Decimal('-14.76028842')], 4) - - assert_almost_equal(np.ppmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000')), - [Decimal('-74.998201'), Decimal('-75.62318601'), Decimal('-76.25337923'), - Decimal('-76.88882405'), Decimal('-77.52956425')], 4) - - assert_almost_equal(np.ppmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000'), - Decimal('0'), [Decimal('0'), Decimal('0'), Decimal('1'), 'end', 'begin']), - [Decimal('-74.998201'), Decimal('-75.62318601'), Decimal('-75.62318601'), - Decimal('-76.88882405'), Decimal('-76.88882405')], 4) diff --git a/numpy/lib/tests/test_financial_expired.py b/numpy/lib/tests/test_financial_expired.py new file mode 100644 index 000000000..838f999a6 --- /dev/null +++ b/numpy/lib/tests/test_financial_expired.py @@ -0,0 +1,11 @@ +import sys +import pytest +import numpy as np + + +def test_financial_expired(): + match = 'NEP 32' + with pytest.warns(DeprecationWarning, match=match): + func = np.fv + with pytest.raises(RuntimeError, match=match): + func(1, 2, 3) diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 1bfd63bba..58d08f1e5 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -276,8 +276,6 @@ Test the header writing. ''' import sys import os -import shutil -import tempfile import warnings import pytest from io import BytesIO @@ -285,28 +283,12 @@ from io import BytesIO import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_raises, assert_raises_regex, - assert_warns, IS_PYPY, break_cycles + assert_warns, IS_PYPY, IS_WASM ) +from numpy.testing._private.utils import requires_memory from numpy.lib import format -tempdir = None - -# Module-level setup. - - -def setup_module(): - global tempdir - tempdir = tempfile.mkdtemp() - - -def teardown_module(): - global tempdir - if tempdir is not None and os.path.isdir(tempdir): - shutil.rmtree(tempdir) - tempdir = None - - # Generate some basic arrays to test with. scalars = [ np.uint8, @@ -419,7 +401,7 @@ class BytesIOSRandomSize(BytesIO): def read(self, size=None): import random size = random.randint(1, size) - return super(BytesIOSRandomSize, self).read(size) + return super().read(size) def roundtrip(arr): @@ -477,46 +459,41 @@ def test_long_str(): assert_array_equal(long_str_arr, long_str_arr2) +@pytest.mark.skipif(IS_WASM, reason="memmap doesn't work correctly") @pytest.mark.slow -def test_memmap_roundtrip(): - # Fixme: used to crash on windows - if not (sys.platform == 'win32' or sys.platform == 'cygwin'): - for arr in basic_arrays + record_arrays: - if arr.dtype.hasobject: - # Skip these since they can't be mmap'ed. - continue - # Write it out normally and through mmap. - nfn = os.path.join(tempdir, 'normal.npy') - mfn = os.path.join(tempdir, 'memmap.npy') - with open(nfn, 'wb') as fp: - format.write_array(fp, arr) - - fortran_order = ( - arr.flags.f_contiguous and not arr.flags.c_contiguous) - ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype, - shape=arr.shape, fortran_order=fortran_order) - ma[...] = arr - del ma - if IS_PYPY: - break_cycles() - - # Check that both of these files' contents are the same. - with open(nfn, 'rb') as fp: - normal_bytes = fp.read() - with open(mfn, 'rb') as fp: - memmap_bytes = fp.read() - assert_equal_(normal_bytes, memmap_bytes) - - # Check that reading the file using memmap works. - ma = format.open_memmap(nfn, mode='r') - del ma - if IS_PYPY: - break_cycles() - - -def test_compressed_roundtrip(): +def test_memmap_roundtrip(tmpdir): + for i, arr in enumerate(basic_arrays + record_arrays): + if arr.dtype.hasobject: + # Skip these since they can't be mmap'ed. + continue + # Write it out normally and through mmap. + nfn = os.path.join(tmpdir, f'normal{i}.npy') + mfn = os.path.join(tmpdir, f'memmap{i}.npy') + with open(nfn, 'wb') as fp: + format.write_array(fp, arr) + + fortran_order = ( + arr.flags.f_contiguous and not arr.flags.c_contiguous) + ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype, + shape=arr.shape, fortran_order=fortran_order) + ma[...] = arr + ma.flush() + + # Check that both of these files' contents are the same. + with open(nfn, 'rb') as fp: + normal_bytes = fp.read() + with open(mfn, 'rb') as fp: + memmap_bytes = fp.read() + assert_equal_(normal_bytes, memmap_bytes) + + # Check that reading the file using memmap works. + ma = format.open_memmap(nfn, mode='r') + ma.flush() + + +def test_compressed_roundtrip(tmpdir): arr = np.random.rand(200, 200) - npz_file = os.path.join(tempdir, 'compressed.npz') + npz_file = os.path.join(tmpdir, 'compressed.npz') np.savez_compressed(npz_file, arr=arr) with np.load(npz_file) as npz: arr1 = npz['arr'] @@ -539,21 +516,23 @@ dt5 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], dt6 = np.dtype({'names': [], 'formats': [], 'itemsize': 8}) @pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5, dt6]) -def test_load_padded_dtype(dt): +def test_load_padded_dtype(tmpdir, dt): arr = np.zeros(3, dt) for i in range(3): arr[i] = i + 5 - npz_file = os.path.join(tempdir, 'aligned.npz') + npz_file = os.path.join(tmpdir, 'aligned.npz') np.savez(npz_file, arr=arr) with np.load(npz_file) as npz: arr1 = npz['arr'] assert_array_equal(arr, arr1) +@pytest.mark.xfail(IS_WASM, reason="Emscripten NODEFS has a buggy dup") def test_python2_python3_interoperability(): fname = 'win64python2.npy' path = os.path.join(os.path.dirname(__file__), 'data', fname) - data = np.load(path) + with pytest.warns(UserWarning, match="Reading.*this warning\\."): + data = np.load(path) assert_array_equal(data, np.ones(2)) def test_pickle_python2_python3(): @@ -561,7 +540,7 @@ def test_pickle_python2_python3(): # Python 2 and Python 3 and vice versa data_dir = os.path.join(os.path.dirname(__file__), 'data') - expected = np.array([None, range, u'\u512a\u826f', + expected = np.array([None, range, '\u512a\u826f', b'\xe4\xb8\x8d\xe8\x89\xaf'], dtype=object) @@ -603,7 +582,7 @@ def test_pickle_python2_python3(): encoding='latin1') -def test_pickle_disallow(): +def test_pickle_disallow(tmpdir): data_dir = os.path.join(os.path.dirname(__file__), 'data') path = os.path.join(data_dir, 'py2-objarr.npy') @@ -614,7 +593,7 @@ def test_pickle_disallow(): with np.load(path, allow_pickle=False, encoding='latin1') as f: assert_raises(ValueError, f.__getitem__, 'x') - path = os.path.join(tempdir, 'pickle-disabled.npy') + path = os.path.join(tmpdir, 'pickle-disabled.npy') assert_raises(ValueError, np.save, path, np.array([None], dtype=object), allow_pickle=False) @@ -692,47 +671,83 @@ def test_version_2_0(): assert_(len(header) % format.ARRAY_ALIGN == 0) f.seek(0) - n = format.read_array(f) + n = format.read_array(f, max_header_size=200000) assert_array_equal(d, n) # 1.0 requested but data cannot be saved this way assert_raises(ValueError, format.write_array, f, d, (1, 0)) -@pytest.mark.slow -def test_version_2_0_memmap(): +@pytest.mark.skipif(IS_WASM, reason="memmap doesn't work correctly") +def test_version_2_0_memmap(tmpdir): # requires more than 2 byte for header dt = [(("%d" % i) * 100, float) for i in range(500)] d = np.ones(1000, dtype=dt) - tf = tempfile.mktemp('', 'mmap', dir=tempdir) + tf1 = os.path.join(tmpdir, f'version2_01.npy') + tf2 = os.path.join(tmpdir, f'version2_02.npy') # 1.0 requested but data cannot be saved this way - assert_raises(ValueError, format.open_memmap, tf, mode='w+', dtype=d.dtype, + assert_raises(ValueError, format.open_memmap, tf1, mode='w+', dtype=d.dtype, shape=d.shape, version=(1, 0)) - ma = format.open_memmap(tf, mode='w+', dtype=d.dtype, + ma = format.open_memmap(tf1, mode='w+', dtype=d.dtype, shape=d.shape, version=(2, 0)) ma[...] = d - del ma - if IS_PYPY: - break_cycles() + ma.flush() + ma = format.open_memmap(tf1, mode='r', max_header_size=200000) + assert_array_equal(ma, d) with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', UserWarning) - ma = format.open_memmap(tf, mode='w+', dtype=d.dtype, + ma = format.open_memmap(tf2, mode='w+', dtype=d.dtype, shape=d.shape, version=None) assert_(w[0].category is UserWarning) ma[...] = d - del ma - if IS_PYPY: - break_cycles() + ma.flush() + + ma = format.open_memmap(tf2, mode='r', max_header_size=200000) - ma = format.open_memmap(tf, mode='r') assert_array_equal(ma, d) - del ma - if IS_PYPY: - break_cycles() +@pytest.mark.parametrize("mmap_mode", ["r", None]) +def test_huge_header(tmpdir, mmap_mode): + f = os.path.join(tmpdir, f'large_header.npy') + arr = np.array(1, dtype="i,"*10000+"i") + + with pytest.warns(UserWarning, match=".*format 2.0"): + np.save(f, arr) + + with pytest.raises(ValueError, match="Header.*large"): + np.load(f, mmap_mode=mmap_mode) + + with pytest.raises(ValueError, match="Header.*large"): + np.load(f, mmap_mode=mmap_mode, max_header_size=20000) + + res = np.load(f, mmap_mode=mmap_mode, allow_pickle=True) + assert_array_equal(res, arr) + + res = np.load(f, mmap_mode=mmap_mode, max_header_size=180000) + assert_array_equal(res, arr) + +def test_huge_header_npz(tmpdir): + f = os.path.join(tmpdir, f'large_header.npz') + arr = np.array(1, dtype="i,"*10000+"i") + + with pytest.warns(UserWarning, match=".*format 2.0"): + np.savez(f, arr=arr) + + # Only getting the array from the file actually reads it + with pytest.raises(ValueError, match="Header.*large"): + np.load(f)["arr"] + + with pytest.raises(ValueError, match="Header.*large"): + np.load(f, max_header_size=20000)["arr"] + + res = np.load(f, allow_pickle=True)["arr"] + assert_array_equal(res, arr) + + res = np.load(f, max_header_size=180000)["arr"] + assert_array_equal(res, arr) def test_write_version(): f = BytesIO() @@ -821,11 +836,11 @@ def test_bad_magic_args(): def test_large_header(): s = BytesIO() - d = {'a': 1, 'b': 2} + d = {'shape': tuple(), 'fortran_order': False, 'descr': '<i8'} format.write_array_header_1_0(s, d) s = BytesIO() - d = {'a': 1, 'b': 2, 'c': 'x'*256*256} + d['descr'] = [('x'*256*256, '<i8')] assert_raises(ValueError, format.write_array_header_1_0, s, d) @@ -867,10 +882,12 @@ def test_bad_header(): assert_raises(ValueError, format.read_array_header_1_0, s) # headers without the exact keys required should fail - d = {"shape": (1, 2), - "descr": "x"} - s = BytesIO() - format.write_array_header_1_0(s, d) + # d = {"shape": (1, 2), + # "descr": "x"} + s = BytesIO( + b"\x93NUMPY\x01\x006\x00{'descr': 'x', 'shape': (1, 2), }" + + b" \n" + ) assert_raises(ValueError, format.read_array_header_1_0, s) d = {"shape": (1, 2), @@ -882,11 +899,11 @@ def test_bad_header(): assert_raises(ValueError, format.read_array_header_1_0, s) -def test_large_file_support(): +def test_large_file_support(tmpdir): if (sys.platform == 'win32' or sys.platform == 'cygwin'): pytest.skip("Unknown if Windows has sparse filesystems") # try creating a large sparse file - tf_name = os.path.join(tempdir, 'sparse_file') + tf_name = os.path.join(tmpdir, 'sparse_file') try: # seek past end would work too, but linux truncate somewhat # increases the chances that we have a sparse filesystem and can @@ -907,37 +924,42 @@ def test_large_file_support(): assert_array_equal(r, d) +@pytest.mark.skipif(IS_PYPY, reason="flaky on PyPy") @pytest.mark.skipif(np.dtype(np.intp).itemsize < 8, reason="test requires 64-bit system") @pytest.mark.slow -def test_large_archive(): +@requires_memory(free_bytes=2 * 2**30) +def test_large_archive(tmpdir): # Regression test for product of saving arrays with dimensions of array # having a product that doesn't fit in int32. See gh-7598 for details. + shape = (2**30, 2) try: - a = np.empty((2**30, 2), dtype=np.uint8) + a = np.empty(shape, dtype=np.uint8) except MemoryError: pytest.skip("Could not create large file") - fname = os.path.join(tempdir, "large_archive") + fname = os.path.join(tmpdir, "large_archive") with open(fname, "wb") as f: np.savez(f, arr=a) + del a + with open(fname, "rb") as f: new_a = np.load(f)["arr"] - assert_(a.shape == new_a.shape) + assert new_a.shape == shape -def test_empty_npz(): +def test_empty_npz(tmpdir): # Test for gh-9989 - fname = os.path.join(tempdir, "nothing.npz") + fname = os.path.join(tmpdir, "nothing.npz") np.savez(fname) with np.load(fname) as nps: pass -def test_unicode_field_names(): +def test_unicode_field_names(tmpdir): # gh-7391 arr = np.array([ (1, 3), @@ -946,9 +968,9 @@ def test_unicode_field_names(): (1, 2) ], dtype=[ ('int', int), - (u'\N{CJK UNIFIED IDEOGRAPH-6574}\N{CJK UNIFIED IDEOGRAPH-5F62}', int) + ('\N{CJK UNIFIED IDEOGRAPH-6574}\N{CJK UNIFIED IDEOGRAPH-5F62}', int) ]) - fname = os.path.join(tempdir, "unicode.npy") + fname = os.path.join(tmpdir, "unicode.npy") with open(fname, 'wb') as f: format.write_array(f, arr, version=(3, 0)) with open(fname, 'rb') as f: @@ -960,6 +982,19 @@ def test_unicode_field_names(): with assert_warns(UserWarning): format.write_array(f, arr, version=None) +def test_header_growth_axis(): + for is_fortran_array, dtype_space, expected_header_length in [ + [False, 22, 128], [False, 23, 192], [True, 23, 128], [True, 24, 192] + ]: + for size in [10**i for i in range(format.GROWTH_AXIS_MAX_DIGITS)]: + fp = BytesIO() + format.write_array_header_1_0(fp, { + 'shape': (2, size) if is_fortran_array else (size, 2), + 'fortran_order': is_fortran_array, + 'descr': np.dtype([(' '*dtype_space, int)]) + }) + + assert len(fp.getvalue()) == expected_header_length @pytest.mark.parametrize('dt, fail', [ (np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3', @@ -971,6 +1006,8 @@ def test_unicode_field_names(): float, np.dtype({'names': ['c'], 'formats': [np.dtype(int, metadata={})]}) ]}), False) ]) +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") def test_metadata_dtype(dt, fail): # gh-14142 arr = np.ones(10, dtype=dt) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index eb2fc3311..b0944ec85 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -8,14 +8,14 @@ import pytest import hypothesis from hypothesis.extra.numpy import arrays import hypothesis.strategies as st - +from functools import partial import numpy as np from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY, - assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, + assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, IS_WASM ) import numpy.lib.function_base as nfb from numpy.random import rand @@ -25,6 +25,7 @@ from numpy.lib import ( i0, insert, interp, kaiser, meshgrid, msort, piecewise, place, rot90, select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize ) +from numpy.core.numeric import normalize_axis_tuple def get_mat(n): @@ -228,8 +229,8 @@ class TestAny: def test_nd(self): y1 = [[0, 0, 0], [0, 1, 0], [1, 1, 0]] assert_(np.any(y1)) - assert_array_equal(np.sometrue(y1, axis=0), [1, 1, 0]) - assert_array_equal(np.sometrue(y1, axis=1), [0, 1, 1]) + assert_array_equal(np.any(y1, axis=0), [1, 1, 0]) + assert_array_equal(np.any(y1, axis=1), [0, 1, 1]) class TestAll: @@ -246,8 +247,8 @@ class TestAll: def test_nd(self): y1 = [[0, 0, 1], [0, 1, 1], [1, 1, 1]] assert_(not np.all(y1)) - assert_array_equal(np.alltrue(y1, axis=0), [0, 0, 1]) - assert_array_equal(np.alltrue(y1, axis=1), [0, 0, 1]) + assert_array_equal(np.all(y1, axis=0), [0, 0, 1]) + assert_array_equal(np.all(y1, axis=1), [0, 0, 1]) class TestCopy: @@ -305,6 +306,29 @@ class TestAverage: assert_almost_equal(y5.mean(0), average(y5, 0)) assert_almost_equal(y5.mean(1), average(y5, 1)) + @pytest.mark.parametrize( + 'x, axis, expected_avg, weights, expected_wavg, expected_wsum', + [([1, 2, 3], None, [2.0], [3, 4, 1], [1.75], [8.0]), + ([[1, 2, 5], [1, 6, 11]], 0, [[1.0, 4.0, 8.0]], + [1, 3], [[1.0, 5.0, 9.5]], [[4, 4, 4]])], + ) + def test_basic_keepdims(self, x, axis, expected_avg, + weights, expected_wavg, expected_wsum): + avg = np.average(x, axis=axis, keepdims=True) + assert avg.shape == np.shape(expected_avg) + assert_array_equal(avg, expected_avg) + + wavg = np.average(x, axis=axis, weights=weights, keepdims=True) + assert wavg.shape == np.shape(expected_wavg) + assert_array_equal(wavg, expected_wavg) + + wavg, wsum = np.average(x, axis=axis, weights=weights, returned=True, + keepdims=True) + assert wavg.shape == np.shape(expected_wavg) + assert_array_equal(wavg, expected_wavg) + assert wsum.shape == np.shape(expected_wsum) + assert_array_equal(wsum, expected_wsum) + def test_weights(self): y = np.arange(10) w = np.arange(10) @@ -337,6 +361,18 @@ class TestAverage: assert_(np.average(y3, weights=w3).dtype == np.result_type(y3, w3)) + # test weights with `keepdims=False` and `keepdims=True` + x = np.array([2, 3, 4]).reshape(3, 1) + w = np.array([4, 5, 6]).reshape(3, 1) + + actual = np.average(x, weights=w, axis=1, keepdims=False) + desired = np.array([2., 3., 4.]) + assert_array_equal(actual, desired) + + actual = np.average(x, weights=w, axis=1, keepdims=True) + desired = np.array([[2.], [3.], [4.]]) + assert_array_equal(actual, desired) + def test_returned(self): y = np.array([[1, 2, 3], [4, 5, 6]]) @@ -386,6 +422,11 @@ class TestAverage: w /= w.sum() assert_almost_equal(a.mean(0), average(a, weights=w)) + def test_average_class_without_dtype(self): + # see gh-21988 + a = np.array([Fraction(1, 5), Fraction(3, 5)]) + assert_equal(np.average(a), Fraction(2, 5)) + class TestSelect: choices = [np.array([1, 2, 3]), np.array([4, 5, 6]), @@ -553,6 +594,11 @@ class TestInsert: with pytest.raises(IndexError): np.insert([0, 1, 2], np.array([], dtype=float), []) + @pytest.mark.parametrize('idx', [4, -4]) + def test_index_out_of_bounds(self, idx): + with pytest.raises(IndexError, match='out of bounds'): + np.insert([0, 1, 2], [idx], [3, 4]) + class TestAmax: @@ -805,7 +851,7 @@ class TestDiff: class TestDelete: - def setup(self): + def setup_method(self): self.a = np.arange(5) self.nd_a = np.arange(5).repeat(2).reshape(1, 5, 2) @@ -885,6 +931,40 @@ class TestDelete: with pytest.raises(IndexError): np.delete([0, 1, 2], np.array([], dtype=float)) + @pytest.mark.parametrize("indexer", [np.array([1]), [1]]) + def test_single_item_array(self, indexer): + a_del_int = delete(self.a, 1) + a_del = delete(self.a, indexer) + assert_equal(a_del_int, a_del) + + nd_a_del_int = delete(self.nd_a, 1, axis=1) + nd_a_del = delete(self.nd_a, np.array([1]), axis=1) + assert_equal(nd_a_del_int, nd_a_del) + + def test_single_item_array_non_int(self): + # Special handling for integer arrays must not affect non-integer ones. + # If `False` was cast to `0` it would delete the element: + res = delete(np.ones(1), np.array([False])) + assert_array_equal(res, np.ones(1)) + + # Test the more complicated (with axis) case from gh-21840 + x = np.ones((3, 1)) + false_mask = np.array([False], dtype=bool) + true_mask = np.array([True], dtype=bool) + + res = delete(x, false_mask, axis=-1) + assert_array_equal(res, x) + res = delete(x, true_mask, axis=-1) + assert_array_equal(res, x[:, :0]) + + # Object or e.g. timedeltas should *not* be allowed + with pytest.raises(IndexError): + delete(np.ones(2), np.array([0], dtype=object)) + + with pytest.raises(IndexError): + # timedeltas are sometimes "integral, but clearly not allowed: + delete(np.ones(2), np.array([0], dtype="m8[ns]")) + class TestGradient: @@ -1137,6 +1217,13 @@ class TestGradient: dfdx = gradient(f, x) assert_array_equal(dfdx, [0.5, 0.5]) + def test_return_type(self): + res = np.gradient(([1, 2], [2, 3])) + if np._using_numpy2_behavior(): + assert type(res) is tuple + else: + assert type(res) is list + class TestAngle: @@ -1166,25 +1253,67 @@ class TestAngle: class TestTrimZeros: - """ - Only testing for integer splits. + a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) + b = a.astype(float) + c = a.astype(complex) + d = a.astype(object) - """ + def values(self): + attr_names = ('a', 'b', 'c', 'd') + return (getattr(self, name) for name in attr_names) def test_basic(self): - a = np.array([0, 0, 1, 2, 3, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 2, 3, 4])) + slc = np.s_[2:-1] + for arr in self.values(): + res = trim_zeros(arr) + assert_array_equal(res, arr[slc]) def test_leading_skip(self): - a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 0, 2, 3, 4])) + slc = np.s_[:-1] + for arr in self.values(): + res = trim_zeros(arr, trim='b') + assert_array_equal(res, arr[slc]) def test_trailing_skip(self): - a = np.array([0, 0, 1, 0, 2, 3, 0, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 0, 2, 3, 0, 4])) + slc = np.s_[2:] + for arr in self.values(): + res = trim_zeros(arr, trim='F') + assert_array_equal(res, arr[slc]) + + def test_all_zero(self): + for _arr in self.values(): + arr = np.zeros_like(_arr, dtype=_arr.dtype) + + res1 = trim_zeros(arr, trim='B') + assert len(res1) == 0 + + res2 = trim_zeros(arr, trim='f') + assert len(res2) == 0 + + def test_size_zero(self): + arr = np.zeros(0) + res = trim_zeros(arr) + assert_array_equal(arr, res) + + @pytest.mark.parametrize( + 'arr', + [np.array([0, 2**62, 0]), + np.array([0, 2**63, 0]), + np.array([0, 2**64, 0])] + ) + def test_overflow(self, arr): + slc = np.s_[1:2] + res = trim_zeros(arr) + assert_array_equal(res, arr[slc]) + + def test_no_trim(self): + arr = np.array([None, 1, None]) + res = trim_zeros(arr) + assert_array_equal(arr, res) + + def test_list_to_list(self): + res = trim_zeros(self.a.tolist()) + assert isinstance(res, list) class TestExtins: @@ -1486,6 +1615,21 @@ class TestVectorize: ([('x',)], [('y',), ()])) assert_equal(nfb._parse_gufunc_signature('(),(a,b,c),(d)->(d,e)'), ([(), ('a', 'b', 'c'), ('d',)], [('d', 'e')])) + + # Tests to check if whitespaces are ignored + assert_equal(nfb._parse_gufunc_signature('(x )->()'), ([('x',)], [()])) + assert_equal(nfb._parse_gufunc_signature('( x , y )->( )'), + ([('x', 'y')], [()])) + assert_equal(nfb._parse_gufunc_signature('(x),( y) ->()'), + ([('x',), ('y',)], [()])) + assert_equal(nfb._parse_gufunc_signature('( x)-> (y ) '), + ([('x',)], [('y',)])) + assert_equal(nfb._parse_gufunc_signature(' (x)->( y),( )'), + ([('x',)], [('y',), ()])) + assert_equal(nfb._parse_gufunc_signature( + '( ), ( a, b,c ) ,( d) -> (d , e)'), + ([(), ('a', 'b', 'c'), ('d',)], [('d', 'e')])) + with assert_raises(ValueError): nfb._parse_gufunc_signature('(x)(y)->()') with assert_raises(ValueError): @@ -1623,6 +1767,90 @@ class TestVectorize: with assert_raises_regex(ValueError, 'new output dimensions'): f(x) + def test_subclasses(self): + class subclass(np.ndarray): + pass + + m = np.array([[1., 0., 0.], + [0., 0., 1.], + [0., 1., 0.]]).view(subclass) + v = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]).view(subclass) + # generalized (gufunc) + matvec = np.vectorize(np.matmul, signature='(m,m),(m)->(m)') + r = matvec(m, v) + assert_equal(type(r), subclass) + assert_equal(r, [[1., 3., 2.], [4., 6., 5.], [7., 9., 8.]]) + + # element-wise (ufunc) + mult = np.vectorize(lambda x, y: x*y) + r = mult(m, v) + assert_equal(type(r), subclass) + assert_equal(r, m * v) + + def test_name(self): + #See gh-23021 + @np.vectorize + def f2(a, b): + return a + b + + assert f2.__name__ == 'f2' + + def test_decorator(self): + @vectorize + def addsubtract(a, b): + if a > b: + return a - b + else: + return a + b + + r = addsubtract([0, 3, 6, 9], [1, 3, 5, 7]) + assert_array_equal(r, [1, 6, 1, 2]) + + def test_docstring(self): + @vectorize + def f(x): + """Docstring""" + return x + + if sys.flags.optimize < 2: + assert f.__doc__ == "Docstring" + + def test_partial(self): + def foo(x, y): + return x + y + + bar = partial(foo, 3) + vbar = np.vectorize(bar) + assert vbar(1) == 4 + + def test_signature_otypes_decorator(self): + @vectorize(signature='(n)->(n)', otypes=['float64']) + def f(x): + return x + + r = f([1, 2, 3]) + assert_equal(r.dtype, np.dtype('float64')) + assert_array_equal(r, [1, 2, 3]) + assert f.__name__ == 'f' + + def test_bad_input(self): + with assert_raises(TypeError): + A = np.vectorize(pyfunc = 3) + + def test_no_keywords(self): + with assert_raises(TypeError): + @np.vectorize("string") + def foo(): + return "bar" + + def test_positional_regression_9477(self): + # This supplies the first keyword argument as a positional, + # to ensure that they are still properly forwarded after the + # enhancement for #9477 + f = vectorize((lambda x: x), ['float64']) + r = f([2]) + assert_equal(r.dtype, np.dtype('float64')) + class TestLeaks: class A: @@ -1664,6 +1892,7 @@ class TestLeaks: finally: gc.enable() + class TestDigitize: def test_forward(self): @@ -1757,36 +1986,135 @@ class TestUnwrap: # check that unwrap maintains continuity assert_(np.all(diff(unwrap(rand(10) * 100)) < np.pi)) - + def test_period(self): + # check that unwrap removes jumps greater that 255 + assert_array_equal(unwrap([1, 1 + 256], period=255), [1, 2]) + # check that unwrap maintains continuity + assert_(np.all(diff(unwrap(rand(10) * 1000, period=255)) < 255)) + # check simple case + simple_seq = np.array([0, 75, 150, 225, 300]) + wrap_seq = np.mod(simple_seq, 255) + assert_array_equal(unwrap(wrap_seq, period=255), simple_seq) + # check custom discont value + uneven_seq = np.array([0, 75, 150, 225, 300, 430]) + wrap_uneven = np.mod(uneven_seq, 250) + no_discont = unwrap(wrap_uneven, period=250) + assert_array_equal(no_discont, [0, 75, 150, 225, 300, 180]) + sm_discont = unwrap(wrap_uneven, period=250, discont=140) + assert_array_equal(sm_discont, [0, 75, 150, 225, 300, 430]) + assert sm_discont.dtype == wrap_uneven.dtype + + +@pytest.mark.parametrize( + "dtype", "O" + np.typecodes["AllInteger"] + np.typecodes["Float"] +) +@pytest.mark.parametrize("M", [0, 1, 10]) class TestFilterwindows: - def test_hanning(self): + def test_hanning(self, dtype: str, M: int) -> None: + scalar = np.array(M, dtype=dtype)[()] + + w = hanning(scalar) + if dtype == "O": + ref_dtype = np.float64 + else: + ref_dtype = np.result_type(scalar.dtype, np.float64) + assert w.dtype == ref_dtype + # check symmetry - w = hanning(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) + # check known value - assert_almost_equal(np.sum(w, axis=0), 4.500, 4) + if scalar < 1: + assert_array_equal(w, np.array([])) + elif scalar == 1: + assert_array_equal(w, np.ones(1)) + else: + assert_almost_equal(np.sum(w, axis=0), 4.500, 4) + + def test_hamming(self, dtype: str, M: int) -> None: + scalar = np.array(M, dtype=dtype)[()] + + w = hamming(scalar) + if dtype == "O": + ref_dtype = np.float64 + else: + ref_dtype = np.result_type(scalar.dtype, np.float64) + assert w.dtype == ref_dtype - def test_hamming(self): # check symmetry - w = hamming(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) + # check known value - assert_almost_equal(np.sum(w, axis=0), 4.9400, 4) + if scalar < 1: + assert_array_equal(w, np.array([])) + elif scalar == 1: + assert_array_equal(w, np.ones(1)) + else: + assert_almost_equal(np.sum(w, axis=0), 4.9400, 4) + + def test_bartlett(self, dtype: str, M: int) -> None: + scalar = np.array(M, dtype=dtype)[()] + + w = bartlett(scalar) + if dtype == "O": + ref_dtype = np.float64 + else: + ref_dtype = np.result_type(scalar.dtype, np.float64) + assert w.dtype == ref_dtype - def test_bartlett(self): # check symmetry - w = bartlett(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) + # check known value - assert_almost_equal(np.sum(w, axis=0), 4.4444, 4) + if scalar < 1: + assert_array_equal(w, np.array([])) + elif scalar == 1: + assert_array_equal(w, np.ones(1)) + else: + assert_almost_equal(np.sum(w, axis=0), 4.4444, 4) + + def test_blackman(self, dtype: str, M: int) -> None: + scalar = np.array(M, dtype=dtype)[()] + + w = blackman(scalar) + if dtype == "O": + ref_dtype = np.float64 + else: + ref_dtype = np.result_type(scalar.dtype, np.float64) + assert w.dtype == ref_dtype - def test_blackman(self): # check symmetry - w = blackman(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) + # check known value - assert_almost_equal(np.sum(w, axis=0), 3.7800, 4) + if scalar < 1: + assert_array_equal(w, np.array([])) + elif scalar == 1: + assert_array_equal(w, np.ones(1)) + else: + assert_almost_equal(np.sum(w, axis=0), 3.7800, 4) + + def test_kaiser(self, dtype: str, M: int) -> None: + scalar = np.array(M, dtype=dtype)[()] + + w = kaiser(scalar, 0) + if dtype == "O": + ref_dtype = np.float64 + else: + ref_dtype = np.result_type(scalar.dtype, np.float64) + assert w.dtype == ref_dtype + + # check symmetry + assert_equal(w, flipud(w)) + + # check known value + if scalar < 1: + assert_array_equal(w, np.array([])) + elif scalar == 1: + assert_array_equal(w, np.ones(1)) + else: + assert_almost_equal(np.sum(w, axis=0), 10, 15) class TestTrapz: @@ -1981,6 +2309,12 @@ class TestCorrCoef: assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]])) assert_(np.all(np.abs(c) <= 1.0)) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_corrcoef_dtype(self, test_type): + cast_A = self.A.astype(test_type) + res = corrcoef(cast_A, dtype=test_type) + assert test_type == res.dtype + class TestCov: x1 = np.array([[0, 2], [1, 1], [2, 0]]).T @@ -2081,6 +2415,12 @@ class TestCov: aweights=self.unit_weights), self.res1) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_cov_dtype(self, test_type): + cast_x1 = self.x1.astype(test_type) + res = cov(cast_x1, dtype=test_type) + assert test_type == res.dtype + class Test_I0: @@ -2089,8 +2429,9 @@ class Test_I0: i0(0.5), np.array(1.0634833707413234)) - A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549]) - expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049]) + # need at least one test above 8, as the implementation is piecewise + A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549, 10.0]) + expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049, 2815.71662847]) assert_almost_equal(i0(A), expected) assert_almost_equal(i0(-A), expected) @@ -2127,6 +2468,11 @@ class Test_I0: assert_array_equal(exp, res) + def test_complex(self): + a = np.array([0, 1 + 2j]) + with pytest.raises(TypeError, match="i0 not supported for complex values"): + res = i0(a) + class TestKaiser: @@ -2153,11 +2499,12 @@ class TestMsort: A = np.array([[0.44567325, 0.79115165, 0.54900530], [0.36844147, 0.37325583, 0.96098397], [0.64864341, 0.52929049, 0.39172155]]) - assert_almost_equal( - msort(A), - np.array([[0.36844147, 0.37325583, 0.39172155], - [0.44567325, 0.52929049, 0.54900530], - [0.64864341, 0.79115165, 0.96098397]])) + with pytest.warns(DeprecationWarning, match="msort is deprecated"): + assert_almost_equal( + msort(A), + np.array([[0.36844147, 0.37325583, 0.39172155], + [0.44567325, 0.52929049, 0.54900530], + [0.64864341, 0.79115165, 0.96098397]])) class TestMeshgrid: @@ -2248,6 +2595,27 @@ class TestMeshgrid: assert_equal(x[0, :], 0) assert_equal(x[1, :], X) + def test_nd_shape(self): + a, b, c, d, e = np.meshgrid(*([0] * i for i in range(1, 6))) + expected_shape = (2, 1, 3, 4, 5) + assert_equal(a.shape, expected_shape) + assert_equal(b.shape, expected_shape) + assert_equal(c.shape, expected_shape) + assert_equal(d.shape, expected_shape) + assert_equal(e.shape, expected_shape) + + def test_nd_values(self): + a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5]) + assert_equal(a, [[[0, 0, 0]], [[0, 0, 0]]]) + assert_equal(b, [[[1, 1, 1]], [[2, 2, 2]]]) + assert_equal(c, [[[3, 4, 5]], [[3, 4, 5]]]) + + def test_nd_indexing(self): + a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5], indexing='ij') + assert_equal(a, [[[0, 0, 0], [0, 0, 0]]]) + assert_equal(b, [[[1, 1, 1], [2, 2, 2]]]) + assert_equal(c, [[[3, 4, 5], [3, 4, 5]]]) + class TestPiecewise: @@ -2340,6 +2708,14 @@ class TestPiecewise: assert_array_equal(y, np.array([[-1., -1., -1.], [3., 3., 1.]])) + def test_subclasses(self): + class subclass(np.ndarray): + pass + x = np.arange(5.).view(subclass) + r = piecewise(x, [x<2., x>=4], [-1., 1., 0.]) + assert_equal(type(r), subclass) + assert_equal(r, [-1., -1., 0., 0., 1.]) + class TestBincount: @@ -2631,11 +3007,6 @@ class TestInterp: assert_almost_equal(np.interp(x, xp, fp, period=360), y) -def compare_results(res, desired): - for i in range(len(desired)): - assert_array_equal(res[i], desired[i]) - - class TestPercentile: def test_basic(self): @@ -2645,7 +3016,7 @@ class TestPercentile: assert_equal(np.percentile(x, 50), 1.75) x[1] = np.nan assert_equal(np.percentile(x, 0), np.nan) - assert_equal(np.percentile(x, 0, interpolation='nearest'), np.nan) + assert_equal(np.percentile(x, 0, method='nearest'), np.nan) def test_fraction(self): x = [Fraction(i, 2) for i in range(8)] @@ -2662,6 +3033,10 @@ class TestPercentile: assert_equal(p, Fraction(7, 4)) assert_equal(type(p), Fraction) + p = np.percentile(x, [Fraction(50)]) + assert_equal(p, np.array([Fraction(7, 4)])) + assert_equal(type(p), np.ndarray) + def test_api(self): d = np.ones(5) np.percentile(d, 5, None, None, False) @@ -2669,6 +3044,14 @@ class TestPercentile: o = np.ones((1,)) np.percentile(d, 5, None, o, False, 'linear') + def test_complex(self): + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='G') + assert_raises(TypeError, np.percentile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='D') + assert_raises(TypeError, np.percentile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='F') + assert_raises(TypeError, np.percentile, arr_c, 0.5) + def test_2D(self): x = np.array([[1, 1, 1], [1, 1, 1], @@ -2677,36 +3060,97 @@ class TestPercentile: [1, 1, 1]]) assert_array_equal(np.percentile(x, 50, axis=0), [1, 1, 1]) - def test_linear(self): - - # Test defaults - assert_equal(np.percentile(range(10), 50), 4.5) - - # explicitly specify interpolation_method 'linear' (the default) - assert_equal(np.percentile(range(10), 50, - interpolation='linear'), 4.5) - - def test_lower_higher(self): - - # interpolation_method 'lower'/'higher' - assert_equal(np.percentile(range(10), 50, - interpolation='lower'), 4) - assert_equal(np.percentile(range(10), 50, - interpolation='higher'), 5) - - def test_midpoint(self): - assert_equal(np.percentile(range(10), 51, - interpolation='midpoint'), 4.5) - assert_equal(np.percentile(range(11), 51, - interpolation='midpoint'), 5.5) - assert_equal(np.percentile(range(11), 50, - interpolation='midpoint'), 5) - - def test_nearest(self): - assert_equal(np.percentile(range(10), 51, - interpolation='nearest'), 5) - assert_equal(np.percentile(range(10), 49, - interpolation='nearest'), 4) + @pytest.mark.parametrize("dtype", np.typecodes["Float"]) + def test_linear_nan_1D(self, dtype): + # METHOD 1 of H&F + arr = np.asarray([15.0, np.NAN, 35.0, 40.0, 50.0], dtype=dtype) + res = np.percentile( + arr, + 40.0, + method="linear") + np.testing.assert_equal(res, np.NAN) + np.testing.assert_equal(res.dtype, arr.dtype) + + H_F_TYPE_CODES = [(int_type, np.float64) + for int_type in np.typecodes["AllInteger"] + ] + [(np.float16, np.float16), + (np.float32, np.float32), + (np.float64, np.float64), + (np.longdouble, np.longdouble), + (np.dtype("O"), np.float64)] + + @pytest.mark.parametrize(["input_dtype", "expected_dtype"], H_F_TYPE_CODES) + @pytest.mark.parametrize(["method", "expected"], + [("inverted_cdf", 20), + ("averaged_inverted_cdf", 27.5), + ("closest_observation", 20), + ("interpolated_inverted_cdf", 20), + ("hazen", 27.5), + ("weibull", 26), + ("linear", 29), + ("median_unbiased", 27), + ("normal_unbiased", 27.125), + ]) + def test_linear_interpolation(self, + method, + expected, + input_dtype, + expected_dtype): + expected_dtype = np.dtype(expected_dtype) + if np._get_promotion_state() == "legacy": + expected_dtype = np.promote_types(expected_dtype, np.float64) + + arr = np.asarray([15.0, 20.0, 35.0, 40.0, 50.0], dtype=input_dtype) + actual = np.percentile(arr, 40.0, method=method) + + np.testing.assert_almost_equal( + actual, expected_dtype.type(expected), 14) + + if method in ["inverted_cdf", "closest_observation"]: + if input_dtype == "O": + np.testing.assert_equal(np.asarray(actual).dtype, np.float64) + else: + np.testing.assert_equal(np.asarray(actual).dtype, + np.dtype(input_dtype)) + else: + np.testing.assert_equal(np.asarray(actual).dtype, + np.dtype(expected_dtype)) + + TYPE_CODES = np.typecodes["AllInteger"] + np.typecodes["Float"] + "O" + + @pytest.mark.parametrize("dtype", TYPE_CODES) + def test_lower_higher(self, dtype): + assert_equal(np.percentile(np.arange(10, dtype=dtype), 50, + method='lower'), 4) + assert_equal(np.percentile(np.arange(10, dtype=dtype), 50, + method='higher'), 5) + + @pytest.mark.parametrize("dtype", TYPE_CODES) + def test_midpoint(self, dtype): + assert_equal(np.percentile(np.arange(10, dtype=dtype), 51, + method='midpoint'), 4.5) + assert_equal(np.percentile(np.arange(9, dtype=dtype) + 1, 50, + method='midpoint'), 5) + assert_equal(np.percentile(np.arange(11, dtype=dtype), 51, + method='midpoint'), 5.5) + assert_equal(np.percentile(np.arange(11, dtype=dtype), 50, + method='midpoint'), 5) + + @pytest.mark.parametrize("dtype", TYPE_CODES) + def test_nearest(self, dtype): + assert_equal(np.percentile(np.arange(10, dtype=dtype), 51, + method='nearest'), 5) + assert_equal(np.percentile(np.arange(10, dtype=dtype), 49, + method='nearest'), 4) + + def test_linear_interpolation_extrapolation(self): + arr = np.random.rand(5) + + actual = np.percentile(arr, 100) + np.testing.assert_equal(actual, arr.max()) + + actual = np.percentile(arr, 0) + np.testing.assert_equal(actual, arr.min()) def test_sequence(self): x = np.arange(8) * 0.5 @@ -2734,19 +3178,19 @@ class TestPercentile: assert_equal( np.percentile(x, (25, 50, 75), axis=1).shape, (3, 3, 5, 6)) assert_equal(np.percentile(x, (25, 50), - interpolation="higher").shape, (2,)) + method="higher").shape, (2,)) assert_equal(np.percentile(x, (25, 50, 75), - interpolation="higher").shape, (3,)) + method="higher").shape, (3,)) assert_equal(np.percentile(x, (25, 50), axis=0, - interpolation="higher").shape, (2, 4, 5, 6)) + method="higher").shape, (2, 4, 5, 6)) assert_equal(np.percentile(x, (25, 50), axis=1, - interpolation="higher").shape, (2, 3, 5, 6)) + method="higher").shape, (2, 3, 5, 6)) assert_equal(np.percentile(x, (25, 50), axis=2, - interpolation="higher").shape, (2, 3, 4, 6)) + method="higher").shape, (2, 3, 4, 6)) assert_equal(np.percentile(x, (25, 50), axis=3, - interpolation="higher").shape, (2, 3, 4, 5)) + method="higher").shape, (2, 3, 4, 5)) assert_equal(np.percentile(x, (25, 50, 75), axis=1, - interpolation="higher").shape, (3, 3, 5, 6)) + method="higher").shape, (3, 3, 5, 6)) def test_scalar_q(self): # test for no empty dimensions for compatibility with old percentile @@ -2772,33 +3216,33 @@ class TestPercentile: # test for no empty dimensions for compatibility with old percentile x = np.arange(12).reshape(3, 4) - assert_equal(np.percentile(x, 50, interpolation='lower'), 5.) + assert_equal(np.percentile(x, 50, method='lower'), 5.) assert_(np.isscalar(np.percentile(x, 50))) r0 = np.array([4., 5., 6., 7.]) - c0 = np.percentile(x, 50, interpolation='lower', axis=0) + c0 = np.percentile(x, 50, method='lower', axis=0) assert_equal(c0, r0) assert_equal(c0.shape, r0.shape) r1 = np.array([1., 5., 9.]) - c1 = np.percentile(x, 50, interpolation='lower', axis=1) + c1 = np.percentile(x, 50, method='lower', axis=1) assert_almost_equal(c1, r1) assert_equal(c1.shape, r1.shape) out = np.empty((), dtype=x.dtype) - c = np.percentile(x, 50, interpolation='lower', out=out) + c = np.percentile(x, 50, method='lower', out=out) assert_equal(c, 5) assert_equal(out, 5) out = np.empty(4, dtype=x.dtype) - c = np.percentile(x, 50, interpolation='lower', axis=0, out=out) + c = np.percentile(x, 50, method='lower', axis=0, out=out) assert_equal(c, r0) assert_equal(out, r0) out = np.empty(3, dtype=x.dtype) - c = np.percentile(x, 50, interpolation='lower', axis=1, out=out) + c = np.percentile(x, 50, method='lower', axis=1, out=out) assert_equal(c, r1) assert_equal(out, r1) def test_exception(self): assert_raises(ValueError, np.percentile, [1, 2], 56, - interpolation='foobar') + method='foobar') assert_raises(ValueError, np.percentile, [1], 101) assert_raises(ValueError, np.percentile, [1], -1) assert_raises(ValueError, np.percentile, [1], list(range(50)) + [101]) @@ -2812,18 +3256,18 @@ class TestPercentile: y = np.zeros((3,)) p = (1, 2, 3) np.percentile(x, p, out=y) - assert_equal(y, np.percentile(x, p)) + assert_equal(np.percentile(x, p), y) x = np.array([[1, 2, 3], [4, 5, 6]]) y = np.zeros((3, 3)) np.percentile(x, p, axis=0, out=y) - assert_equal(y, np.percentile(x, p, axis=0)) + assert_equal(np.percentile(x, p, axis=0), y) y = np.zeros((3, 2)) np.percentile(x, p, axis=1, out=y) - assert_equal(y, np.percentile(x, p, axis=1)) + assert_equal(np.percentile(x, p, axis=1), y) x = np.arange(12).reshape(3, 4) # q.dim > 1, float @@ -2839,12 +3283,12 @@ class TestPercentile: # q.dim > 1, int r0 = np.array([[0, 1, 2, 3], [4, 5, 6, 7]]) out = np.empty((2, 4), dtype=x.dtype) - c = np.percentile(x, (25, 50), interpolation='lower', axis=0, out=out) + c = np.percentile(x, (25, 50), method='lower', axis=0, out=out) assert_equal(c, r0) assert_equal(out, r0) r1 = np.array([[0, 4, 8], [1, 5, 9]]) out = np.empty((2, 3), dtype=x.dtype) - c = np.percentile(x, (25, 50), interpolation='lower', axis=1, out=out) + c = np.percentile(x, (25, 50), method='lower', axis=1, out=out) assert_equal(c, r1) assert_equal(out, r1) @@ -2861,10 +3305,10 @@ class TestPercentile: assert_array_equal(np.percentile(d, 50, axis=-4).shape, (1, 2, 1)) assert_array_equal(np.percentile(d, 50, axis=2, - interpolation='midpoint').shape, + method='midpoint').shape, (11, 1, 1)) assert_array_equal(np.percentile(d, 50, axis=-2, - interpolation='midpoint').shape, + method='midpoint').shape, (11, 1, 1)) assert_array_equal(np.array(np.percentile(d, [10, 50], axis=0)).shape, @@ -2887,10 +3331,10 @@ class TestPercentile: def test_no_p_overwrite(self): p = np.linspace(0., 100., num=5) - np.percentile(np.arange(100.), p, interpolation="midpoint") + np.percentile(np.arange(100.), p, method="midpoint") assert_array_equal(p, np.linspace(0., 100., num=5)) p = np.linspace(0., 100., num=5).tolist() - np.percentile(np.arange(100.), p, interpolation="midpoint") + np.percentile(np.arange(100.), p, method="midpoint") assert_array_equal(p, np.linspace(0., 100., num=5).tolist()) def test_percentile_overwrite(self): @@ -2964,18 +3408,44 @@ class TestPercentile: assert_equal(np.percentile(d, [1, 7], axis=(0, 3), keepdims=True).shape, (2, 1, 5, 7, 1)) + @pytest.mark.parametrize('q', [7, [1, 7]]) + @pytest.mark.parametrize( + argnames='axis', + argvalues=[ + None, + 1, + (1,), + (0, 1), + (-3, -1), + ] + ) + def test_keepdims_out(self, q, axis): + d = np.ones((3, 5, 7, 11)) + if axis is None: + shape_out = (1,) * d.ndim + else: + axis_norm = normalize_axis_tuple(axis, d.ndim) + shape_out = tuple( + 1 if i in axis_norm else d.shape[i] for i in range(d.ndim)) + shape_out = np.shape(q) + shape_out + + out = np.empty(shape_out) + result = np.percentile(d, q, axis=axis, keepdims=True, out=out) + assert result is out + assert_equal(result.shape, shape_out) + def test_out(self): o = np.zeros((4,)) d = np.ones((3, 4)) assert_equal(np.percentile(d, 0, 0, out=o), o) - assert_equal(np.percentile(d, 0, 0, interpolation='nearest', out=o), o) + assert_equal(np.percentile(d, 0, 0, method='nearest', out=o), o) o = np.zeros((3,)) assert_equal(np.percentile(d, 1, 1, out=o), o) - assert_equal(np.percentile(d, 1, 1, interpolation='nearest', out=o), o) + assert_equal(np.percentile(d, 1, 1, method='nearest', out=o), o) o = np.zeros(()) assert_equal(np.percentile(d, 2, out=o), o) - assert_equal(np.percentile(d, 2, interpolation='nearest', out=o), o) + assert_equal(np.percentile(d, 2, method='nearest', out=o), o) def test_out_nan(self): with warnings.catch_warnings(record=True): @@ -2985,15 +3455,15 @@ class TestPercentile: d[2, 1] = np.nan assert_equal(np.percentile(d, 0, 0, out=o), o) assert_equal( - np.percentile(d, 0, 0, interpolation='nearest', out=o), o) + np.percentile(d, 0, 0, method='nearest', out=o), o) o = np.zeros((3,)) assert_equal(np.percentile(d, 1, 1, out=o), o) assert_equal( - np.percentile(d, 1, 1, interpolation='nearest', out=o), o) + np.percentile(d, 1, 1, method='nearest', out=o), o) o = np.zeros(()) assert_equal(np.percentile(d, 1, out=o), o) assert_equal( - np.percentile(d, 1, interpolation='nearest', out=o), o) + np.percentile(d, 1, method='nearest', out=o), o) def test_nan_behavior(self): a = np.arange(24, dtype=float) @@ -3048,18 +3518,48 @@ class TestPercentile: b[:, 1] = np.nan b[:, 2] = np.nan assert_equal(np.percentile(a, [0.3, 0.6], (0, 2)), b) - # axis02 not zerod with nearest interpolation + # axis02 not zerod with method='nearest' b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), - [0.3, 0.6], (0, 2), interpolation='nearest') + [0.3, 0.6], (0, 2), method='nearest') b[:, 1] = np.nan b[:, 2] = np.nan assert_equal(np.percentile( - a, [0.3, 0.6], (0, 2), interpolation='nearest'), b) + a, [0.3, 0.6], (0, 2), method='nearest'), b) + + def test_nan_q(self): + # GH18830 + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], np.nan) + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], [np.nan]) + q = np.linspace(1.0, 99.0, 16) + q[0] = np.nan + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], q) + + +quantile_methods = [ + 'inverted_cdf', 'averaged_inverted_cdf', 'closest_observation', + 'interpolated_inverted_cdf', 'hazen', 'weibull', 'linear', + 'median_unbiased', 'normal_unbiased', 'nearest', 'lower', 'higher', + 'midpoint'] class TestQuantile: # most of this is already tested by TestPercentile + def V(self, x, y, alpha): + # Identification function used in several tests. + return (x >= y) - alpha + + def test_max_ulp(self): + x = [0.0, 0.2, 0.4] + a = np.quantile(x, 0.45) + # The default linear method would result in 0 + 0.2 * (0.45/2) = 0.18. + # 0.18 is not exactly representable and the formula leads to a 1 ULP + # different result. Ensure it is this exact within 1 ULP, see gh-20331. + np.testing.assert_array_max_ulp(a, 0.18, maxulp=1) + def test_basic(self): x = np.arange(8) * 0.5 assert_equal(np.quantile(x, 0), 0.) @@ -3074,12 +3574,11 @@ class TestQuantile: a = np.array([False, True, True]) quant_res = np.quantile(a, a) assert_array_equal(quant_res, a) - assert_equal(a.dtype, quant_res.dtype) + assert_equal(quant_res.dtype, a.dtype) def test_fraction(self): # fractional input, integral quantile x = [Fraction(i, 2) for i in range(8)] - q = np.quantile(x, 0) assert_equal(q, 0) assert_equal(type(q), Fraction) @@ -3092,28 +3591,57 @@ class TestQuantile: assert_equal(q, Fraction(7, 4)) assert_equal(type(q), Fraction) + q = np.quantile(x, [Fraction(1, 2)]) + assert_equal(q, np.array([Fraction(7, 4)])) + assert_equal(type(q), np.ndarray) + + q = np.quantile(x, [[Fraction(1, 2)]]) + assert_equal(q, np.array([[Fraction(7, 4)]])) + assert_equal(type(q), np.ndarray) + # repeat with integral input but fractional quantile x = np.arange(8) assert_equal(np.quantile(x, Fraction(1, 2)), Fraction(7, 2)) + def test_complex(self): + #See gh-22652 + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='G') + assert_raises(TypeError, np.quantile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='D') + assert_raises(TypeError, np.quantile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='F') + assert_raises(TypeError, np.quantile, arr_c, 0.5) + def test_no_p_overwrite(self): # this is worth retesting, because quantile does not make a copy p0 = np.array([0, 0.75, 0.25, 0.5, 1.0]) p = p0.copy() - np.quantile(np.arange(100.), p, interpolation="midpoint") + np.quantile(np.arange(100.), p, method="midpoint") assert_array_equal(p, p0) p0 = p0.tolist() p = p.tolist() - np.quantile(np.arange(100.), p, interpolation="midpoint") + np.quantile(np.arange(100.), p, method="midpoint") assert_array_equal(p, p0) - def test_quantile_monotonic(self): + @pytest.mark.parametrize("dtype", np.typecodes["AllInteger"]) + def test_quantile_preserve_int_type(self, dtype): + res = np.quantile(np.array([1, 2], dtype=dtype), [0.5], + method="nearest") + assert res.dtype == dtype + + @pytest.mark.parametrize("method", quantile_methods) + def test_quantile_monotonic(self, method): # GH 14685 # test that the return value of quantile is monotonic if p0 is ordered - p0 = np.arange(0, 1, 0.01) + # Also tests that the boundary values are not mishandled. + p0 = np.linspace(0, 1, 101) quantile = np.quantile(np.array([0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 1, 1, 9, 9, 9, - 8, 8, 7]) * 0.1, p0) + 8, 8, 7]) * 0.1, p0, method=method) + assert_equal(np.sort(quantile), quantile) + + # Also test one where the number of data points is clearly divisible: + quantile = np.quantile([0., 1., 2., 3.], p0, method=method) assert_equal(np.sort(quantile), quantile) @hypothesis.given( @@ -3126,6 +3654,101 @@ class TestQuantile: quantile = np.quantile(arr, p0) assert_equal(np.sort(quantile), quantile) + def test_quantile_scalar_nan(self): + a = np.array([[10., 7., 4.], [3., 2., 1.]]) + a[0][1] = np.nan + actual = np.quantile(a, 0.5) + assert np.isscalar(actual) + assert_equal(np.quantile(a, 0.5), np.nan) + + @pytest.mark.parametrize("method", quantile_methods) + @pytest.mark.parametrize("alpha", [0.2, 0.5, 0.9]) + def test_quantile_identification_equation(self, method, alpha): + # Test that the identification equation holds for the empirical + # CDF: + # E[V(x, Y)] = 0 <=> x is quantile + # with Y the random variable for which we have observed values and + # V(x, y) the canonical identification function for the quantile (at + # level alpha), see + # https://doi.org/10.48550/arXiv.0912.0902 + rng = np.random.default_rng(4321) + # We choose n and alpha such that we cover 3 cases: + # - n * alpha is an integer + # - n * alpha is a float that gets rounded down + # - n * alpha is a float that gest rounded up + n = 102 # n * alpha = 20.4, 51. , 91.8 + y = rng.random(n) + x = np.quantile(y, alpha, method=method) + if method in ("higher",): + # These methods do not fulfill the identification equation. + assert np.abs(np.mean(self.V(x, y, alpha))) > 0.1 / n + elif int(n * alpha) == n * alpha: + # We can expect exact results, up to machine precision. + assert_allclose(np.mean(self.V(x, y, alpha)), 0, atol=1e-14) + else: + # V = (x >= y) - alpha cannot sum to zero exactly but within + # "sample precision". + assert_allclose(np.mean(self.V(x, y, alpha)), 0, + atol=1 / n / np.amin([alpha, 1 - alpha])) + + @pytest.mark.parametrize("method", quantile_methods) + @pytest.mark.parametrize("alpha", [0.2, 0.5, 0.9]) + def test_quantile_add_and_multiply_constant(self, method, alpha): + # Test that + # 1. quantile(c + x) = c + quantile(x) + # 2. quantile(c * x) = c * quantile(x) + # 3. quantile(-x) = -quantile(x, 1 - alpha) + # On empirical quantiles, this equation does not hold exactly. + # Koenker (2005) "Quantile Regression" Chapter 2.2.3 calls these + # properties equivariance. + rng = np.random.default_rng(4321) + # We choose n and alpha such that we have cases for + # - n * alpha is an integer + # - n * alpha is a float that gets rounded down + # - n * alpha is a float that gest rounded up + n = 102 # n * alpha = 20.4, 51. , 91.8 + y = rng.random(n) + q = np.quantile(y, alpha, method=method) + c = 13.5 + + # 1 + assert_allclose(np.quantile(c + y, alpha, method=method), c + q) + # 2 + assert_allclose(np.quantile(c * y, alpha, method=method), c * q) + # 3 + q = -np.quantile(-y, 1 - alpha, method=method) + if method == "inverted_cdf": + if ( + n * alpha == int(n * alpha) + or np.round(n * alpha) == int(n * alpha) + 1 + ): + assert_allclose(q, np.quantile(y, alpha, method="higher")) + else: + assert_allclose(q, np.quantile(y, alpha, method="lower")) + elif method == "closest_observation": + if n * alpha == int(n * alpha): + assert_allclose(q, np.quantile(y, alpha, method="higher")) + elif np.round(n * alpha) == int(n * alpha) + 1: + assert_allclose( + q, np.quantile(y, alpha + 1/n, method="higher")) + else: + assert_allclose(q, np.quantile(y, alpha, method="lower")) + elif method == "interpolated_inverted_cdf": + assert_allclose(q, np.quantile(y, alpha + 1/n, method=method)) + elif method == "nearest": + if n * alpha == int(n * alpha): + assert_allclose(q, np.quantile(y, alpha + 1/n, method=method)) + else: + assert_allclose(q, np.quantile(y, alpha, method=method)) + elif method == "lower": + assert_allclose(q, np.quantile(y, alpha, method="higher")) + elif method == "higher": + assert_allclose(q, np.quantile(y, alpha, method="lower")) + else: + # "averaged_inverted_cdf", "hazen", "weibull", "linear", + # "median_unbiased", "normal_unbiased", "midpoint" + assert_allclose(q, np.quantile(y, alpha, method=method)) + class TestLerp: @hypothesis.given(t0=st.floats(allow_nan=False, allow_infinity=False, @@ -3136,9 +3759,9 @@ class TestLerp: min_value=-1e300, max_value=1e300), b = st.floats(allow_nan=False, allow_infinity=False, min_value=-1e300, max_value=1e300)) - def test_lerp_monotonic(self, t0, t1, a, b): - l0 = np.lib.function_base._lerp(a, b, t0) - l1 = np.lib.function_base._lerp(a, b, t1) + def test_linear_interpolation_formula_monotonic(self, t0, t1, a, b): + l0 = nfb._lerp(a, b, t0) + l1 = nfb._lerp(a, b, t1) if t0 == t1 or a == b: assert l0 == l1 # uninteresting elif (t0 < t1) == (a < b): @@ -3152,11 +3775,11 @@ class TestLerp: min_value=-1e300, max_value=1e300), b=st.floats(allow_nan=False, allow_infinity=False, min_value=-1e300, max_value=1e300)) - def test_lerp_bounded(self, t, a, b): + def test_linear_interpolation_formula_bounded(self, t, a, b): if a <= b: - assert a <= np.lib.function_base._lerp(a, b, t) <= b + assert a <= nfb._lerp(a, b, t) <= b else: - assert b <= np.lib.function_base._lerp(a, b, t) <= a + assert b <= nfb._lerp(a, b, t) <= a @hypothesis.given(t=st.floats(allow_nan=False, allow_infinity=False, min_value=0, max_value=1), @@ -3164,17 +3787,17 @@ class TestLerp: min_value=-1e300, max_value=1e300), b=st.floats(allow_nan=False, allow_infinity=False, min_value=-1e300, max_value=1e300)) - def test_lerp_symmetric(self, t, a, b): + def test_linear_interpolation_formula_symmetric(self, t, a, b): # double subtraction is needed to remove the extra precision of t < 0.5 - left = np.lib.function_base._lerp(a, b, 1 - (1 - t)) - right = np.lib.function_base._lerp(b, a, 1 - t) - assert left == right + left = nfb._lerp(a, b, 1 - (1 - t)) + right = nfb._lerp(b, a, 1 - t) + assert_allclose(left, right) - def test_lerp_0d_inputs(self): + def test_linear_interpolation_formula_0d_inputs(self): a = np.array(2) b = np.array(5) t = np.array(0.2) - assert np.lib.function_base._lerp(a, b, t) == 2.6 + assert nfb._lerp(a, b, t) == 2.6 class TestMedian: @@ -3274,6 +3897,16 @@ class TestMedian: a = MySubClass([1, 2, 3]) assert_equal(np.median(a), -7) + @pytest.mark.parametrize('arr', + ([1., 2., 3.], [1., np.nan, 3.], np.nan, 0.)) + def test_subclass2(self, arr): + """Check that we return subclasses, even if a NaN scalar.""" + class MySubclass(np.ndarray): + pass + + m = np.median(np.array(arr).view(MySubclass)) + assert isinstance(m, MySubclass) + def test_out(self): o = np.zeros((4,)) d = np.ones((3, 4)) @@ -3327,6 +3960,7 @@ class TestMedian: b[2] = np.nan assert_equal(np.median(a, (0, 2)), b) + @pytest.mark.skipif(IS_WASM, reason="fp errors don't work correctly") def test_empty(self): # mean(empty array) emits two warnings: empty slice and divide by 0 a = np.array([], dtype=float) @@ -3415,6 +4049,29 @@ class TestMedian: assert_equal(np.median(d, axis=(0, 1, 3), keepdims=True).shape, (1, 1, 7, 1)) + @pytest.mark.parametrize( + argnames='axis', + argvalues=[ + None, + 1, + (1, ), + (0, 1), + (-3, -1), + ] + ) + def test_keepdims_out(self, axis): + d = np.ones((3, 5, 7, 11)) + if axis is None: + shape_out = (1,) * d.ndim + else: + axis_norm = normalize_axis_tuple(axis, d.ndim) + shape_out = tuple( + 1 if i in axis_norm else d.shape[i] for i in range(d.ndim)) + out = np.empty(shape_out) + result = np.median(d, axis=axis, keepdims=True, out=out) + assert result is out + assert_equal(result.shape, shape_out) + class TestAdd_newdoc_ufunc: diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index fc16b7396..87e6e1d41 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -6,15 +6,16 @@ from numpy.testing import ( assert_array_almost_equal, assert_raises, assert_allclose, assert_array_max_ulp, assert_raises_regex, suppress_warnings, ) +from numpy.testing._private.utils import requires_memory import pytest class TestHistogram: - def setup(self): + def setup_method(self): pass - def teardown(self): + def teardown_method(self): pass def test_simple(self): @@ -38,30 +39,6 @@ class TestHistogram: assert_equal(h, np.array([2])) assert_allclose(e, np.array([1., 2.])) - def test_normed(self): - sup = suppress_warnings() - with sup: - rec = sup.record(np.VisibleDeprecationWarning, '.*normed.*') - # Check that the integral of the density equals 1. - n = 100 - v = np.random.rand(n) - a, b = histogram(v, normed=True) - area = np.sum(a * np.diff(b)) - assert_almost_equal(area, 1) - assert_equal(len(rec), 1) - - sup = suppress_warnings() - with sup: - rec = sup.record(np.VisibleDeprecationWarning, '.*normed.*') - # Check with non-constant bin widths (buggy but backwards - # compatible) - v = np.arange(10) - bins = [0, 1, 5, 9, 10] - a, b = histogram(v, bins, normed=True) - area = np.sum(a * np.diff(b)) - assert_almost_equal(area, 1) - assert_equal(len(rec), 1) - def test_density(self): # Check that the integral of the density equals 1. n = 100 @@ -421,6 +398,16 @@ class TestHistogram: edges = histogram_bin_edges(arr, bins='auto', range=(0, 1)) assert_array_equal(edges, e) + @requires_memory(free_bytes=1e10) + @pytest.mark.slow + def test_big_arrays(self): + sample = np.zeros([100000000, 3]) + xbins = 400 + ybins = 400 + zbins = np.arange(16000) + hist = np.histogramdd(sample=sample, bins=(xbins, ybins, zbins)) + assert_equal(type(hist), type((1, 2))) + class TestHistogramOptimBinNums: """ @@ -819,20 +806,3 @@ class TestHistogramdd: hist_dd, edges_dd = histogramdd((v,), (bins,), density=True) assert_equal(hist, hist_dd) assert_equal(edges, edges_dd[0]) - - def test_density_via_normed(self): - # normed should simply alias to density argument - v = np.arange(10) - bins = np.array([0, 1, 3, 6, 10]) - hist, edges = histogram(v, bins, density=True) - hist_dd, edges_dd = histogramdd((v,), (bins,), normed=True) - assert_equal(hist, hist_dd) - assert_equal(edges, edges_dd[0]) - - def test_density_normed_redundancy(self): - v = np.arange(10) - bins = np.array([0, 1, 3, 6, 10]) - with assert_raises_regex(TypeError, "Cannot specify both"): - hist_dd, edges_dd = histogramdd((v,), (bins,), - density=True, - normed=True) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 843e27cef..b599cb345 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -4,7 +4,6 @@ import numpy as np from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_raises_regex, - assert_warns ) from numpy.lib.index_tricks import ( mgrid, ogrid, ndenumerate, fill_diagonal, diag_indices, diag_indices_from, @@ -16,23 +15,13 @@ class TestRavelUnravelIndex: def test_basic(self): assert_equal(np.unravel_index(2, (2, 2)), (1, 0)) - # test backwards compatibility with older dims - # keyword argument; see Issue #10586 - with assert_warns(DeprecationWarning): - # we should achieve the correct result - # AND raise the appropriate warning - # when using older "dims" kw argument - assert_equal(np.unravel_index(indices=2, - dims=(2, 2)), - (1, 0)) - # test that new shape argument works properly assert_equal(np.unravel_index(indices=2, shape=(2, 2)), (1, 0)) # test that an invalid second keyword argument - # is properly handled + # is properly handled, including the old name `dims`. with assert_raises(TypeError): np.unravel_index(indices=2, hape=(2, 2)) @@ -42,6 +31,9 @@ class TestRavelUnravelIndex: with assert_raises(TypeError): np.unravel_index(254, ims=(17, 94)) + with assert_raises(TypeError): + np.unravel_index(254, dims=(17, 94)) + assert_equal(np.ravel_multi_index((1, 0), (2, 2)), 2) assert_equal(np.unravel_index(254, (17, 94)), (2, 66)) assert_equal(np.ravel_multi_index((2, 66), (17, 94)), 254) @@ -262,6 +254,28 @@ class TestGrid: assert_(grid32.dtype == np.float64) assert_array_almost_equal(grid64, grid32) + def test_accepts_longdouble(self): + # regression tests for #16945 + grid64 = mgrid[0.1:0.33:0.1, ] + grid128 = mgrid[ + np.longdouble(0.1):np.longdouble(0.33):np.longdouble(0.1), + ] + assert_(grid128.dtype == np.longdouble) + assert_array_almost_equal(grid64, grid128) + + grid128c_a = mgrid[0:np.longdouble(1):3.4j] + grid128c_b = mgrid[0:np.longdouble(1):3.4j, ] + assert_(grid128c_a.dtype == grid128c_b.dtype == np.longdouble) + assert_array_equal(grid128c_a, grid128c_b[0]) + + # different code path for single slice + grid64 = mgrid[0.1:0.33:0.1] + grid128 = mgrid[ + np.longdouble(0.1):np.longdouble(0.33):np.longdouble(0.1) + ] + assert_(grid128.dtype == np.longdouble) + assert_array_almost_equal(grid64, grid128) + def test_accepts_npcomplexfloating(self): # Related to #16466 assert_array_almost_equal( @@ -273,6 +287,18 @@ class TestGrid: mgrid[0.1:0.3:3j], mgrid[0.1:0.3:np.complex64(3j)] ) + # Related to #16945 + grid64_a = mgrid[0.1:0.3:3.3j] + grid64_b = mgrid[0.1:0.3:3.3j, ][0] + assert_(grid64_a.dtype == grid64_b.dtype == np.float64) + assert_array_equal(grid64_a, grid64_b) + + grid128_a = mgrid[0.1:0.3:np.clongdouble(3.3j)] + grid128_b = mgrid[0.1:0.3:np.clongdouble(3.3j), ][0] + assert_(grid128_a.dtype == grid128_b.dtype == np.longdouble) + assert_array_equal(grid64_a, grid64_b) + + class TestConcatenator: def test_1d(self): assert_array_equal(r_[1, 2, 3, 4, 5, 6], np.array([1, 2, 3, 4, 5, 6])) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index a23c6b007..5a68fbc97 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -13,19 +13,19 @@ from tempfile import NamedTemporaryFile from io import BytesIO, StringIO from datetime import datetime import locale -from multiprocessing import Process, Value +from multiprocessing import Value, get_context from ctypes import c_bool import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes +from numpy.compat import asbytes from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings, - break_cycles + break_cycles, IS_WASM ) from numpy.testing._private.utils import requires_memory @@ -203,6 +203,7 @@ class TestSavezLoad(RoundtripTest): self.arr_reloaded.fid.close() os.remove(self.arr_reloaded.fid.name) + @pytest.mark.skipif(IS_PYPY, reason="Hangs on PyPy") @pytest.mark.skipif(not IS_64BIT, reason="Needs 64bit platform") @pytest.mark.slow def test_big_arrays(self): @@ -242,6 +243,7 @@ class TestSavezLoad(RoundtripTest): assert_equal(a, l.f.file_a) assert_equal(b, l.f.file_b) + @pytest.mark.skipif(IS_WASM, reason="Cannot start thread") def test_savez_filename_clashes(self): # Test that issue #852 is fixed # and savez functions in multithreaded environment @@ -319,6 +321,21 @@ class TestSavezLoad(RoundtripTest): data.close() assert_(fp.closed) + @pytest.mark.parametrize("count, expected_repr", [ + (1, "NpzFile {fname!r} with keys: arr_0"), + (5, "NpzFile {fname!r} with keys: arr_0, arr_1, arr_2, arr_3, arr_4"), + # _MAX_REPR_ARRAY_COUNT is 5, so files with more than 5 keys are + # expected to end in '...' + (6, "NpzFile {fname!r} with keys: arr_0, arr_1, arr_2, arr_3, arr_4..."), + ]) + def test_repr_lists_keys(self, count, expected_repr): + a = np.array([[1, 2], [3, 4]], float) + with temppath(suffix='.npz') as tmp: + np.savez(tmp, *[a]*count) + l = np.load(tmp) + assert repr(l) == expected_repr.format(fname=tmp) + l.close() + class TestSaveTxt: def test_array(self): @@ -520,7 +537,7 @@ class TestSaveTxt: def test_unicode(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode_) + a = np.array([utf8], dtype=np.str_) with tempdir() as tmpdir: # set encoding as on windows it may not be unicode even on py3 np.savetxt(os.path.join(tmpdir, 'test.csv'), a, fmt=['%s'], @@ -528,7 +545,7 @@ class TestSaveTxt: def test_unicode_roundtrip(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode_) + a = np.array([utf8], dtype=np.str_) # our gz wrapper support encoding suffixes = ['', '.gz'] if HAS_BZ2: @@ -540,12 +557,12 @@ class TestSaveTxt: np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a, fmt=['%s'], encoding='UTF-16-LE') b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix), - encoding='UTF-16-LE', dtype=np.unicode_) + encoding='UTF-16-LE', dtype=np.str_) assert_array_equal(a, b) def test_unicode_bytestream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode_) + a = np.array([utf8], dtype=np.str_) s = BytesIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -553,13 +570,13 @@ class TestSaveTxt: def test_unicode_stringstream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode_) + a = np.array([utf8], dtype=np.str_) s = StringIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) assert_equal(s.read(), utf8 + '\n') - @pytest.mark.parametrize("fmt", [u"%f", b"%f"]) + @pytest.mark.parametrize("fmt", ["%f", b"%f"]) @pytest.mark.parametrize("iotype", [StringIO, BytesIO]) def test_unicode_and_bytes_fmt(self, fmt, iotype): # string type of fmt should not matter, see also gh-4053 @@ -568,7 +585,7 @@ class TestSaveTxt: np.savetxt(s, a, fmt=fmt) s.seek(0) if iotype is StringIO: - assert_equal(s.read(), u"%f\n" % 1.) + assert_equal(s.read(), "%f\n" % 1.) else: assert_equal(s.read(), b"%f\n" % 1.) @@ -580,7 +597,7 @@ class TestSaveTxt: memoryerror_raised.value = False try: # The test takes at least 6GB of memory, writes a file larger - # than 4GB + # than 4GB. This tests the ``allowZip64`` kwarg to ``zipfile`` test_data = np.asarray([np.random.rand( np.random.randint(50,100),4) for i in range(800000)], dtype=object) @@ -594,11 +611,19 @@ class TestSaveTxt: # Use an object in shared memory to re-raise the MemoryError exception # in our process if needed, see gh-16889 memoryerror_raised = Value(c_bool) - p = Process(target=check_large_zip, args=(memoryerror_raised,)) + + # Since Python 3.8, the default start method for multiprocessing has + # been changed from 'fork' to 'spawn' on macOS, causing inconsistency + # on memory sharing model, lead to failed test for check_large_zip + ctx = get_context('fork') + p = ctx.Process(target=check_large_zip, args=(memoryerror_raised,)) p.start() p.join() if memoryerror_raised.value: raise MemoryError("Child process raised a MemoryError exception") + # -9 indicates a SIGKILL, probably an OOM. + if p.exitcode == -9: + pytest.xfail("subprocess got a SIGKILL, apparently free memory was not sufficient") assert p.exitcode == 0 class LoadTxtBase: @@ -642,12 +667,12 @@ class LoadTxtBase: with temppath() as path: with open(path, "wb") as f: f.write(nonascii.encode("UTF-16")) - x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode_) + x = self.loadfunc(path, encoding="UTF-16", dtype=np.str_) assert_array_equal(x, nonascii) def test_binary_decode(self): utf16 = b'\xff\xfeh\x04 \x00i\x04 \x00j\x04' - v = self.loadfunc(BytesIO(utf16), dtype=np.unicode_, encoding='UTF-16') + v = self.loadfunc(BytesIO(utf16), dtype=np.str_, encoding='UTF-16') assert_array_equal(v, np.array(utf16.decode('UTF-16').split())) def test_converters_decode(self): @@ -655,7 +680,7 @@ class LoadTxtBase: c = TextIO() c.write(b'\xcf\x96') c.seek(0) - x = self.loadfunc(c, dtype=np.unicode_, + x = self.loadfunc(c, dtype=np.str_, converters={0: lambda x: x.decode('UTF-8')}) a = np.array([b'\xcf\x96'.decode('UTF-8')]) assert_array_equal(x, a) @@ -666,7 +691,7 @@ class LoadTxtBase: with temppath() as path: with io.open(path, 'wt', encoding='UTF-8') as f: f.write(utf8) - x = self.loadfunc(path, dtype=np.unicode_, + x = self.loadfunc(path, dtype=np.str_, converters={0: lambda x: x + 't'}, encoding='UTF-8') a = np.array([utf8 + 't']) @@ -676,11 +701,12 @@ class LoadTxtBase: class TestLoadTxt(LoadTxtBase): loadfunc = staticmethod(np.loadtxt) - def setup(self): + def setup_method(self): # lower chunksize for testing self.orig_chunk = np.lib.npyio._loadtxt_chunksize np.lib.npyio._loadtxt_chunksize = 1 - def teardown(self): + + def teardown_method(self): np.lib.npyio._loadtxt_chunksize = self.orig_chunk def test_record(self): @@ -692,7 +718,7 @@ class TestLoadTxt(LoadTxtBase): assert_array_equal(x, a) d = TextIO() - d.write('M 64.0 75.0\nF 25.0 60.0') + d.write('M 64 75.0\nF 25 60.0') d.seek(0) mydescriptor = {'names': ('gender', 'age', 'weight'), 'formats': ('S1', 'i4', 'f4')} @@ -754,7 +780,7 @@ class TestLoadTxt(LoadTxtBase): c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', - comments=u'#') + comments='#') a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a) @@ -776,6 +802,8 @@ class TestLoadTxt(LoadTxtBase): a = np.array([[1, 2, 3], [4, 5, 6]], int) assert_array_equal(x, a) + @pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") def test_comments_multi_chars(self): c = TextIO() c.write('/* comment\n1,2,3,5\n') @@ -868,16 +896,27 @@ class TestLoadTxt(LoadTxtBase): bogus_idx = 1.5 assert_raises_regex( TypeError, - '^usecols must be.*%s' % type(bogus_idx), + '^usecols must be.*%s' % type(bogus_idx).__name__, np.loadtxt, c, usecols=bogus_idx ) assert_raises_regex( TypeError, - '^usecols must be.*%s' % type(bogus_idx), + '^usecols must be.*%s' % type(bogus_idx).__name__, np.loadtxt, c, usecols=[0, bogus_idx, 0] ) + def test_bad_usecols(self): + with pytest.raises(OverflowError): + np.loadtxt(["1\n"], usecols=[2**64], delimiter=",") + with pytest.raises((ValueError, OverflowError)): + # Overflow error on 32bit platforms + np.loadtxt(["1\n"], usecols=[2**62], delimiter=",") + with pytest.raises(TypeError, + match="If a structured dtype .*. But 1 usecols were given and " + "the number of fields is 3."): + np.loadtxt(["1,1\n"], dtype="i,(2)i", usecols=[0], delimiter=",") + def test_fancy_dtype(self): c = TextIO() c.write('1,2,3.0\n4,5,6.0\n') @@ -916,8 +955,7 @@ class TestLoadTxt(LoadTxtBase): assert_array_equal(x, a) def test_empty_file(self): - with suppress_warnings() as sup: - sup.filter(message="loadtxt: Empty input file:") + with pytest.warns(UserWarning, match="input contained no data"): c = TextIO() x = np.loadtxt(c) assert_equal(x.shape, (0,)) @@ -978,9 +1016,34 @@ class TestLoadTxt(LoadTxtBase): c.write(inp) for dt in [float, np.float32]: c.seek(0) - res = np.loadtxt(c, dtype=dt) + res = np.loadtxt( + c, dtype=dt, converters=float.fromhex, encoding="latin1") assert_equal(res, tgt, err_msg="%s" % dt) + @pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") + def test_default_float_converter_no_default_hex_conversion(self): + """ + Ensure that fromhex is only used for values with the correct prefix and + is not called by default. Regression test related to gh-19598. + """ + c = TextIO("a b c") + with pytest.raises(ValueError, + match=".*convert string 'a' to float64 at row 0, column 1"): + np.loadtxt(c) + + @pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") + def test_default_float_converter_exception(self): + """ + Ensure that the exception message raised during failed floating point + conversion is correct. Regression test related to gh-19598. + """ + c = TextIO("qrs tuv") # Invalid values for default float converter + with pytest.raises(ValueError, + match="could not convert string 'qrs' to float64"): + np.loadtxt(c) + def test_from_complex(self): tgt = (complex(1, 1), complex(1, -1)) c = TextIO() @@ -1026,7 +1089,7 @@ class TestLoadTxt(LoadTxtBase): a = np.array([b'start ', b' ', b'']) assert_array_equal(x['comment'], a) - def test_structure_unpack(self): + def test_unpack_structured(self): txt = TextIO("M 21 72\nF 35 58") dt = {'names': ('a', 'b', 'c'), 'formats': ('|S1', '<i4', '<f4')} a, b, c = np.loadtxt(txt, dtype=dt, unpack=True) @@ -1074,8 +1137,7 @@ class TestLoadTxt(LoadTxtBase): assert_(x.shape == (3,)) # Test ndmin kw with empty file. - with suppress_warnings() as sup: - sup.filter(message="loadtxt: Empty input file:") + with pytest.warns(UserWarning, match="input contained no data"): f = TextIO() assert_(np.loadtxt(f, ndmin=2).shape == (0, 1,)) assert_(np.loadtxt(f, ndmin=1).shape == (0,)) @@ -1107,14 +1169,14 @@ class TestLoadTxt(LoadTxtBase): @pytest.mark.skipif(locale.getpreferredencoding() == 'ANSI_X3.4-1968', reason="Wrong preferred encoding") def test_binary_load(self): - butf8 = b"5,6,7,\xc3\x95scarscar\n\r15,2,3,hello\n\r"\ - b"20,2,3,\xc3\x95scar\n\r" + butf8 = b"5,6,7,\xc3\x95scarscar\r\n15,2,3,hello\r\n"\ + b"20,2,3,\xc3\x95scar\r\n" sutf8 = butf8.decode("UTF-8").replace("\r", "").splitlines() with temppath() as path: with open(path, "wb") as f: f.write(butf8) with open(path, "rb") as f: - x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode_) + x = np.loadtxt(f, encoding="UTF-8", dtype=np.str_) assert_array_equal(x, sutf8) # test broken latin1 conversion people now rely on with open(path, "rb") as f: @@ -1171,6 +1233,31 @@ class TestLoadTxt(LoadTxtBase): a = np.array([[1, 2, 3, 5], [4, 5, 7, 8], [2, 1, 4, 5]], int) assert_array_equal(x, a) + @pytest.mark.parametrize(["skip", "data"], [ + (1, ["ignored\n", "1,2\n", "\n", "3,4\n"]), + # "Bad" lines that do not end in newlines: + (1, ["ignored", "1,2", "", "3,4"]), + (1, StringIO("ignored\n1,2\n\n3,4")), + # Same as above, but do not skip any lines: + (0, ["-1,0\n", "1,2\n", "\n", "3,4\n"]), + (0, ["-1,0", "1,2", "", "3,4"]), + (0, StringIO("-1,0\n1,2\n\n3,4"))]) + def test_max_rows_empty_lines(self, skip, data): + with pytest.warns(UserWarning, + match=f"Input line 3.*max_rows={3-skip}"): + res = np.loadtxt(data, dtype=int, skiprows=skip, delimiter=",", + max_rows=3-skip) + assert_array_equal(res, [[-1, 0], [1, 2], [3, 4]][skip:]) + + if isinstance(data, StringIO): + data.seek(0) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + with pytest.raises(UserWarning): + np.loadtxt(data, dtype=int, skiprows=skip, delimiter=",", + max_rows=3-skip) + class Testfromregex: def test_record(self): c = TextIO() @@ -1204,9 +1291,11 @@ class Testfromregex: a = np.array([(1312,), (1534,), (4444,)], dtype=dt) assert_array_equal(x, a) - def test_record_unicode(self): + @pytest.mark.parametrize("path_type", [str, Path]) + def test_record_unicode(self, path_type): utf8 = b'\xcf\x96' - with temppath() as path: + with temppath() as str_path: + path = path_type(str_path) with open(path, 'wb') as f: f.write(b'1.312 foo' + utf8 + b' \n1.534 bar\n4.444 qux') @@ -1228,6 +1317,13 @@ class Testfromregex: x = np.fromregex(c, regexp, dt) assert_array_equal(x, a) + def test_bad_dtype_not_structured(self): + regexp = re.compile(b'(\\d)') + c = BytesIO(b'123') + with pytest.raises(TypeError, match='structured datatype'): + np.fromregex(c, regexp, dtype=np.float64) + + #####-------------------------------------------------------------------------- @@ -1390,6 +1486,10 @@ class TestFromTxt(LoadTxtBase): ('F', 25.0, 60.0)], dtype=descriptor) assert_equal(test, control) + def test_bad_fname(self): + with pytest.raises(TypeError, match='fname must be a string,'): + np.genfromtxt(123) + def test_commented_header(self): # Check that names can be retrieved even if the line is commented out. data = TextIO(""" @@ -1431,7 +1531,7 @@ M 33 21.99 with tempdir() as tmpdir: fpath = os.path.join(tmpdir, "test.csv") with open(fpath, "wb") as f: - f.write(u'\N{GREEK PI SYMBOL}'.encode('utf8')) + f.write('\N{GREEK PI SYMBOL}'.encode()) # ResourceWarnings are emitted from a destructor, so won't be # detected by regular propagation to errors. @@ -2091,9 +2191,9 @@ M 33 21.99 test = np.genfromtxt(TextIO(s), dtype=None, comments=None, delimiter=',', encoding='latin1') - assert_equal(test[1, 0], u"test1") - assert_equal(test[1, 1], u"testNonethe" + latin1.decode('latin1')) - assert_equal(test[1, 2], u"test3") + assert_equal(test[1, 0], "test1") + assert_equal(test[1, 1], "testNonethe" + latin1.decode('latin1')) + assert_equal(test[1, 2], "test3") with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) @@ -2134,7 +2234,7 @@ M 33 21.99 ctl = np.array([ ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"], ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]], - dtype=np.unicode_) + dtype=np.str_) assert_array_equal(test, ctl) # test a mixed dtype @@ -2147,8 +2247,8 @@ M 33 21.99 def test_utf8_file_nodtype_unicode(self): # bytes encoding with non-latin1 -> unicode upcast - utf8 = u'\u03d6' - latin1 = u'\xf6\xfc\xf6' + utf8 = '\u03d6' + latin1 = '\xf6\xfc\xf6' # skip test if cannot encode utf8 test string with preferred # encoding. The preferred encoding is assumed to be the default @@ -2163,9 +2263,9 @@ M 33 21.99 with temppath() as path: with io.open(path, "wt") as f: - f.write(u"norm1,norm2,norm3\n") - f.write(u"norm1," + latin1 + u",norm3\n") - f.write(u"test1,testNonethe" + utf8 + u",test3\n") + f.write("norm1,norm2,norm3\n") + f.write("norm1," + latin1 + ",norm3\n") + f.write("test1,testNonethe" + utf8 + ",test3\n") with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', np.VisibleDeprecationWarning) @@ -2177,7 +2277,7 @@ M 33 21.99 ["norm1", "norm2", "norm3"], ["norm1", latin1, "norm3"], ["test1", "testNonethe" + utf8, "test3"]], - dtype=np.unicode_) + dtype=np.str_) assert_array_equal(test, ctl) def test_recfromtxt(self): @@ -2358,6 +2458,69 @@ M 33 21.99 assert_equal(test['f1'], 17179869184) assert_equal(test['f2'], 1024) + def test_unpack_float_data(self): + txt = TextIO("1,2,3\n4,5,6\n7,8,9\n0.0,1.0,2.0") + a, b, c = np.loadtxt(txt, delimiter=",", unpack=True) + assert_array_equal(a, np.array([1.0, 4.0, 7.0, 0.0])) + assert_array_equal(b, np.array([2.0, 5.0, 8.0, 1.0])) + assert_array_equal(c, np.array([3.0, 6.0, 9.0, 2.0])) + + def test_unpack_structured(self): + # Regression test for gh-4341 + # Unpacking should work on structured arrays + txt = TextIO("M 21 72\nF 35 58") + dt = {'names': ('a', 'b', 'c'), 'formats': ('S1', 'i4', 'f4')} + a, b, c = np.genfromtxt(txt, dtype=dt, unpack=True) + assert_equal(a.dtype, np.dtype('S1')) + assert_equal(b.dtype, np.dtype('i4')) + assert_equal(c.dtype, np.dtype('f4')) + assert_array_equal(a, np.array([b'M', b'F'])) + assert_array_equal(b, np.array([21, 35])) + assert_array_equal(c, np.array([72., 58.])) + + def test_unpack_auto_dtype(self): + # Regression test for gh-4341 + # Unpacking should work when dtype=None + txt = TextIO("M 21 72.\nF 35 58.") + expected = (np.array(["M", "F"]), np.array([21, 35]), np.array([72., 58.])) + test = np.genfromtxt(txt, dtype=None, unpack=True, encoding="utf-8") + for arr, result in zip(expected, test): + assert_array_equal(arr, result) + assert_equal(arr.dtype, result.dtype) + + def test_unpack_single_name(self): + # Regression test for gh-4341 + # Unpacking should work when structured dtype has only one field + txt = TextIO("21\n35") + dt = {'names': ('a',), 'formats': ('i4',)} + expected = np.array([21, 35], dtype=np.int32) + test = np.genfromtxt(txt, dtype=dt, unpack=True) + assert_array_equal(expected, test) + assert_equal(expected.dtype, test.dtype) + + def test_squeeze_scalar(self): + # Regression test for gh-4341 + # Unpacking a scalar should give zero-dim output, + # even if dtype is structured + txt = TextIO("1") + dt = {'names': ('a',), 'formats': ('i4',)} + expected = np.array((1,), dtype=np.int32) + test = np.genfromtxt(txt, dtype=dt, unpack=True) + assert_array_equal(expected, test) + assert_equal((), test.shape) + assert_equal(expected.dtype, test.dtype) + + @pytest.mark.parametrize("ndim", [0, 1, 2]) + def test_ndmin_keyword(self, ndim: int): + # lets have the same behaviour of ndmin as loadtxt + # as they should be the same for non-missing values + txt = "42" + + a = np.loadtxt(StringIO(txt), ndmin=ndim) + b = np.genfromtxt(StringIO(txt), ndmin=ndim) + + assert_array_equal(a, b) + class TestPathUsage: # Test that pathlib.Path can be used @@ -2392,6 +2555,7 @@ class TestPathUsage: break_cycles() break_cycles() + @pytest.mark.xfail(IS_WASM, reason="memmap doesn't work correctly") def test_save_load_memmap_readwrite(self): # Test that pathlib.Path instances can be written mem-mapped. with temppath(suffix='.npy') as path: @@ -2433,33 +2597,11 @@ class TestPathUsage: data = np.genfromtxt(path) assert_array_equal(a, data) - def test_ndfromtxt(self): - # Test outputting a standard ndarray - with temppath(suffix='.txt') as path: - path = Path(path) - with path.open('w') as f: - f.write(u'1 2\n3 4') - - control = np.array([[1, 2], [3, 4]], dtype=int) - test = np.genfromtxt(path, dtype=int) - assert_array_equal(test, control) - - def test_mafromtxt(self): - # From `test_fancy_dtype_alt` above - with temppath(suffix='.txt') as path: - path = Path(path) - with path.open('w') as f: - f.write(u'1,2,3.0\n4,5,6.0\n') - - test = np.genfromtxt(path, delimiter=',', usemask=True) - control = ma.array([(1.0, 2.0, 3.0), (4.0, 5.0, 6.0)]) - assert_equal(test, control) - def test_recfromtxt(self): with temppath(suffix='.txt') as path: path = Path(path) with path.open('w') as f: - f.write(u'A,B\n0,1\n2,3') + f.write('A,B\n0,1\n2,3') kwargs = dict(delimiter=",", missing_values="N/A", names=True) test = np.recfromtxt(path, **kwargs) @@ -2472,7 +2614,7 @@ class TestPathUsage: with temppath(suffix='.txt') as path: path = Path(path) with path.open('w') as f: - f.write(u'A,B\n0,1\n2,3') + f.write('A,B\n0,1\n2,3') kwargs = dict(missing_values="N/A", names=True, case_sensitive=True) test = np.recfromcsv(path, dtype=None, **kwargs) @@ -2610,3 +2752,13 @@ def test_load_refcount(): with assert_no_gc_cycles(): x = np.loadtxt(TextIO("0 1 2 3"), dtype=dt) assert_equal(x, np.array([((0, 1), (2, 3))], dtype=dt)) + +def test_load_multiple_arrays_until_eof(): + f = BytesIO() + np.save(f, 1) + np.save(f, 2) + f.seek(0) + assert np.load(f) == 1 + assert np.load(f) == 2 + with pytest.raises(EOFError): + np.load(f) diff --git a/numpy/lib/tests/test_loadtxt.py b/numpy/lib/tests/test_loadtxt.py new file mode 100644 index 000000000..2d805e434 --- /dev/null +++ b/numpy/lib/tests/test_loadtxt.py @@ -0,0 +1,1048 @@ +""" +Tests specific to `np.loadtxt` added during the move of loadtxt to be backed +by C code. +These tests complement those found in `test_io.py`. +""" + +import sys +import os +import pytest +from tempfile import NamedTemporaryFile, mkstemp +from io import StringIO + +import numpy as np +from numpy.ma.testutils import assert_equal +from numpy.testing import assert_array_equal, HAS_REFCOUNT, IS_PYPY + + +def test_scientific_notation(): + """Test that both 'e' and 'E' are parsed correctly.""" + data = StringIO( + ( + "1.0e-1,2.0E1,3.0\n" + "4.0e-2,5.0E-1,6.0\n" + "7.0e-3,8.0E1,9.0\n" + "0.0e-4,1.0E-1,2.0" + ) + ) + expected = np.array( + [[0.1, 20., 3.0], [0.04, 0.5, 6], [0.007, 80., 9], [0, 0.1, 2]] + ) + assert_array_equal(np.loadtxt(data, delimiter=","), expected) + + +@pytest.mark.parametrize("comment", ["..", "//", "@-", "this is a comment:"]) +def test_comment_multiple_chars(comment): + content = "# IGNORE\n1.5, 2.5# ABC\n3.0,4.0# XXX\n5.5,6.0\n" + txt = StringIO(content.replace("#", comment)) + a = np.loadtxt(txt, delimiter=",", comments=comment) + assert_equal(a, [[1.5, 2.5], [3.0, 4.0], [5.5, 6.0]]) + + +@pytest.fixture +def mixed_types_structured(): + """ + Fixture providing hetergeneous input data with a structured dtype, along + with the associated structured array. + """ + data = StringIO( + ( + "1000;2.4;alpha;-34\n" + "2000;3.1;beta;29\n" + "3500;9.9;gamma;120\n" + "4090;8.1;delta;0\n" + "5001;4.4;epsilon;-99\n" + "6543;7.8;omega;-1\n" + ) + ) + dtype = np.dtype( + [('f0', np.uint16), ('f1', np.float64), ('f2', 'S7'), ('f3', np.int8)] + ) + expected = np.array( + [ + (1000, 2.4, "alpha", -34), + (2000, 3.1, "beta", 29), + (3500, 9.9, "gamma", 120), + (4090, 8.1, "delta", 0), + (5001, 4.4, "epsilon", -99), + (6543, 7.8, "omega", -1) + ], + dtype=dtype + ) + return data, dtype, expected + + +@pytest.mark.parametrize('skiprows', [0, 1, 2, 3]) +def test_structured_dtype_and_skiprows_no_empty_lines( + skiprows, mixed_types_structured): + data, dtype, expected = mixed_types_structured + a = np.loadtxt(data, dtype=dtype, delimiter=";", skiprows=skiprows) + assert_array_equal(a, expected[skiprows:]) + + +def test_unpack_structured(mixed_types_structured): + data, dtype, expected = mixed_types_structured + + a, b, c, d = np.loadtxt(data, dtype=dtype, delimiter=";", unpack=True) + assert_array_equal(a, expected["f0"]) + assert_array_equal(b, expected["f1"]) + assert_array_equal(c, expected["f2"]) + assert_array_equal(d, expected["f3"]) + + +def test_structured_dtype_with_shape(): + dtype = np.dtype([("a", "u1", 2), ("b", "u1", 2)]) + data = StringIO("0,1,2,3\n6,7,8,9\n") + expected = np.array([((0, 1), (2, 3)), ((6, 7), (8, 9))], dtype=dtype) + assert_array_equal(np.loadtxt(data, delimiter=",", dtype=dtype), expected) + + +def test_structured_dtype_with_multi_shape(): + dtype = np.dtype([("a", "u1", (2, 2))]) + data = StringIO("0 1 2 3\n") + expected = np.array([(((0, 1), (2, 3)),)], dtype=dtype) + assert_array_equal(np.loadtxt(data, dtype=dtype), expected) + + +def test_nested_structured_subarray(): + # Test from gh-16678 + point = np.dtype([('x', float), ('y', float)]) + dt = np.dtype([('code', int), ('points', point, (2,))]) + data = StringIO("100,1,2,3,4\n200,5,6,7,8\n") + expected = np.array( + [ + (100, [(1., 2.), (3., 4.)]), + (200, [(5., 6.), (7., 8.)]), + ], + dtype=dt + ) + assert_array_equal(np.loadtxt(data, dtype=dt, delimiter=","), expected) + + +def test_structured_dtype_offsets(): + # An aligned structured dtype will have additional padding + dt = np.dtype("i1, i4, i1, i4, i1, i4", align=True) + data = StringIO("1,2,3,4,5,6\n7,8,9,10,11,12\n") + expected = np.array([(1, 2, 3, 4, 5, 6), (7, 8, 9, 10, 11, 12)], dtype=dt) + assert_array_equal(np.loadtxt(data, delimiter=",", dtype=dt), expected) + + +@pytest.mark.parametrize("param", ("skiprows", "max_rows")) +def test_exception_negative_row_limits(param): + """skiprows and max_rows should raise for negative parameters.""" + with pytest.raises(ValueError, match="argument must be nonnegative"): + np.loadtxt("foo.bar", **{param: -3}) + + +@pytest.mark.parametrize("param", ("skiprows", "max_rows")) +def test_exception_noninteger_row_limits(param): + with pytest.raises(TypeError, match="argument must be an integer"): + np.loadtxt("foo.bar", **{param: 1.0}) + + +@pytest.mark.parametrize( + "data, shape", + [ + ("1 2 3 4 5\n", (1, 5)), # Single row + ("1\n2\n3\n4\n5\n", (5, 1)), # Single column + ] +) +def test_ndmin_single_row_or_col(data, shape): + arr = np.array([1, 2, 3, 4, 5]) + arr2d = arr.reshape(shape) + + assert_array_equal(np.loadtxt(StringIO(data), dtype=int), arr) + assert_array_equal(np.loadtxt(StringIO(data), dtype=int, ndmin=0), arr) + assert_array_equal(np.loadtxt(StringIO(data), dtype=int, ndmin=1), arr) + assert_array_equal(np.loadtxt(StringIO(data), dtype=int, ndmin=2), arr2d) + + +@pytest.mark.parametrize("badval", [-1, 3, None, "plate of shrimp"]) +def test_bad_ndmin(badval): + with pytest.raises(ValueError, match="Illegal value of ndmin keyword"): + np.loadtxt("foo.bar", ndmin=badval) + + +@pytest.mark.parametrize( + "ws", + ( + " ", # space + "\t", # tab + "\u2003", # em + "\u00A0", # non-break + "\u3000", # ideographic space + ) +) +def test_blank_lines_spaces_delimit(ws): + txt = StringIO( + f"1 2{ws}30\n\n{ws}\n" + f"4 5 60{ws}\n {ws} \n" + f"7 8 {ws} 90\n # comment\n" + f"3 2 1" + ) + # NOTE: It is unclear that the ` # comment` should succeed. Except + # for delimiter=None, which should use any whitespace (and maybe + # should just be implemented closer to Python + expected = np.array([[1, 2, 30], [4, 5, 60], [7, 8, 90], [3, 2, 1]]) + assert_equal( + np.loadtxt(txt, dtype=int, delimiter=None, comments="#"), expected + ) + + +def test_blank_lines_normal_delimiter(): + txt = StringIO('1,2,30\n\n4,5,60\n\n7,8,90\n# comment\n3,2,1') + expected = np.array([[1, 2, 30], [4, 5, 60], [7, 8, 90], [3, 2, 1]]) + assert_equal( + np.loadtxt(txt, dtype=int, delimiter=',', comments="#"), expected + ) + + +@pytest.mark.parametrize("dtype", (float, object)) +def test_maxrows_no_blank_lines(dtype): + txt = StringIO("1.5,2.5\n3.0,4.0\n5.5,6.0") + res = np.loadtxt(txt, dtype=dtype, delimiter=",", max_rows=2) + assert_equal(res.dtype, dtype) + assert_equal(res, np.array([["1.5", "2.5"], ["3.0", "4.0"]], dtype=dtype)) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", (np.dtype("f8"), np.dtype("i2"))) +def test_exception_message_bad_values(dtype): + txt = StringIO("1,2\n3,XXX\n5,6") + msg = f"could not convert string 'XXX' to {dtype} at row 1, column 2" + with pytest.raises(ValueError, match=msg): + np.loadtxt(txt, dtype=dtype, delimiter=",") + + +def test_converters_negative_indices(): + txt = StringIO('1.5,2.5\n3.0,XXX\n5.5,6.0') + conv = {-1: lambda s: np.nan if s == 'XXX' else float(s)} + expected = np.array([[1.5, 2.5], [3.0, np.nan], [5.5, 6.0]]) + res = np.loadtxt( + txt, dtype=np.float64, delimiter=",", converters=conv, encoding=None + ) + assert_equal(res, expected) + + +def test_converters_negative_indices_with_usecols(): + txt = StringIO('1.5,2.5,3.5\n3.0,4.0,XXX\n5.5,6.0,7.5\n') + conv = {-1: lambda s: np.nan if s == 'XXX' else float(s)} + expected = np.array([[1.5, 3.5], [3.0, np.nan], [5.5, 7.5]]) + res = np.loadtxt( + txt, + dtype=np.float64, + delimiter=",", + converters=conv, + usecols=[0, -1], + encoding=None, + ) + assert_equal(res, expected) + + # Second test with variable number of rows: + res = np.loadtxt(StringIO('''0,1,2\n0,1,2,3,4'''), delimiter=",", + usecols=[0, -1], converters={-1: (lambda x: -1)}) + assert_array_equal(res, [[0, -1], [0, -1]]) + + +def test_ragged_error(): + rows = ["1,2,3", "1,2,3", "4,3,2,1"] + with pytest.raises(ValueError, + match="the number of columns changed from 3 to 4 at row 3"): + np.loadtxt(rows, delimiter=",") + + +def test_ragged_usecols(): + # usecols, and negative ones, work even with varying number of columns. + txt = StringIO("0,0,XXX\n0,XXX,0,XXX\n0,XXX,XXX,0,XXX\n") + expected = np.array([[0, 0], [0, 0], [0, 0]]) + res = np.loadtxt(txt, dtype=float, delimiter=",", usecols=[0, -2]) + assert_equal(res, expected) + + txt = StringIO("0,0,XXX\n0\n0,XXX,XXX,0,XXX\n") + with pytest.raises(ValueError, + match="invalid column index -2 at row 2 with 1 columns"): + # There is no -2 column in the second row: + np.loadtxt(txt, dtype=float, delimiter=",", usecols=[0, -2]) + + +def test_empty_usecols(): + txt = StringIO("0,0,XXX\n0,XXX,0,XXX\n0,XXX,XXX,0,XXX\n") + res = np.loadtxt(txt, dtype=np.dtype([]), delimiter=",", usecols=[]) + assert res.shape == (3,) + assert res.dtype == np.dtype([]) + + +@pytest.mark.parametrize("c1", ["a", "の", "🫕"]) +@pytest.mark.parametrize("c2", ["a", "の", "🫕"]) +def test_large_unicode_characters(c1, c2): + # c1 and c2 span ascii, 16bit and 32bit range. + txt = StringIO(f"a,{c1},c,1.0\ne,{c2},2.0,g") + res = np.loadtxt(txt, dtype=np.dtype('U12'), delimiter=",") + expected = np.array( + [f"a,{c1},c,1.0".split(","), f"e,{c2},2.0,g".split(",")], + dtype=np.dtype('U12') + ) + assert_equal(res, expected) + + +def test_unicode_with_converter(): + txt = StringIO("cat,dog\nαβγ,δεζ\nabc,def\n") + conv = {0: lambda s: s.upper()} + res = np.loadtxt( + txt, + dtype=np.dtype("U12"), + converters=conv, + delimiter=",", + encoding=None + ) + expected = np.array([['CAT', 'dog'], ['ΑΒΓ', 'δεζ'], ['ABC', 'def']]) + assert_equal(res, expected) + + +def test_converter_with_structured_dtype(): + txt = StringIO('1.5,2.5,Abc\n3.0,4.0,dEf\n5.5,6.0,ghI\n') + dt = np.dtype([('m', np.int32), ('r', np.float32), ('code', 'U8')]) + conv = {0: lambda s: int(10*float(s)), -1: lambda s: s.upper()} + res = np.loadtxt(txt, dtype=dt, delimiter=",", converters=conv) + expected = np.array( + [(15, 2.5, 'ABC'), (30, 4.0, 'DEF'), (55, 6.0, 'GHI')], dtype=dt + ) + assert_equal(res, expected) + + +def test_converter_with_unicode_dtype(): + """ + With the default 'bytes' encoding, tokens are encoded prior to being + passed to the converter. This means that the output of the converter may + be bytes instead of unicode as expected by `read_rows`. + + This test checks that outputs from the above scenario are properly decoded + prior to parsing by `read_rows`. + """ + txt = StringIO('abc,def\nrst,xyz') + conv = bytes.upper + res = np.loadtxt( + txt, dtype=np.dtype("U3"), converters=conv, delimiter=",") + expected = np.array([['ABC', 'DEF'], ['RST', 'XYZ']]) + assert_equal(res, expected) + + +def test_read_huge_row(): + row = "1.5, 2.5," * 50000 + row = row[:-1] + "\n" + txt = StringIO(row * 2) + res = np.loadtxt(txt, delimiter=",", dtype=float) + assert_equal(res, np.tile([1.5, 2.5], (2, 50000))) + + +@pytest.mark.parametrize("dtype", "edfgFDG") +def test_huge_float(dtype): + # Covers a non-optimized path that is rarely taken: + field = "0" * 1000 + ".123456789" + dtype = np.dtype(dtype) + value = np.loadtxt([field], dtype=dtype)[()] + assert value == dtype.type("0.123456789") + + +@pytest.mark.parametrize( + ("given_dtype", "expected_dtype"), + [ + ("S", np.dtype("S5")), + ("U", np.dtype("U5")), + ], +) +def test_string_no_length_given(given_dtype, expected_dtype): + """ + The given dtype is just 'S' or 'U' with no length. In these cases, the + length of the resulting dtype is determined by the longest string found + in the file. + """ + txt = StringIO("AAA,5-1\nBBBBB,0-3\nC,4-9\n") + res = np.loadtxt(txt, dtype=given_dtype, delimiter=",") + expected = np.array( + [['AAA', '5-1'], ['BBBBB', '0-3'], ['C', '4-9']], dtype=expected_dtype + ) + assert_equal(res, expected) + assert_equal(res.dtype, expected_dtype) + + +def test_float_conversion(): + """ + Some tests that the conversion to float64 works as accurately as the + Python built-in `float` function. In a naive version of the float parser, + these strings resulted in values that were off by an ULP or two. + """ + strings = [ + '0.9999999999999999', + '9876543210.123456', + '5.43215432154321e+300', + '0.901', + '0.333', + ] + txt = StringIO('\n'.join(strings)) + res = np.loadtxt(txt) + expected = np.array([float(s) for s in strings]) + assert_equal(res, expected) + + +def test_bool(): + # Simple test for bool via integer + txt = StringIO("1, 0\n10, -1") + res = np.loadtxt(txt, dtype=bool, delimiter=",") + assert res.dtype == bool + assert_array_equal(res, [[True, False], [True, True]]) + # Make sure we use only 1 and 0 on the byte level: + assert_array_equal(res.view(np.uint8), [[1, 0], [1, 1]]) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", np.typecodes["AllInteger"]) +@pytest.mark.filterwarnings("error:.*integer via a float.*:DeprecationWarning") +def test_integer_signs(dtype): + dtype = np.dtype(dtype) + assert np.loadtxt(["+2"], dtype=dtype) == 2 + if dtype.kind == "u": + with pytest.raises(ValueError): + np.loadtxt(["-1\n"], dtype=dtype) + else: + assert np.loadtxt(["-2\n"], dtype=dtype) == -2 + + for sign in ["++", "+-", "--", "-+"]: + with pytest.raises(ValueError): + np.loadtxt([f"{sign}2\n"], dtype=dtype) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", np.typecodes["AllInteger"]) +@pytest.mark.filterwarnings("error:.*integer via a float.*:DeprecationWarning") +def test_implicit_cast_float_to_int_fails(dtype): + txt = StringIO("1.0, 2.1, 3.7\n4, 5, 6") + with pytest.raises(ValueError): + np.loadtxt(txt, dtype=dtype, delimiter=",") + +@pytest.mark.parametrize("dtype", (np.complex64, np.complex128)) +@pytest.mark.parametrize("with_parens", (False, True)) +def test_complex_parsing(dtype, with_parens): + s = "(1.0-2.5j),3.75,(7+-5.0j)\n(4),(-19e2j),(0)" + if not with_parens: + s = s.replace("(", "").replace(")", "") + + res = np.loadtxt(StringIO(s), dtype=dtype, delimiter=",") + expected = np.array( + [[1.0-2.5j, 3.75, 7-5j], [4.0, -1900j, 0]], dtype=dtype + ) + assert_equal(res, expected) + + +def test_read_from_generator(): + def gen(): + for i in range(4): + yield f"{i},{2*i},{i**2}" + + res = np.loadtxt(gen(), dtype=int, delimiter=",") + expected = np.array([[0, 0, 0], [1, 2, 1], [2, 4, 4], [3, 6, 9]]) + assert_equal(res, expected) + + +def test_read_from_generator_multitype(): + def gen(): + for i in range(3): + yield f"{i} {i / 4}" + + res = np.loadtxt(gen(), dtype="i, d", delimiter=" ") + expected = np.array([(0, 0.0), (1, 0.25), (2, 0.5)], dtype="i, d") + assert_equal(res, expected) + + +def test_read_from_bad_generator(): + def gen(): + for entry in ["1,2", b"3, 5", 12738]: + yield entry + + with pytest.raises( + TypeError, match=r"non-string returned while reading data"): + np.loadtxt(gen(), dtype="i, i", delimiter=",") + + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") +def test_object_cleanup_on_read_error(): + sentinel = object() + already_read = 0 + + def conv(x): + nonlocal already_read + if already_read > 4999: + raise ValueError("failed half-way through!") + already_read += 1 + return sentinel + + txt = StringIO("x\n" * 10000) + + with pytest.raises(ValueError, match="at row 5000, column 1"): + np.loadtxt(txt, dtype=object, converters={0: conv}) + + assert sys.getrefcount(sentinel) == 2 + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +def test_character_not_bytes_compatible(): + """Test exception when a character cannot be encoded as 'S'.""" + data = StringIO("–") # == \u2013 + with pytest.raises(ValueError): + np.loadtxt(data, dtype="S5") + + +@pytest.mark.parametrize("conv", (0, [float], "")) +def test_invalid_converter(conv): + msg = ( + "converters must be a dictionary mapping columns to converter " + "functions or a single callable." + ) + with pytest.raises(TypeError, match=msg): + np.loadtxt(StringIO("1 2\n3 4"), converters=conv) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +def test_converters_dict_raises_non_integer_key(): + with pytest.raises(TypeError, match="keys of the converters dict"): + np.loadtxt(StringIO("1 2\n3 4"), converters={"a": int}) + with pytest.raises(TypeError, match="keys of the converters dict"): + np.loadtxt(StringIO("1 2\n3 4"), converters={"a": int}, usecols=0) + + +@pytest.mark.parametrize("bad_col_ind", (3, -3)) +def test_converters_dict_raises_non_col_key(bad_col_ind): + data = StringIO("1 2\n3 4") + with pytest.raises(ValueError, match="converter specified for column"): + np.loadtxt(data, converters={bad_col_ind: int}) + + +def test_converters_dict_raises_val_not_callable(): + with pytest.raises(TypeError, + match="values of the converters dictionary must be callable"): + np.loadtxt(StringIO("1 2\n3 4"), converters={0: 1}) + + +@pytest.mark.parametrize("q", ('"', "'", "`")) +def test_quoted_field(q): + txt = StringIO( + f"{q}alpha, x{q}, 2.5\n{q}beta, y{q}, 4.5\n{q}gamma, z{q}, 5.0\n" + ) + dtype = np.dtype([('f0', 'U8'), ('f1', np.float64)]) + expected = np.array( + [("alpha, x", 2.5), ("beta, y", 4.5), ("gamma, z", 5.0)], dtype=dtype + ) + + res = np.loadtxt(txt, dtype=dtype, delimiter=",", quotechar=q) + assert_array_equal(res, expected) + + +@pytest.mark.parametrize("q", ('"', "'", "`")) +def test_quoted_field_with_whitepace_delimiter(q): + txt = StringIO( + f"{q}alpha, x{q} 2.5\n{q}beta, y{q} 4.5\n{q}gamma, z{q} 5.0\n" + ) + dtype = np.dtype([('f0', 'U8'), ('f1', np.float64)]) + expected = np.array( + [("alpha, x", 2.5), ("beta, y", 4.5), ("gamma, z", 5.0)], dtype=dtype + ) + + res = np.loadtxt(txt, dtype=dtype, delimiter=None, quotechar=q) + assert_array_equal(res, expected) + + +def test_quote_support_default(): + """Support for quoted fields is disabled by default.""" + txt = StringIO('"lat,long", 45, 30\n') + dtype = np.dtype([('f0', 'U24'), ('f1', np.float64), ('f2', np.float64)]) + + with pytest.raises(ValueError, + match="the dtype passed requires 3 columns but 4 were"): + np.loadtxt(txt, dtype=dtype, delimiter=",") + + # Enable quoting support with non-None value for quotechar param + txt.seek(0) + expected = np.array([("lat,long", 45., 30.)], dtype=dtype) + + res = np.loadtxt(txt, dtype=dtype, delimiter=",", quotechar='"') + assert_array_equal(res, expected) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +def test_quotechar_multichar_error(): + txt = StringIO("1,2\n3,4") + msg = r".*must be a single unicode character or None" + with pytest.raises(TypeError, match=msg): + np.loadtxt(txt, delimiter=",", quotechar="''") + + +def test_comment_multichar_error_with_quote(): + txt = StringIO("1,2\n3,4") + msg = ( + "when multiple comments or a multi-character comment is given, " + "quotes are not supported." + ) + with pytest.raises(ValueError, match=msg): + np.loadtxt(txt, delimiter=",", comments="123", quotechar='"') + with pytest.raises(ValueError, match=msg): + np.loadtxt(txt, delimiter=",", comments=["#", "%"], quotechar='"') + + # A single character string in a tuple is unpacked though: + res = np.loadtxt(txt, delimiter=",", comments=("#",), quotechar="'") + assert_equal(res, [[1, 2], [3, 4]]) + + +def test_structured_dtype_with_quotes(): + data = StringIO( + ( + "1000;2.4;'alpha';-34\n" + "2000;3.1;'beta';29\n" + "3500;9.9;'gamma';120\n" + "4090;8.1;'delta';0\n" + "5001;4.4;'epsilon';-99\n" + "6543;7.8;'omega';-1\n" + ) + ) + dtype = np.dtype( + [('f0', np.uint16), ('f1', np.float64), ('f2', 'S7'), ('f3', np.int8)] + ) + expected = np.array( + [ + (1000, 2.4, "alpha", -34), + (2000, 3.1, "beta", 29), + (3500, 9.9, "gamma", 120), + (4090, 8.1, "delta", 0), + (5001, 4.4, "epsilon", -99), + (6543, 7.8, "omega", -1) + ], + dtype=dtype + ) + res = np.loadtxt(data, dtype=dtype, delimiter=";", quotechar="'") + assert_array_equal(res, expected) + + +def test_quoted_field_is_not_empty(): + txt = StringIO('1\n\n"4"\n""') + expected = np.array(["1", "4", ""], dtype="U1") + res = np.loadtxt(txt, delimiter=",", dtype="U1", quotechar='"') + assert_equal(res, expected) + +def test_quoted_field_is_not_empty_nonstrict(): + # Same as test_quoted_field_is_not_empty but check that we are not strict + # about missing closing quote (this is the `csv.reader` default also) + txt = StringIO('1\n\n"4"\n"') + expected = np.array(["1", "4", ""], dtype="U1") + res = np.loadtxt(txt, delimiter=",", dtype="U1", quotechar='"') + assert_equal(res, expected) + +def test_consecutive_quotechar_escaped(): + txt = StringIO('"Hello, my name is ""Monty""!"') + expected = np.array('Hello, my name is "Monty"!', dtype="U40") + res = np.loadtxt(txt, dtype="U40", delimiter=",", quotechar='"') + assert_equal(res, expected) + + +@pytest.mark.parametrize("data", ("", "\n\n\n", "# 1 2 3\n# 4 5 6\n")) +@pytest.mark.parametrize("ndmin", (0, 1, 2)) +@pytest.mark.parametrize("usecols", [None, (1, 2, 3)]) +def test_warn_on_no_data(data, ndmin, usecols): + """Check that a UserWarning is emitted when no data is read from input.""" + if usecols is not None: + expected_shape = (0, 3) + elif ndmin == 2: + expected_shape = (0, 1) # guess a single column?! + else: + expected_shape = (0,) + + txt = StringIO(data) + with pytest.warns(UserWarning, match="input contained no data"): + res = np.loadtxt(txt, ndmin=ndmin, usecols=usecols) + assert res.shape == expected_shape + + with NamedTemporaryFile(mode="w") as fh: + fh.write(data) + fh.seek(0) + with pytest.warns(UserWarning, match="input contained no data"): + res = np.loadtxt(txt, ndmin=ndmin, usecols=usecols) + assert res.shape == expected_shape + +@pytest.mark.parametrize("skiprows", (2, 3)) +def test_warn_on_skipped_data(skiprows): + data = "1 2 3\n4 5 6" + txt = StringIO(data) + with pytest.warns(UserWarning, match="input contained no data"): + np.loadtxt(txt, skiprows=skiprows) + + +@pytest.mark.parametrize(["dtype", "value"], [ + ("i2", 0x0001), ("u2", 0x0001), + ("i4", 0x00010203), ("u4", 0x00010203), + ("i8", 0x0001020304050607), ("u8", 0x0001020304050607), + # The following values are constructed to lead to unique bytes: + ("float16", 3.07e-05), + ("float32", 9.2557e-41), ("complex64", 9.2557e-41+2.8622554e-29j), + ("float64", -1.758571353180402e-24), + # Here and below, the repr side-steps a small loss of precision in + # complex `str` in PyPy (which is probably fine, as repr works): + ("complex128", repr(5.406409232372729e-29-1.758571353180402e-24j)), + # Use integer values that fit into double. Everything else leads to + # problems due to longdoubles going via double and decimal strings + # causing rounding errors. + ("longdouble", 0x01020304050607), + ("clongdouble", repr(0x01020304050607 + (0x00121314151617 * 1j))), + ("U2", "\U00010203\U000a0b0c")]) +@pytest.mark.parametrize("swap", [True, False]) +def test_byteswapping_and_unaligned(dtype, value, swap): + # Try to create "interesting" values within the valid unicode range: + dtype = np.dtype(dtype) + data = [f"x,{value}\n"] # repr as PyPy `str` truncates some + if swap: + dtype = dtype.newbyteorder() + full_dt = np.dtype([("a", "S1"), ("b", dtype)], align=False) + # The above ensures that the interesting "b" field is unaligned: + assert full_dt.fields["b"][1] == 1 + res = np.loadtxt(data, dtype=full_dt, delimiter=",", encoding=None, + max_rows=1) # max-rows prevents over-allocation + assert res["b"] == dtype.type(value) + + +@pytest.mark.parametrize("dtype", + np.typecodes["AllInteger"] + "efdFD" + "?") +def test_unicode_whitespace_stripping(dtype): + # Test that all numeric types (and bool) strip whitespace correctly + # \u202F is a narrow no-break space, `\n` is just a whitespace if quoted. + # Currently, skip float128 as it did not always support this and has no + # "custom" parsing: + txt = StringIO(' 3 ,"\u202F2\n"') + res = np.loadtxt(txt, dtype=dtype, delimiter=",", quotechar='"') + assert_array_equal(res, np.array([3, 2]).astype(dtype)) + + +@pytest.mark.parametrize("dtype", "FD") +def test_unicode_whitespace_stripping_complex(dtype): + # Complex has a few extra cases since it has two components and + # parentheses + line = " 1 , 2+3j , ( 4+5j ), ( 6+-7j ) , 8j , ( 9j ) \n" + data = [line, line.replace(" ", "\u202F")] + res = np.loadtxt(data, dtype=dtype, delimiter=',') + assert_array_equal(res, np.array([[1, 2+3j, 4+5j, 6-7j, 8j, 9j]] * 2)) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", "FD") +@pytest.mark.parametrize("field", + ["1 +2j", "1+ 2j", "1+2 j", "1+-+3", "(1j", "(1", "(1+2j", "1+2j)"]) +def test_bad_complex(dtype, field): + with pytest.raises(ValueError): + np.loadtxt([field + "\n"], dtype=dtype, delimiter=",") + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", + np.typecodes["AllInteger"] + "efgdFDG" + "?") +def test_nul_character_error(dtype): + # Test that a \0 character is correctly recognized as an error even if + # what comes before is valid (not everything gets parsed internally). + if dtype.lower() == "g": + pytest.xfail("longdouble/clongdouble assignment may misbehave.") + with pytest.raises(ValueError): + np.loadtxt(["1\000"], dtype=dtype, delimiter=",", quotechar='"') + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +@pytest.mark.parametrize("dtype", + np.typecodes["AllInteger"] + "efgdFDG" + "?") +def test_no_thousands_support(dtype): + # Mainly to document behaviour, Python supports thousands like 1_1. + # (e and G may end up using different conversion and support it, this is + # a bug but happens...) + if dtype == "e": + pytest.skip("half assignment currently uses Python float converter") + if dtype in "eG": + pytest.xfail("clongdouble assignment is buggy (uses `complex`?).") + + assert int("1_1") == float("1_1") == complex("1_1") == 11 + with pytest.raises(ValueError): + np.loadtxt(["1_1\n"], dtype=dtype) + + +@pytest.mark.parametrize("data", [ + ["1,2\n", "2\n,3\n"], + ["1,2\n", "2\r,3\n"]]) +def test_bad_newline_in_iterator(data): + # In NumPy <=1.22 this was accepted, because newlines were completely + # ignored when the input was an iterable. This could be changed, but right + # now, we raise an error. + msg = "Found an unquoted embedded newline within a single line" + with pytest.raises(ValueError, match=msg): + np.loadtxt(data, delimiter=",") + + +@pytest.mark.parametrize("data", [ + ["1,2\n", "2,3\r\n"], # a universal newline + ["1,2\n", "'2\n',3\n"], # a quoted newline + ["1,2\n", "'2\r',3\n"], + ["1,2\n", "'2\r\n',3\n"], +]) +def test_good_newline_in_iterator(data): + # The quoted newlines will be untransformed here, but are just whitespace. + res = np.loadtxt(data, delimiter=",", quotechar="'") + assert_array_equal(res, [[1., 2.], [2., 3.]]) + + +@pytest.mark.parametrize("newline", ["\n", "\r", "\r\n"]) +def test_universal_newlines_quoted(newline): + # Check that universal newline support within the tokenizer is not applied + # to quoted fields. (note that lines must end in newline or quoted + # fields will not include a newline at all) + data = ['1,"2\n"\n', '3,"4\n', '1"\n'] + data = [row.replace("\n", newline) for row in data] + res = np.loadtxt(data, dtype=object, delimiter=",", quotechar='"') + assert_array_equal(res, [['1', f'2{newline}'], ['3', f'4{newline}1']]) + + +def test_null_character(): + # Basic tests to check that the NUL character is not special: + res = np.loadtxt(["1\0002\0003\n", "4\0005\0006"], delimiter="\000") + assert_array_equal(res, [[1, 2, 3], [4, 5, 6]]) + + # Also not as part of a field (avoid unicode/arrays as unicode strips \0) + res = np.loadtxt(["1\000,2\000,3\n", "4\000,5\000,6"], + delimiter=",", dtype=object) + assert res.tolist() == [["1\000", "2\000", "3"], ["4\000", "5\000", "6"]] + + +def test_iterator_fails_getting_next_line(): + class BadSequence: + def __len__(self): + return 100 + + def __getitem__(self, item): + if item == 50: + raise RuntimeError("Bad things happened!") + return f"{item}, {item+1}" + + with pytest.raises(RuntimeError, match="Bad things happened!"): + np.loadtxt(BadSequence(), dtype=int, delimiter=",") + + +class TestCReaderUnitTests: + # These are internal tests for path that should not be possible to hit + # unless things go very very wrong somewhere. + def test_not_an_filelike(self): + with pytest.raises(AttributeError, match=".*read"): + np.core._multiarray_umath._load_from_filelike( + object(), dtype=np.dtype("i"), filelike=True) + + def test_filelike_read_fails(self): + # Can only be reached if loadtxt opens the file, so it is hard to do + # via the public interface (although maybe not impossible considering + # the current "DataClass" backing). + class BadFileLike: + counter = 0 + + def read(self, size): + self.counter += 1 + if self.counter > 20: + raise RuntimeError("Bad bad bad!") + return "1,2,3\n" + + with pytest.raises(RuntimeError, match="Bad bad bad!"): + np.core._multiarray_umath._load_from_filelike( + BadFileLike(), dtype=np.dtype("i"), filelike=True) + + def test_filelike_bad_read(self): + # Can only be reached if loadtxt opens the file, so it is hard to do + # via the public interface (although maybe not impossible considering + # the current "DataClass" backing). + + class BadFileLike: + counter = 0 + + def read(self, size): + return 1234 # not a string! + + with pytest.raises(TypeError, + match="non-string returned while reading data"): + np.core._multiarray_umath._load_from_filelike( + BadFileLike(), dtype=np.dtype("i"), filelike=True) + + def test_not_an_iter(self): + with pytest.raises(TypeError, + match="error reading from object, expected an iterable"): + np.core._multiarray_umath._load_from_filelike( + object(), dtype=np.dtype("i"), filelike=False) + + def test_bad_type(self): + with pytest.raises(TypeError, match="internal error: dtype must"): + np.core._multiarray_umath._load_from_filelike( + object(), dtype="i", filelike=False) + + def test_bad_encoding(self): + with pytest.raises(TypeError, match="encoding must be a unicode"): + np.core._multiarray_umath._load_from_filelike( + object(), dtype=np.dtype("i"), filelike=False, encoding=123) + + @pytest.mark.parametrize("newline", ["\r", "\n", "\r\n"]) + def test_manual_universal_newlines(self, newline): + # This is currently not available to users, because we should always + # open files with universal newlines enabled `newlines=None`. + # (And reading from an iterator uses slightly different code paths.) + # We have no real support for `newline="\r"` or `newline="\n" as the + # user cannot specify those options. + data = StringIO('0\n1\n"2\n"\n3\n4 #\n'.replace("\n", newline), + newline="") + + res = np.core._multiarray_umath._load_from_filelike( + data, dtype=np.dtype("U10"), filelike=True, + quote='"', comment="#", skiplines=1) + assert_array_equal(res[:, 0], ["1", f"2{newline}", "3", "4 "]) + + +def test_delimiter_comment_collision_raises(): + with pytest.raises(TypeError, match=".*control characters.*incompatible"): + np.loadtxt(StringIO("1, 2, 3"), delimiter=",", comments=",") + + +def test_delimiter_quotechar_collision_raises(): + with pytest.raises(TypeError, match=".*control characters.*incompatible"): + np.loadtxt(StringIO("1, 2, 3"), delimiter=",", quotechar=",") + + +def test_comment_quotechar_collision_raises(): + with pytest.raises(TypeError, match=".*control characters.*incompatible"): + np.loadtxt(StringIO("1 2 3"), comments="#", quotechar="#") + + +def test_delimiter_and_multiple_comments_collision_raises(): + with pytest.raises( + TypeError, match="Comment characters.*cannot include the delimiter" + ): + np.loadtxt(StringIO("1, 2, 3"), delimiter=",", comments=["#", ","]) + + +@pytest.mark.parametrize( + "ws", + ( + " ", # space + "\t", # tab + "\u2003", # em + "\u00A0", # non-break + "\u3000", # ideographic space + ) +) +def test_collision_with_default_delimiter_raises(ws): + with pytest.raises(TypeError, match=".*control characters.*incompatible"): + np.loadtxt(StringIO(f"1{ws}2{ws}3\n4{ws}5{ws}6\n"), comments=ws) + with pytest.raises(TypeError, match=".*control characters.*incompatible"): + np.loadtxt(StringIO(f"1{ws}2{ws}3\n4{ws}5{ws}6\n"), quotechar=ws) + + +@pytest.mark.parametrize("nl", ("\n", "\r")) +def test_control_character_newline_raises(nl): + txt = StringIO(f"1{nl}2{nl}3{nl}{nl}4{nl}5{nl}6{nl}{nl}") + msg = "control character.*cannot be a newline" + with pytest.raises(TypeError, match=msg): + np.loadtxt(txt, delimiter=nl) + with pytest.raises(TypeError, match=msg): + np.loadtxt(txt, comments=nl) + with pytest.raises(TypeError, match=msg): + np.loadtxt(txt, quotechar=nl) + + +@pytest.mark.parametrize( + ("generic_data", "long_datum", "unitless_dtype", "expected_dtype"), + [ + ("2012-03", "2013-01-15", "M8", "M8[D]"), # Datetimes + ("spam-a-lot", "tis_but_a_scratch", "U", "U17"), # str + ], +) +@pytest.mark.parametrize("nrows", (10, 50000, 60000)) # lt, eq, gt chunksize +def test_parametric_unit_discovery( + generic_data, long_datum, unitless_dtype, expected_dtype, nrows +): + """Check that the correct unit (e.g. month, day, second) is discovered from + the data when a user specifies a unitless datetime.""" + # Unit should be "D" (days) due to last entry + data = [generic_data] * 50000 + [long_datum] + expected = np.array(data, dtype=expected_dtype) + + # file-like path + txt = StringIO("\n".join(data)) + a = np.loadtxt(txt, dtype=unitless_dtype) + assert a.dtype == expected.dtype + assert_equal(a, expected) + + # file-obj path + fd, fname = mkstemp() + os.close(fd) + with open(fname, "w") as fh: + fh.write("\n".join(data)) + a = np.loadtxt(fname, dtype=unitless_dtype) + os.remove(fname) + assert a.dtype == expected.dtype + assert_equal(a, expected) + + +def test_str_dtype_unit_discovery_with_converter(): + data = ["spam-a-lot"] * 60000 + ["XXXtis_but_a_scratch"] + expected = np.array( + ["spam-a-lot"] * 60000 + ["tis_but_a_scratch"], dtype="U17" + ) + conv = lambda s: s.strip("XXX") + + # file-like path + txt = StringIO("\n".join(data)) + a = np.loadtxt(txt, dtype="U", converters=conv, encoding=None) + assert a.dtype == expected.dtype + assert_equal(a, expected) + + # file-obj path + fd, fname = mkstemp() + os.close(fd) + with open(fname, "w") as fh: + fh.write("\n".join(data)) + a = np.loadtxt(fname, dtype="U", converters=conv, encoding=None) + os.remove(fname) + assert a.dtype == expected.dtype + assert_equal(a, expected) + + +@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8), + reason="PyPy bug in error formatting") +def test_control_character_empty(): + with pytest.raises(TypeError, match="Text reading control character must"): + np.loadtxt(StringIO("1 2 3"), delimiter="") + with pytest.raises(TypeError, match="Text reading control character must"): + np.loadtxt(StringIO("1 2 3"), quotechar="") + with pytest.raises(ValueError, match="comments cannot be an empty string"): + np.loadtxt(StringIO("1 2 3"), comments="") + with pytest.raises(ValueError, match="comments cannot be an empty string"): + np.loadtxt(StringIO("1 2 3"), comments=["#", ""]) + + +def test_control_characters_as_bytes(): + """Byte control characters (comments, delimiter) are supported.""" + a = np.loadtxt(StringIO("#header\n1,2,3"), comments=b"#", delimiter=b",") + assert_equal(a, [1, 2, 3]) + + +@pytest.mark.filterwarnings('ignore::UserWarning') +def test_field_growing_cases(): + # Test empty field appending/growing (each field still takes 1 character) + # to see if the final field appending does not create issues. + res = np.loadtxt([""], delimiter=",", dtype=bytes) + assert len(res) == 0 + + for i in range(1, 1024): + res = np.loadtxt(["," * i], delimiter=",", dtype=bytes) + assert len(res) == i+1 diff --git a/numpy/lib/tests/test_nanfunctions.py b/numpy/lib/tests/test_nanfunctions.py index e0f723a3c..257de381b 100644 --- a/numpy/lib/tests/test_nanfunctions.py +++ b/numpy/lib/tests/test_nanfunctions.py @@ -1,11 +1,13 @@ import warnings import pytest +import inspect import numpy as np +from numpy.core.numeric import normalize_axis_tuple from numpy.lib.nanfunctions import _nan_mask, _replace_nan from numpy.testing import ( - assert_, assert_equal, assert_almost_equal, assert_no_warnings, - assert_raises, assert_array_equal, suppress_warnings + assert_, assert_equal, assert_almost_equal, assert_raises, + assert_array_equal, suppress_warnings ) @@ -35,6 +37,53 @@ _ndat_zeros = np.array([[0.6244, 0.0, 0.2692, 0.0116, 0.0, 0.1170], [0.1610, 0.0, 0.0, 0.1859, 0.3146, 0.0]]) +class TestSignatureMatch: + NANFUNCS = { + np.nanmin: np.amin, + np.nanmax: np.amax, + np.nanargmin: np.argmin, + np.nanargmax: np.argmax, + np.nansum: np.sum, + np.nanprod: np.prod, + np.nancumsum: np.cumsum, + np.nancumprod: np.cumprod, + np.nanmean: np.mean, + np.nanmedian: np.median, + np.nanpercentile: np.percentile, + np.nanquantile: np.quantile, + np.nanvar: np.var, + np.nanstd: np.std, + } + IDS = [k.__name__ for k in NANFUNCS] + + @staticmethod + def get_signature(func, default="..."): + """Construct a signature and replace all default parameter-values.""" + prm_list = [] + signature = inspect.signature(func) + for prm in signature.parameters.values(): + if prm.default is inspect.Parameter.empty: + prm_list.append(prm) + else: + prm_list.append(prm.replace(default=default)) + return inspect.Signature(prm_list) + + @pytest.mark.parametrize("nan_func,func", NANFUNCS.items(), ids=IDS) + def test_signature_match(self, nan_func, func): + # Ignore the default parameter-values as they can sometimes differ + # between the two functions (*e.g.* one has `False` while the other + # has `np._NoValue`) + signature = self.get_signature(func) + nan_signature = self.get_signature(nan_func) + np.testing.assert_equal(signature, nan_signature) + + def test_exhaustiveness(self): + """Validate that all nan functions are actually tested.""" + np.testing.assert_equal( + set(self.IDS), set(np.lib.nanfunctions.__all__) + ) + + class TestNanFunctions_MinMax: nanfuncs = [np.nanmin, np.nanmax] @@ -83,21 +132,23 @@ class TestNanFunctions_MinMax: res = nf(_ndat, axis=1) assert_almost_equal(res, tgt) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for f in self.nanfuncs: - for axis in [None, 0, 1]: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - assert_(np.isnan(f(mat, axis=axis)).all()) - assert_(len(w) == 1, 'no warning raised') - assert_(issubclass(w[0].category, RuntimeWarning)) - # Check scalars - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - assert_(np.isnan(f(np.nan))) - assert_(len(w) == 1, 'no warning raised') - assert_(issubclass(w[0].category, RuntimeWarning)) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + match = "All-NaN slice encountered" + for func in self.nanfuncs: + with pytest.warns(RuntimeWarning, match=match): + out = func(array, axis=axis) + assert np.isnan(out).all() + assert out.dtype == array.dtype def test_masked(self): mat = np.ma.fix_invalid(_ndat) @@ -168,6 +219,46 @@ class TestNanFunctions_MinMax: assert_(len(w) == 1, 'no warning raised') assert_(issubclass(w[0].category, RuntimeWarning)) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_initial(self, dtype): + class MyNDArray(np.ndarray): + pass + + ar = np.arange(9).astype(dtype) + ar[:5] = np.nan + + for f in self.nanfuncs: + initial = 100 if f is np.nanmax else 0 + + ret1 = f(ar, initial=initial) + assert ret1.dtype == dtype + assert ret1 == initial + + ret2 = f(ar.view(MyNDArray), initial=initial) + assert ret2.dtype == dtype + assert ret2 == initial + + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_where(self, dtype): + class MyNDArray(np.ndarray): + pass + + ar = np.arange(9).reshape(3, 3).astype(dtype) + ar[0, :] = np.nan + where = np.ones_like(ar, dtype=np.bool_) + where[:, 0] = False + + for f in self.nanfuncs: + reference = 4 if f is np.nanmin else 8 + + ret1 = f(ar, where=where, initial=5) + assert ret1.dtype == dtype + assert ret1 == reference + + ret2 = f(ar.view(MyNDArray), where=where, initial=5) + assert ret2.dtype == dtype + assert ret2 == reference + class TestNanFunctions_ArgminArgmax: @@ -193,12 +284,20 @@ class TestNanFunctions_ArgminArgmax: assert_(not fcmp(val, row).any()) assert_(not np.equal(val, row[:ind]).any()) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for f in self.nanfuncs: - for axis in [None, 0, 1]: - assert_raises(ValueError, f, mat, axis=axis) - assert_raises(ValueError, f, np.nan) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + for func in self.nanfuncs: + with pytest.raises(ValueError, match="All-NaN slice encountered"): + func(array, axis=axis) def test_empty(self): mat = np.zeros((0, 3)) @@ -230,80 +329,111 @@ class TestNanFunctions_ArgminArgmax: res = f(mine) assert_(res.shape == ()) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_keepdims(self, dtype): + ar = np.arange(9).astype(dtype) + ar[:5] = np.nan -class TestNanFunctions_IntTypes: - - int_types = (np.int8, np.int16, np.int32, np.int64, np.uint8, - np.uint16, np.uint32, np.uint64) - - mat = np.array([127, 39, 93, 87, 46]) - - def integer_arrays(self): - for dtype in self.int_types: - yield self.mat.astype(dtype) - - def test_nanmin(self): - tgt = np.min(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanmin(mat), tgt) - - def test_nanmax(self): - tgt = np.max(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanmax(mat), tgt) - - def test_nanargmin(self): - tgt = np.argmin(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanargmin(mat), tgt) - - def test_nanargmax(self): - tgt = np.argmax(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanargmax(mat), tgt) - - def test_nansum(self): - tgt = np.sum(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nansum(mat), tgt) - - def test_nanprod(self): - tgt = np.prod(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanprod(mat), tgt) - - def test_nancumsum(self): - tgt = np.cumsum(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nancumsum(mat), tgt) - - def test_nancumprod(self): - tgt = np.cumprod(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nancumprod(mat), tgt) - - def test_nanmean(self): - tgt = np.mean(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanmean(mat), tgt) - - def test_nanvar(self): - tgt = np.var(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanvar(mat), tgt) + for f in self.nanfuncs: + reference = 5 if f is np.nanargmin else 8 + ret = f(ar, keepdims=True) + assert ret.ndim == ar.ndim + assert ret == reference - tgt = np.var(mat, ddof=1) - for mat in self.integer_arrays(): - assert_equal(np.nanvar(mat, ddof=1), tgt) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_out(self, dtype): + ar = np.arange(9).astype(dtype) + ar[:5] = np.nan - def test_nanstd(self): - tgt = np.std(self.mat) - for mat in self.integer_arrays(): - assert_equal(np.nanstd(mat), tgt) + for f in self.nanfuncs: + out = np.zeros((), dtype=np.intp) + reference = 5 if f is np.nanargmin else 8 + ret = f(ar, out=out) + assert ret is out + assert ret == reference + + + +_TEST_ARRAYS = { + "0d": np.array(5), + "1d": np.array([127, 39, 93, 87, 46]) +} +for _v in _TEST_ARRAYS.values(): + _v.setflags(write=False) + + +@pytest.mark.parametrize( + "dtype", + np.typecodes["AllInteger"] + np.typecodes["AllFloat"] + "O", +) +@pytest.mark.parametrize("mat", _TEST_ARRAYS.values(), ids=_TEST_ARRAYS.keys()) +class TestNanFunctions_NumberTypes: + nanfuncs = { + np.nanmin: np.min, + np.nanmax: np.max, + np.nanargmin: np.argmin, + np.nanargmax: np.argmax, + np.nansum: np.sum, + np.nanprod: np.prod, + np.nancumsum: np.cumsum, + np.nancumprod: np.cumprod, + np.nanmean: np.mean, + np.nanmedian: np.median, + np.nanvar: np.var, + np.nanstd: np.std, + } + nanfunc_ids = [i.__name__ for i in nanfuncs] + + @pytest.mark.parametrize("nanfunc,func", nanfuncs.items(), ids=nanfunc_ids) + @np.errstate(over="ignore") + def test_nanfunc(self, mat, dtype, nanfunc, func): + mat = mat.astype(dtype) + tgt = func(mat) + out = nanfunc(mat) + + assert_almost_equal(out, tgt) + if dtype == "O": + assert type(out) is type(tgt) + else: + assert out.dtype == tgt.dtype + + @pytest.mark.parametrize( + "nanfunc,func", + [(np.nanquantile, np.quantile), (np.nanpercentile, np.percentile)], + ids=["nanquantile", "nanpercentile"], + ) + def test_nanfunc_q(self, mat, dtype, nanfunc, func): + mat = mat.astype(dtype) + if mat.dtype.kind == "c": + assert_raises(TypeError, func, mat, q=1) + assert_raises(TypeError, nanfunc, mat, q=1) + + else: + tgt = func(mat, q=1) + out = nanfunc(mat, q=1) + + assert_almost_equal(out, tgt) + + if dtype == "O": + assert type(out) is type(tgt) + else: + assert out.dtype == tgt.dtype + + @pytest.mark.parametrize( + "nanfunc,func", + [(np.nanvar, np.var), (np.nanstd, np.std)], + ids=["nanvar", "nanstd"], + ) + def test_nanfunc_ddof(self, mat, dtype, nanfunc, func): + mat = mat.astype(dtype) + tgt = func(mat, ddof=0.5) + out = nanfunc(mat, ddof=0.5) - tgt = np.std(self.mat, ddof=1) - for mat in self.integer_arrays(): - assert_equal(np.nanstd(mat, ddof=1), tgt) + assert_almost_equal(out, tgt) + if dtype == "O": + assert type(out) is type(tgt) + else: + assert out.dtype == tgt.dtype class SharedNanFunctionsTestsMixin: @@ -416,20 +546,21 @@ class TestNanFunctions_SumProd(SharedNanFunctionsTestsMixin): nanfuncs = [np.nansum, np.nanprod] stdfuncs = [np.sum, np.prod] - def test_allnans(self): - # Check for FutureWarning - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - res = np.nansum([np.nan]*3, axis=None) - assert_(res == 0, 'result is not 0') - assert_(len(w) == 0, 'warning raised') - # Check scalar - res = np.nansum(np.nan) - assert_(res == 0, 'result is not 0') - assert_(len(w) == 0, 'warning raised') - # Check there is no warning for not all-nan - np.nansum([0]*3, axis=None) - assert_(len(w) == 0, 'unwanted warning raised') + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + for func, identity in zip(self.nanfuncs, [0, 1]): + out = func(array, axis=axis) + assert np.all(out == identity) + assert out.dtype == array.dtype def test_empty(self): for f, tgt_value in zip([np.nansum, np.nanprod], [0, 1]): @@ -444,25 +575,51 @@ class TestNanFunctions_SumProd(SharedNanFunctionsTestsMixin): res = f(mat, axis=None) assert_equal(res, tgt) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_initial(self, dtype): + ar = np.arange(9).astype(dtype) + ar[:5] = np.nan + + for f in self.nanfuncs: + reference = 28 if f is np.nansum else 3360 + ret = f(ar, initial=2) + assert ret.dtype == dtype + assert ret == reference + + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_where(self, dtype): + ar = np.arange(9).reshape(3, 3).astype(dtype) + ar[0, :] = np.nan + where = np.ones_like(ar, dtype=np.bool_) + where[:, 0] = False + + for f in self.nanfuncs: + reference = 26 if f is np.nansum else 2240 + ret = f(ar, where=where, initial=2) + assert ret.dtype == dtype + assert ret == reference + class TestNanFunctions_CumSumProd(SharedNanFunctionsTestsMixin): nanfuncs = [np.nancumsum, np.nancumprod] stdfuncs = [np.cumsum, np.cumprod] - def test_allnans(self): - for f, tgt_value in zip(self.nanfuncs, [0, 1]): - # Unlike other nan-functions, sum/prod/cumsum/cumprod don't warn on all nan input - with assert_no_warnings(): - res = f([np.nan]*3, axis=None) - tgt = tgt_value*np.ones((3)) - assert_(np.array_equal(res, tgt), 'result is not %s * np.ones((3))' % (tgt_value)) - # Check scalar - res = f(np.nan) - tgt = tgt_value*np.ones((1)) - assert_(np.array_equal(res, tgt), 'result is not %s * np.ones((1))' % (tgt_value)) - # Check there is no warning for not all-nan - f([0]*3, axis=None) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan) + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + for func, identity in zip(self.nanfuncs, [0, 1]): + out = func(array) + assert np.all(out == identity) + assert out.dtype == array.dtype def test_empty(self): for f, tgt_value in zip(self.nanfuncs, [0, 1]): @@ -558,19 +715,29 @@ class TestNanFunctions_MeanVarStd(SharedNanFunctionsTestsMixin): else: assert_(len(sup.log) == 0) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for f in self.nanfuncs: - for axis in [None, 0, 1]: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - assert_(np.isnan(f(mat, axis=axis)).all()) - assert_(len(w) == 1) - assert_(issubclass(w[0].category, RuntimeWarning)) - # Check scalar - assert_(np.isnan(f(np.nan))) - assert_(len(w) == 2) - assert_(issubclass(w[0].category, RuntimeWarning)) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + match = "(Degrees of freedom <= 0 for slice.)|(Mean of empty slice)" + for func in self.nanfuncs: + with pytest.warns(RuntimeWarning, match=match): + out = func(array, axis=axis) + assert np.isnan(out).all() + + # `nanvar` and `nanstd` convert complex inputs to their + # corresponding floating dtype + if func is np.nanmean: + assert out.dtype == array.dtype + else: + assert out.dtype == np.abs(array).dtype def test_empty(self): mat = np.zeros((0, 3)) @@ -587,6 +754,30 @@ class TestNanFunctions_MeanVarStd(SharedNanFunctionsTestsMixin): assert_equal(f(mat, axis=axis), np.zeros([])) assert_(len(w) == 0) + @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"]) + def test_where(self, dtype): + ar = np.arange(9).reshape(3, 3).astype(dtype) + ar[0, :] = np.nan + where = np.ones_like(ar, dtype=np.bool_) + where[:, 0] = False + + for f, f_std in zip(self.nanfuncs, self.stdfuncs): + reference = f_std(ar[where][2:]) + dtype_reference = dtype if f is np.nanmean else ar.real.dtype + + ret = f(ar, where=where) + assert ret.dtype == dtype_reference + np.testing.assert_allclose(ret, reference) + + +_TIME_UNITS = ( + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns", "ps", "fs", "as" +) + +# All `inexact` + `timdelta64` type codes +_TYPE_CODES = list(np.typecodes["AllFloat"]) +_TYPE_CODES += [f"m8[{unit}]" for unit in _TIME_UNITS] + class TestNanFunctions_Median: @@ -623,6 +814,34 @@ class TestNanFunctions_Median: res = np.nanmedian(d, axis=(0, 1, 3), keepdims=True) assert_equal(res.shape, (1, 1, 7, 1)) + @pytest.mark.parametrize( + argnames='axis', + argvalues=[ + None, + 1, + (1, ), + (0, 1), + (-3, -1), + ] + ) + @pytest.mark.filterwarnings("ignore:All-NaN slice:RuntimeWarning") + def test_keepdims_out(self, axis): + d = np.ones((3, 5, 7, 11)) + # Randomly set some elements to NaN: + w = np.random.random((4, 200)) * np.array(d.shape)[:, None] + w = w.astype(np.intp) + d[tuple(w)] = np.nan + if axis is None: + shape_out = (1,) * d.ndim + else: + axis_norm = normalize_axis_tuple(axis, d.ndim) + shape_out = tuple( + 1 if i in axis_norm else d.shape[i] for i in range(d.ndim)) + out = np.empty(shape_out) + result = np.nanmedian(d, axis=axis, keepdims=True, out=out) + assert result is out + assert_equal(result.shape, shape_out) + def test_out(self): mat = np.random.rand(3, 3) nan_mat = np.insert(mat, [0, 2], np.nan, axis=1) @@ -662,23 +881,32 @@ class TestNanFunctions_Median: res = np.nanmedian(_ndat, axis=1) assert_almost_equal(res, tgt) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for axis in [None, 0, 1]: - with suppress_warnings() as sup: - sup.record(RuntimeWarning) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", _TYPE_CODES) + def test_allnans(self, dtype, axis): + mat = np.full((3, 3), np.nan).astype(dtype) + with suppress_warnings() as sup: + sup.record(RuntimeWarning) - assert_(np.isnan(np.nanmedian(mat, axis=axis)).all()) - if axis is None: - assert_(len(sup.log) == 1) - else: - assert_(len(sup.log) == 3) - # Check scalar - assert_(np.isnan(np.nanmedian(np.nan))) - if axis is None: - assert_(len(sup.log) == 2) - else: - assert_(len(sup.log) == 4) + output = np.nanmedian(mat, axis=axis) + assert output.dtype == mat.dtype + assert np.isnan(output).all() + + if axis is None: + assert_(len(sup.log) == 1) + else: + assert_(len(sup.log) == 3) + + # Check scalar + scalar = np.array(np.nan).astype(dtype)[()] + output_scalar = np.nanmedian(scalar) + assert output_scalar.dtype == scalar.dtype + assert np.isnan(output_scalar) + + if axis is None: + assert_(len(sup.log) == 2) + else: + assert_(len(sup.log) == 4) def test_empty(self): mat = np.zeros((0, 3)) @@ -789,6 +1017,37 @@ class TestNanFunctions_Percentile: res = np.nanpercentile(d, 90, axis=(0, 1, 3), keepdims=True) assert_equal(res.shape, (1, 1, 7, 1)) + @pytest.mark.parametrize('q', [7, [1, 7]]) + @pytest.mark.parametrize( + argnames='axis', + argvalues=[ + None, + 1, + (1,), + (0, 1), + (-3, -1), + ] + ) + @pytest.mark.filterwarnings("ignore:All-NaN slice:RuntimeWarning") + def test_keepdims_out(self, q, axis): + d = np.ones((3, 5, 7, 11)) + # Randomly set some elements to NaN: + w = np.random.random((4, 200)) * np.array(d.shape)[:, None] + w = w.astype(np.intp) + d[tuple(w)] = np.nan + if axis is None: + shape_out = (1,) * d.ndim + else: + axis_norm = normalize_axis_tuple(axis, d.ndim) + shape_out = tuple( + 1 if i in axis_norm else d.shape[i] for i in range(d.ndim)) + shape_out = np.shape(q) + shape_out + + out = np.empty(shape_out) + result = np.nanpercentile(d, q, axis=axis, keepdims=True, out=out) + assert result is out + assert_equal(result.shape, shape_out) + def test_out(self): mat = np.random.rand(3, 3) nan_mat = np.insert(mat, [0, 2], np.nan, axis=1) @@ -807,6 +1066,14 @@ class TestNanFunctions_Percentile: assert_almost_equal(res, resout) assert_almost_equal(res, tgt) + def test_complex(self): + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='G') + assert_raises(TypeError, np.nanpercentile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='D') + assert_raises(TypeError, np.nanpercentile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='F') + assert_raises(TypeError, np.nanpercentile, arr_c, 0.5) + def test_result_values(self): tgt = [np.percentile(d, 28) for d in _rdat] res = np.nanpercentile(_ndat, 28, axis=1) @@ -816,24 +1083,21 @@ class TestNanFunctions_Percentile: res = np.nanpercentile(_ndat, (28, 98), axis=1) assert_almost_equal(res, tgt) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for axis in [None, 0, 1]: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - assert_(np.isnan(np.nanpercentile(mat, 60, axis=axis)).all()) - if axis is None: - assert_(len(w) == 1) - else: - assert_(len(w) == 3) - assert_(issubclass(w[0].category, RuntimeWarning)) - # Check scalar - assert_(np.isnan(np.nanpercentile(np.nan, 60))) - if axis is None: - assert_(len(w) == 2) - else: - assert_(len(w) == 4) - assert_(issubclass(w[0].category, RuntimeWarning)) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["Float"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + with pytest.warns(RuntimeWarning, match="All-NaN slice encountered"): + out = np.nanpercentile(array, 60, axis=axis) + assert np.isnan(out).all() + assert out.dtype == array.dtype def test_empty(self): mat = np.zeros((0, 3)) @@ -914,18 +1178,42 @@ class TestNanFunctions_Quantile: assert_equal(np.nanquantile(x, 1), 3.5) assert_equal(np.nanquantile(x, 0.5), 1.75) + def test_complex(self): + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='G') + assert_raises(TypeError, np.nanquantile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='D') + assert_raises(TypeError, np.nanquantile, arr_c, 0.5) + arr_c = np.array([0.5+3.0j, 2.1+0.5j, 1.6+2.3j], dtype='F') + assert_raises(TypeError, np.nanquantile, arr_c, 0.5) + def test_no_p_overwrite(self): # this is worth retesting, because quantile does not make a copy p0 = np.array([0, 0.75, 0.25, 0.5, 1.0]) p = p0.copy() - np.nanquantile(np.arange(100.), p, interpolation="midpoint") + np.nanquantile(np.arange(100.), p, method="midpoint") assert_array_equal(p, p0) p0 = p0.tolist() p = p.tolist() - np.nanquantile(np.arange(100.), p, interpolation="midpoint") + np.nanquantile(np.arange(100.), p, method="midpoint") assert_array_equal(p, p0) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", np.typecodes["Float"]) + @pytest.mark.parametrize("array", [ + np.array(np.nan), + np.full((3, 3), np.nan), + ], ids=["0d", "2d"]) + def test_allnans(self, axis, dtype, array): + if axis is not None and array.ndim == 0: + pytest.skip(f"`axis != None` not supported for 0d arrays") + + array = array.astype(dtype) + with pytest.warns(RuntimeWarning, match="All-NaN slice encountered"): + out = np.nanquantile(array, 1, axis=axis) + assert np.isnan(out).all() + assert out.dtype == array.dtype + @pytest.mark.parametrize("arr, expected", [ # array of floats with some nans (np.array([np.nan, 5.0, np.nan, np.inf]), diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index cd0b90dc4..3734344d2 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -4,6 +4,12 @@ from numpy.testing import ( assert_array_almost_equal, assert_raises, assert_allclose ) +import pytest + +# `poly1d` has some support for `bool_` and `timedelta64`, +# but it is limited and they are therefore excluded here +TYPE_CODES = np.typecodes["AllInteger"] + np.typecodes["AllFloat"] + "O" + class TestPolynomial: def test_poly1d_str_and_repr(self): @@ -57,11 +63,26 @@ class TestPolynomial: assert_equal(np.polydiv(np.poly1d([1, 0, -1]), np.poly1d([1, 1])), (np.poly1d([1., -1.]), np.poly1d([0.]))) - def test_poly1d_misc(self): - p = np.poly1d([1., 2, 3]) - assert_equal(np.asarray(p), np.array([1., 2., 3.])) + @pytest.mark.parametrize("type_code", TYPE_CODES) + def test_poly1d_misc(self, type_code: str) -> None: + dtype = np.dtype(type_code) + ar = np.array([1, 2, 3], dtype=dtype) + p = np.poly1d(ar) + + # `__eq__` + assert_equal(np.asarray(p), ar) + assert_equal(np.asarray(p).dtype, dtype) assert_equal(len(p), 2) - assert_equal((p[0], p[1], p[2], p[3]), (3.0, 2.0, 1.0, 0)) + + # `__getitem__` + comparison_dct = {-1: 0, 0: 3, 1: 2, 2: 1, 3: 0} + for index, ref in comparison_dct.items(): + scalar = p[index] + assert_equal(scalar, ref) + if dtype == np.object_: + assert isinstance(scalar, int) + else: + assert_equal(scalar.dtype, dtype) def test_poly1d_variable_arg(self): q = np.poly1d([1., 2, 3], variable='y') @@ -227,6 +248,20 @@ class TestPolynomial: v = np.arange(1, 21) assert_almost_equal(np.poly(v), np.poly(np.diag(v))) + def test_zero_poly_dtype(self): + """ + Regression test for gh-16354. + """ + z = np.array([0, 0, 0]) + p = np.poly1d(z.astype(np.int64)) + assert_equal(p.coeffs.dtype, np.int64) + + p = np.poly1d(z.astype(np.float32)) + assert_equal(p.coeffs.dtype, np.float32) + + p = np.poly1d(z.astype(np.complex64)) + assert_equal(p.coeffs.dtype, np.complex64) + def test_poly_eq(self): p = np.poly1d([1, 2, 3]) p2 = np.poly1d([1, 2, 4]) @@ -244,6 +279,15 @@ class TestPolynomial: assert_equal(r.coeffs.dtype, np.complex128) assert_equal(q*a + r, b) + c = [1, 2, 3] + d = np.poly1d([1, 2, 3]) + s, t = np.polydiv(c, d) + assert isinstance(s, np.poly1d) + assert isinstance(t, np.poly1d) + u, v = np.polydiv(d, c) + assert isinstance(u, np.poly1d) + assert isinstance(v, np.poly1d) + def test_poly_coeffs_mutable(self): """ Coefficients should be modifiable """ p = np.poly1d([1, 2, 3]) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 2f3c14df3..0c919a53d 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -20,7 +20,7 @@ zip_dtype = np.lib.recfunctions._zip_dtype class TestRecFunctions: # Misc tests - def setup(self): + def setup_method(self): x = np.array([1, 2, ]) y = np.array([10, 20, 30]) z = np.array([('A', 1.), ('B', 2.)], @@ -318,6 +318,15 @@ class TestRecFunctions: assert_raises(NotImplementedError, unstructured_to_structured, np.zeros((3,0), dtype=np.int32)) + def test_unstructured_to_structured(self): + # test if dtype is the args of np.dtype + a = np.zeros((20, 2)) + test_dtype_args = [('x', float), ('y', float)] + test_dtype = np.dtype(test_dtype_args) + field1 = unstructured_to_structured(a, dtype=test_dtype_args) # now + field2 = unstructured_to_structured(a, dtype=test_dtype) # before + assert_equal(field1, field2) + def test_field_assignment_by_name(self): a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) newdt = [('b', 'f4'), ('c', 'u1')] @@ -372,7 +381,7 @@ class TestRecursiveFillFields: class TestMergeArrays: # Test merge_arrays - def setup(self): + def setup_method(self): x = np.array([1, 2, ]) y = np.array([10, 20, 30]) z = np.array( @@ -505,7 +514,7 @@ class TestMergeArrays: class TestAppendFields: # Test append_fields - def setup(self): + def setup_method(self): x = np.array([1, 2, ]) y = np.array([10, 20, 30]) z = np.array( @@ -558,7 +567,7 @@ class TestAppendFields: class TestStackArrays: # Test stack_arrays - def setup(self): + def setup_method(self): x = np.array([1, 2, ]) y = np.array([10, 20, 30]) z = np.array( @@ -728,7 +737,7 @@ class TestStackArrays: class TestJoinBy: - def setup(self): + def setup_method(self): self.a = np.array(list(zip(np.arange(10), np.arange(50, 60), np.arange(100, 110))), dtype=[('a', int), ('b', int), ('c', int)]) @@ -835,7 +844,6 @@ class TestJoinBy: b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')]) assert_raises(ValueError, join_by, ['a', 'b', 'b'], a, b) - @pytest.mark.xfail(reason="See comment at gh-9343") def test_same_name_different_dtypes_key(self): a_dtype = np.dtype([('key', 'S5'), ('value', '<f4')]) b_dtype = np.dtype([('key', 'S10'), ('value', '<f4')]) @@ -894,7 +902,7 @@ class TestJoinBy: class TestJoinBy2: @classmethod - def setup(cls): + def setup_method(cls): cls.a = np.array(list(zip(np.arange(10), np.arange(50, 60), np.arange(100, 110))), dtype=[('a', int), ('b', int), ('c', int)]) @@ -963,7 +971,7 @@ class TestAppendFieldsObj: """ # https://github.com/numpy/numpy/issues/2346 - def setup(self): + def setup_method(self): from datetime import date self.data = dict(obj=date(2000, 1, 1)) diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index fb7ba7874..eb6628904 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -392,7 +392,7 @@ class TestArraySplit: assert_(a.dtype.type is res[-1].dtype.type) # Same thing for manual splits: - res = array_split(a, [0, 1, 2], axis=0) + res = array_split(a, [0, 1], axis=0) tgt = [np.zeros((0, 10)), np.array([np.arange(10)]), np.array([np.arange(10)])] compare_results(res, tgt) @@ -492,7 +492,7 @@ class TestColumnStack: assert_equal(actual, expected) def test_generator(self): - with assert_warns(FutureWarning): + with pytest.raises(TypeError, match="arrays to stack must be"): column_stack((np.arange(3) for _ in range(2))) @@ -529,7 +529,7 @@ class TestDstack: assert_array_equal(res, desired) def test_generator(self): - with assert_warns(FutureWarning): + with pytest.raises(TypeError, match="arrays to stack must be"): dstack((np.arange(3) for _ in range(2))) @@ -644,17 +644,83 @@ class TestSqueeze: class TestKron: + def test_basic(self): + # Using 0-dimensional ndarray + a = np.array(1) + b = np.array([[1, 2], [3, 4]]) + k = np.array([[1, 2], [3, 4]]) + assert_array_equal(np.kron(a, b), k) + a = np.array([[1, 2], [3, 4]]) + b = np.array(1) + assert_array_equal(np.kron(a, b), k) + + # Using 1-dimensional ndarray + a = np.array([3]) + b = np.array([[1, 2], [3, 4]]) + k = np.array([[3, 6], [9, 12]]) + assert_array_equal(np.kron(a, b), k) + a = np.array([[1, 2], [3, 4]]) + b = np.array([3]) + assert_array_equal(np.kron(a, b), k) + + # Using 3-dimensional ndarray + a = np.array([[[1]], [[2]]]) + b = np.array([[1, 2], [3, 4]]) + k = np.array([[[1, 2], [3, 4]], [[2, 4], [6, 8]]]) + assert_array_equal(np.kron(a, b), k) + a = np.array([[1, 2], [3, 4]]) + b = np.array([[[1]], [[2]]]) + k = np.array([[[1, 2], [3, 4]], [[2, 4], [6, 8]]]) + assert_array_equal(np.kron(a, b), k) + def test_return_type(self): class myarray(np.ndarray): - __array_priority__ = 0.0 + __array_priority__ = 1.0 a = np.ones([2, 2]) ma = myarray(a.shape, a.dtype, a.data) assert_equal(type(kron(a, a)), np.ndarray) assert_equal(type(kron(ma, ma)), myarray) - assert_equal(type(kron(a, ma)), np.ndarray) + assert_equal(type(kron(a, ma)), myarray) assert_equal(type(kron(ma, a)), myarray) + @pytest.mark.parametrize( + "array_class", [np.asarray, np.mat] + ) + def test_kron_smoke(self, array_class): + a = array_class(np.ones([3, 3])) + b = array_class(np.ones([3, 3])) + k = array_class(np.ones([9, 9])) + + assert_array_equal(np.kron(a, b), k) + + def test_kron_ma(self): + x = np.ma.array([[1, 2], [3, 4]], mask=[[0, 1], [1, 0]]) + k = np.ma.array(np.diag([1, 4, 4, 16]), + mask=~np.array(np.identity(4), dtype=bool)) + + assert_array_equal(k, np.kron(x, x)) + + @pytest.mark.parametrize( + "shape_a,shape_b", [ + ((1, 1), (1, 1)), + ((1, 2, 3), (4, 5, 6)), + ((2, 2), (2, 2, 2)), + ((1, 0), (1, 1)), + ((2, 0, 2), (2, 2)), + ((2, 0, 0, 2), (2, 0, 2)), + ]) + def test_kron_shape(self, shape_a, shape_b): + a = np.ones(shape_a) + b = np.ones(shape_b) + normalised_shape_a = (1,) * max(0, len(shape_b)-len(shape_a)) + shape_a + normalised_shape_b = (1,) * max(0, len(shape_a)-len(shape_b)) + shape_b + expected_shape = np.multiply(normalised_shape_a, normalised_shape_b) + + k = np.kron(a, b) + assert np.array_equal( + k.shape, expected_shape), "Unexpected shape from kron" + class TestTile: def test_basic(self): @@ -713,5 +779,9 @@ class TestMayShareMemory: # Utility def compare_results(res, desired): - for i in range(len(desired)): - assert_array_equal(res[i], desired[i]) + """Compare lists of arrays.""" + if len(res) != len(desired): + raise ValueError("Iterables have different lengths") + # See also PEP 618 for Python 3.10 + for x, y in zip(res, desired): + assert_array_equal(x, y) diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index 9d95eb9d0..efec5d24d 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -5,8 +5,11 @@ from numpy.testing import ( assert_raises_regex, assert_warns, ) from numpy.lib.stride_tricks import ( - as_strided, broadcast_arrays, _broadcast_shape, broadcast_to + as_strided, broadcast_arrays, _broadcast_shape, broadcast_to, + broadcast_shapes, sliding_window_view, ) +import pytest + def assert_shapes_correct(input_shapes, expected_shape): # Broadcast a list of arrays with the given input shapes and check the @@ -274,7 +277,9 @@ def test_broadcast_to_raises(): def test_broadcast_shape(): - # broadcast_shape is already exercized indirectly by broadcast_arrays + # tests internal _broadcast_shape + # _broadcast_shape is already exercised indirectly by broadcast_arrays + # _broadcast_shape is also exercised by the public broadcast_shapes function assert_equal(_broadcast_shape(), ()) assert_equal(_broadcast_shape([1, 2]), (2,)) assert_equal(_broadcast_shape(np.ones((1, 1))), (1, 1)) @@ -288,6 +293,64 @@ def test_broadcast_shape(): assert_raises(ValueError, lambda: _broadcast_shape(*bad_args)) +def test_broadcast_shapes_succeeds(): + # tests public broadcast_shapes + data = [ + [[], ()], + [[()], ()], + [[(7,)], (7,)], + [[(1, 2), (2,)], (1, 2)], + [[(1, 1)], (1, 1)], + [[(1, 1), (3, 4)], (3, 4)], + [[(6, 7), (5, 6, 1), (7,), (5, 1, 7)], (5, 6, 7)], + [[(5, 6, 1)], (5, 6, 1)], + [[(1, 3), (3, 1)], (3, 3)], + [[(1, 0), (0, 0)], (0, 0)], + [[(0, 1), (0, 0)], (0, 0)], + [[(1, 0), (0, 1)], (0, 0)], + [[(1, 1), (0, 0)], (0, 0)], + [[(1, 1), (1, 0)], (1, 0)], + [[(1, 1), (0, 1)], (0, 1)], + [[(), (0,)], (0,)], + [[(0,), (0, 0)], (0, 0)], + [[(0,), (0, 1)], (0, 0)], + [[(1,), (0, 0)], (0, 0)], + [[(), (0, 0)], (0, 0)], + [[(1, 1), (0,)], (1, 0)], + [[(1,), (0, 1)], (0, 1)], + [[(1,), (1, 0)], (1, 0)], + [[(), (1, 0)], (1, 0)], + [[(), (0, 1)], (0, 1)], + [[(1,), (3,)], (3,)], + [[2, (3, 2)], (3, 2)], + ] + for input_shapes, target_shape in data: + assert_equal(broadcast_shapes(*input_shapes), target_shape) + + assert_equal(broadcast_shapes(*([(1, 2)] * 32)), (1, 2)) + assert_equal(broadcast_shapes(*([(1, 2)] * 100)), (1, 2)) + + # regression tests for gh-5862 + assert_equal(broadcast_shapes(*([(2,)] * 32)), (2,)) + + +def test_broadcast_shapes_raises(): + # tests public broadcast_shapes + data = [ + [(3,), (4,)], + [(2, 3), (2,)], + [(3,), (3,), (4,)], + [(1, 3, 4), (2, 3, 3)], + [(1, 2), (3,1), (3,2), (10, 5)], + [2, (2, 3)], + ] + for input_shapes in data: + assert_raises(ValueError, lambda: broadcast_shapes(*input_shapes)) + + bad_args = [(2,)] * 32 + [(3,)] * 32 + assert_raises(ValueError, lambda: broadcast_shapes(*bad_args)) + + def test_as_strided(): a = np.array([None]) a_view = as_strided(a) @@ -333,6 +396,109 @@ def test_as_strided(): assert_equal(a.dtype, a_view.dtype) assert_array_equal([r] * 3, a_view) + +class TestSlidingWindowView: + def test_1d(self): + arr = np.arange(5) + arr_view = sliding_window_view(arr, 2) + expected = np.array([[0, 1], + [1, 2], + [2, 3], + [3, 4]]) + assert_array_equal(arr_view, expected) + + def test_2d(self): + i, j = np.ogrid[:3, :4] + arr = 10*i + j + shape = (2, 2) + arr_view = sliding_window_view(arr, shape) + expected = np.array([[[[0, 1], [10, 11]], + [[1, 2], [11, 12]], + [[2, 3], [12, 13]]], + [[[10, 11], [20, 21]], + [[11, 12], [21, 22]], + [[12, 13], [22, 23]]]]) + assert_array_equal(arr_view, expected) + + def test_2d_with_axis(self): + i, j = np.ogrid[:3, :4] + arr = 10*i + j + arr_view = sliding_window_view(arr, 3, 0) + expected = np.array([[[0, 10, 20], + [1, 11, 21], + [2, 12, 22], + [3, 13, 23]]]) + assert_array_equal(arr_view, expected) + + def test_2d_repeated_axis(self): + i, j = np.ogrid[:3, :4] + arr = 10*i + j + arr_view = sliding_window_view(arr, (2, 3), (1, 1)) + expected = np.array([[[[0, 1, 2], + [1, 2, 3]]], + [[[10, 11, 12], + [11, 12, 13]]], + [[[20, 21, 22], + [21, 22, 23]]]]) + assert_array_equal(arr_view, expected) + + def test_2d_without_axis(self): + i, j = np.ogrid[:4, :4] + arr = 10*i + j + shape = (2, 3) + arr_view = sliding_window_view(arr, shape) + expected = np.array([[[[0, 1, 2], [10, 11, 12]], + [[1, 2, 3], [11, 12, 13]]], + [[[10, 11, 12], [20, 21, 22]], + [[11, 12, 13], [21, 22, 23]]], + [[[20, 21, 22], [30, 31, 32]], + [[21, 22, 23], [31, 32, 33]]]]) + assert_array_equal(arr_view, expected) + + def test_errors(self): + i, j = np.ogrid[:4, :4] + arr = 10*i + j + with pytest.raises(ValueError, match='cannot contain negative values'): + sliding_window_view(arr, (-1, 3)) + with pytest.raises( + ValueError, + match='must provide window_shape for all dimensions of `x`'): + sliding_window_view(arr, (1,)) + with pytest.raises( + ValueError, + match='Must provide matching length window_shape and axis'): + sliding_window_view(arr, (1, 3, 4), axis=(0, 1)) + with pytest.raises( + ValueError, + match='window shape cannot be larger than input array'): + sliding_window_view(arr, (5, 5)) + + def test_writeable(self): + arr = np.arange(5) + view = sliding_window_view(arr, 2, writeable=False) + assert_(not view.flags.writeable) + with pytest.raises( + ValueError, + match='assignment destination is read-only'): + view[0, 0] = 3 + view = sliding_window_view(arr, 2, writeable=True) + assert_(view.flags.writeable) + view[0, 1] = 3 + assert_array_equal(arr, np.array([0, 3, 2, 3, 4])) + + def test_subok(self): + class MyArray(np.ndarray): + pass + + arr = np.arange(5).view(MyArray) + assert_(not isinstance(sliding_window_view(arr, 2, + subok=False), + MyArray)) + assert_(isinstance(sliding_window_view(arr, 2, subok=True), MyArray)) + # Default behavior + assert_(not isinstance(sliding_window_view(arr, 2), MyArray)) + + def as_strided_writeable(): arr = np.ones(10) view = as_strided(arr, writeable=False) @@ -435,7 +601,7 @@ def test_writeable(): # check: no warning emitted assert_equal(result.flags.writeable, True) result[:] = 0 - + # keep readonly input readonly original.flags.writeable = False _, result = broadcast_arrays(0, original) diff --git a/numpy/lib/tests/test_twodim_base.py b/numpy/lib/tests/test_twodim_base.py index cce683bfe..eb008c600 100644 --- a/numpy/lib/tests/test_twodim_base.py +++ b/numpy/lib/tests/test_twodim_base.py @@ -4,18 +4,15 @@ from numpy.testing import ( assert_equal, assert_array_equal, assert_array_max_ulp, assert_array_almost_equal, assert_raises, assert_ - ) - +) from numpy import ( arange, add, fliplr, flipud, zeros, ones, eye, array, diag, histogram2d, tri, mask_indices, triu_indices, triu_indices_from, tril_indices, tril_indices_from, vander, - ) - +) import numpy as np - -from numpy.core.tests.test_overrides import requires_array_function +import pytest def get_mat(n): @@ -41,6 +38,12 @@ class TestEye: assert_equal(eye(3) == 1, eye(3, dtype=bool)) + def test_uint64(self): + # Regression test for gh-9982 + assert_equal(eye(np.uint64(2), dtype=int), array([[1, 0], [0, 1]])) + assert_equal(eye(np.uint64(2), M=np.uint64(4), k=np.uint64(1)), + array([[0, 1, 0, 0], [0, 0, 1, 0]])) + def test_diag(self): assert_equal(eye(4, k=1), array([[0, 1, 0, 0], @@ -274,7 +277,6 @@ class TestHistogram2d: assert_array_equal(H, answer) assert_array_equal(xe, array([0., 0.25, 0.5, 0.75, 1])) - @requires_array_function def test_dispatch(self): class ShouldDispatch: def __array_function__(self, function, types, args, kwargs): @@ -295,6 +297,13 @@ class TestHistogram2d: r = histogram2d(xy, xy, weights=s_d) assert_(r, ((ShouldDispatch,), (xy, xy), dict(weights=s_d))) + @pytest.mark.parametrize(("x_len", "y_len"), [(10, 11), (20, 19)]) + def test_bad_length(self, x_len, y_len): + x, y = np.ones(x_len), np.ones(y_len) + with pytest.raises(ValueError, + match='x and y must have the same length.'): + histogram2d(x, y) + class TestTri: def test_dtype(self): @@ -372,7 +381,7 @@ def test_tril_triu_dtype(): assert_equal(np.triu(arr).dtype, arr.dtype) assert_equal(np.tril(arr).dtype, arr.dtype) - arr = np.zeros((3,3), dtype='f4,f4') + arr = np.zeros((3, 3), dtype='f4,f4') assert_equal(np.triu(arr).dtype, arr.dtype) assert_equal(np.tril(arr).dtype, arr.dtype) diff --git a/numpy/lib/tests/test_type_check.py b/numpy/lib/tests/test_type_check.py index 3f4ca6309..ea0326139 100644 --- a/numpy/lib/tests/test_type_check.py +++ b/numpy/lib/tests/test_type_check.py @@ -155,7 +155,7 @@ class TestIscomplex: def test_fail(self): z = np.array([-1, 0, 1]) res = iscomplex(z) - assert_(not np.sometrue(res, axis=0)) + assert_(not np.any(res, axis=0)) def test_pass(self): z = np.array([-1j, 1, 0]) diff --git a/numpy/lib/tests/test_ufunclike.py b/numpy/lib/tests/test_ufunclike.py index c280b6969..fac4f41d0 100644 --- a/numpy/lib/tests/test_ufunclike.py +++ b/numpy/lib/tests/test_ufunclike.py @@ -80,12 +80,6 @@ class TestUfunclike: assert_(isinstance(f0d, MyArray)) assert_equal(f0d.metadata, 'bar') - def test_deprecated(self): - # NumPy 1.13.0, 2017-04-26 - assert_warns(DeprecationWarning, ufl.fix, [1, 2], y=nx.empty(2)) - assert_warns(DeprecationWarning, ufl.isposinf, [1, 2], y=nx.empty(2)) - assert_warns(DeprecationWarning, ufl.isneginf, [1, 2], y=nx.empty(2)) - def test_scalar(self): x = np.inf actual = np.isposinf(x) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 261cfef5d..6ad4bfe6d 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -4,13 +4,17 @@ import pytest from numpy.core import arange from numpy.testing import assert_, assert_equal, assert_raises_regex -from numpy.lib import deprecate +from numpy.lib import deprecate, deprecate_with_doc import numpy.lib.utils as utils from io import StringIO @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") +@pytest.mark.skipif( + sys.version_info == (3, 10, 0, "candidate", 1), + reason="Broken as of bpo-44524", +) def test_lookfor(): out = StringIO() utils.lookfor('eigenvalue', module='numpy', output=out, @@ -60,6 +64,11 @@ def old_func6(self, x): new_func6 = deprecate(old_func6) +@deprecate_with_doc(msg="Rather use new_func7") +def old_func7(self,x): + return x + + def test_deprecate_decorator(): assert_('deprecated' in old_func.__doc__) @@ -73,6 +82,10 @@ def test_deprecate_fn(): assert_('new_func3' in new_func3.__doc__) +def test_deprecate_with_doc_decorator_message(): + assert_('Rather use new_func7' in old_func7.__doc__) + + @pytest.mark.skipif(sys.flags.optimize == 2, reason="-OO discards docstrings") @pytest.mark.parametrize('old_func, new_func', [ (old_func4, new_func4), @@ -102,6 +115,10 @@ def test_deprecate_preserve_whitespace(): assert_('\n Bizarre' in new_func5.__doc__) +def test_deprecate_module(): + assert_(old_func.__module__ == __name__) + + def test_safe_eval_nameconstant(): # Test if safe_eval supports Python 3.4 _ast.NameConstant utils.safe_eval('None') @@ -140,3 +157,22 @@ class TestByteBounds: def test_assert_raises_regex_context_manager(): with assert_raises_regex(ValueError, 'no deprecation warning'): raise ValueError('no deprecation warning') + + +def test_info_method_heading(): + # info(class) should only print "Methods:" heading if methods exist + + class NoPublicMethods: + pass + + class WithPublicMethods: + def first_method(): + pass + + def _has_method_heading(cls): + out = StringIO() + utils.info(cls, output=out) + return 'Methods:' in out.getvalue() + + assert _has_method_heading(WithPublicMethods) + assert not _has_method_heading(NoPublicMethods) diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index cd7484241..6dcb65651 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -2,15 +2,17 @@ """ import functools +import operator from numpy.core.numeric import ( asanyarray, arange, zeros, greater_equal, multiply, ones, - asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal, - nonzero + asarray, where, int8, int16, int32, int64, intp, empty, promote_types, + diagonal, nonzero, indices ) -from numpy.core.overrides import set_module +from numpy.core.overrides import set_array_function_like_doc, set_module from numpy.core import overrides from numpy.core import iinfo +from numpy.lib.stride_tricks import broadcast_to __all__ = [ @@ -46,10 +48,11 @@ def _flip_dispatcher(m): @array_function_dispatch(_flip_dispatcher) def fliplr(m): """ - Flip array in the left/right direction. + Reverse the order of elements along axis 1 (left/right). - Flip the entries in each row in the left/right direction. - Columns are preserved, but appear in a different order than before. + For a 2-D array, this flips the entries in each row in the left/right + direction. Columns are preserved, but appear in a different order than + before. Parameters ---------- @@ -65,11 +68,13 @@ def fliplr(m): See Also -------- flipud : Flip array in the up/down direction. + flip : Flip array in one or more dimensions. rot90 : Rotate array counterclockwise. Notes ----- - Equivalent to m[:,::-1]. Requires the array to be at least 2-D. + Equivalent to ``m[:,::-1]`` or ``np.flip(m, axis=1)``. + Requires the array to be at least 2-D. Examples -------- @@ -97,10 +102,10 @@ def fliplr(m): @array_function_dispatch(_flip_dispatcher) def flipud(m): """ - Flip array in the up/down direction. + Reverse the order of elements along axis 0 (up/down). - Flip the entries in each column in the up/down direction. - Rows are preserved, but appear in a different order than before. + For a 2-D array, this flips the entries in each column in the up/down + direction. Rows are preserved, but appear in a different order than before. Parameters ---------- @@ -116,12 +121,13 @@ def flipud(m): See Also -------- fliplr : Flip array in the left/right direction. + flip : Flip array in one or more dimensions. rot90 : Rotate array counterclockwise. Notes ----- - Equivalent to ``m[::-1,...]``. - Does not require the array to be two-dimensional. + Equivalent to ``m[::-1, ...]`` or ``np.flip(m, axis=0)``. + Requires the array to be at least 1-D. Examples -------- @@ -149,8 +155,9 @@ def flipud(m): return m[::-1, ...] +@set_array_function_like_doc @set_module('numpy') -def eye(N, M=None, k=0, dtype=float, order='C'): +def eye(N, M=None, k=0, dtype=float, order='C', *, like=None): """ Return a 2-D array with ones on the diagonal and zeros elsewhere. @@ -171,6 +178,9 @@ def eye(N, M=None, k=0, dtype=float, order='C'): column-major (Fortran-style) order in memory. .. versionadded:: 1.14.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -194,11 +204,18 @@ def eye(N, M=None, k=0, dtype=float, order='C'): [0., 0., 0.]]) """ + if like is not None: + return _eye_with_like(like, N, M=M, k=k, dtype=dtype, order=order) if M is None: M = N m = zeros((N, M), dtype=dtype, order=order) if k >= M: return m + # Ensure M and k are integers, so we don't get any surprise casting + # results in the expressions `M-k` and `M+1` used below. This avoids + # a problem with inputs with type (for example) np.uint64. + M = operator.index(M) + k = operator.index(k) if k >= 0: i = k else: @@ -207,6 +224,9 @@ def eye(N, M=None, k=0, dtype=float, order='C'): return m +_eye_with_like = array_function_dispatch()(eye) + + def _diag_dispatcher(v, k=None): return (v,) @@ -332,10 +352,10 @@ def diagflat(v, k=0): n = s + abs(k) res = zeros((n, n), v.dtype) if (k >= 0): - i = arange(0, n-k) + i = arange(0, n-k, dtype=intp) fi = i+k+i*n else: - i = arange(0, n+k) + i = arange(0, n+k, dtype=intp) fi = i+(i-k)*n res.flat[fi] = v if not wrap: @@ -343,8 +363,9 @@ def diagflat(v, k=0): return wrap(res) +@set_array_function_like_doc @set_module('numpy') -def tri(N, M=None, k=0, dtype=float): +def tri(N, M=None, k=0, dtype=float, *, like=None): """ An array with ones at and below the given diagonal and zeros elsewhere. @@ -361,6 +382,9 @@ def tri(N, M=None, k=0, dtype=float): and `k` > 0 is above. The default is 0. dtype : dtype, optional Data type of the returned array. The default is float. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -381,6 +405,9 @@ def tri(N, M=None, k=0, dtype=float): [1., 1., 0., 0., 0.]]) """ + if like is not None: + return _tri_with_like(like, N, M=M, k=k, dtype=dtype) + if M is None: M = N @@ -393,6 +420,9 @@ def tri(N, M=None, k=0, dtype=float): return m +_tri_with_like = array_function_dispatch()(tri) + + def _trilu_dispatcher(m, k=None): return (m,) @@ -403,10 +433,12 @@ def tril(m, k=0): Lower triangle of an array. Return a copy of an array with elements above the `k`-th diagonal zeroed. + For arrays with ``ndim`` exceeding 2, `tril` will apply to the final two + axes. Parameters ---------- - m : array_like, shape (M, N) + m : array_like, shape (..., M, N) Input array. k : int, optional Diagonal above which to zero elements. `k = 0` (the default) is the @@ -414,7 +446,7 @@ def tril(m, k=0): Returns ------- - tril : ndarray, shape (M, N) + tril : ndarray, shape (..., M, N) Lower triangle of `m`, of same shape and data-type as `m`. See Also @@ -429,6 +461,20 @@ def tril(m, k=0): [ 7, 8, 0], [10, 11, 12]]) + >>> np.tril(np.arange(3*4*5).reshape(3, 4, 5)) + array([[[ 0, 0, 0, 0, 0], + [ 5, 6, 0, 0, 0], + [10, 11, 12, 0, 0], + [15, 16, 17, 18, 0]], + [[20, 0, 0, 0, 0], + [25, 26, 0, 0, 0], + [30, 31, 32, 0, 0], + [35, 36, 37, 38, 0]], + [[40, 0, 0, 0, 0], + [45, 46, 0, 0, 0], + [50, 51, 52, 0, 0], + [55, 56, 57, 58, 0]]]) + """ m = asanyarray(m) mask = tri(*m.shape[-2:], k=k, dtype=bool) @@ -442,7 +488,8 @@ def triu(m, k=0): Upper triangle of an array. Return a copy of an array with the elements below the `k`-th diagonal - zeroed. + zeroed. For arrays with ``ndim`` exceeding 2, `triu` will apply to the + final two axes. Please refer to the documentation for `tril` for further details. @@ -458,6 +505,20 @@ def triu(m, k=0): [ 0, 8, 9], [ 0, 0, 12]]) + >>> np.triu(np.arange(3*4*5).reshape(3, 4, 5)) + array([[[ 0, 1, 2, 3, 4], + [ 0, 6, 7, 8, 9], + [ 0, 0, 12, 13, 14], + [ 0, 0, 0, 18, 19]], + [[20, 21, 22, 23, 24], + [ 0, 26, 27, 28, 29], + [ 0, 0, 32, 33, 34], + [ 0, 0, 0, 38, 39]], + [[40, 41, 42, 43, 44], + [ 0, 46, 47, 48, 49], + [ 0, 0, 52, 53, 54], + [ 0, 0, 0, 58, 59]]]) + """ m = asanyarray(m) mask = tri(*m.shape[-2:], k=k-1, dtype=bool) @@ -561,8 +622,8 @@ def vander(x, N=None, increasing=False): return v -def _histogram2d_dispatcher(x, y, bins=None, range=None, normed=None, - weights=None, density=None): +def _histogram2d_dispatcher(x, y, bins=None, range=None, density=None, + weights=None): yield x yield y @@ -580,8 +641,7 @@ def _histogram2d_dispatcher(x, y, bins=None, range=None, normed=None, @array_function_dispatch(_histogram2d_dispatcher) -def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, - density=None): +def histogram2d(x, y, bins=10, range=None, density=None, weights=None): """ Compute the bi-dimensional histogram of two data samples. @@ -615,13 +675,9 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, If False, the default, returns the number of samples in each bin. If True, returns the probability *density* function at the bin, ``bin_count / sample_count / bin_area``. - normed : bool, optional - An alias for the density argument that behaves identically. To avoid - confusion with the broken normed argument to `histogram`, `density` - should be preferred. weights : array_like, shape(N,), optional An array of values ``w_i`` weighing each sample ``(x_i, y_i)``. - Weights are normalized to 1 if `normed` is True. If `normed` is + Weights are normalized to 1 if `density` is True. If `density` is False, the values of the returned histogram are equal to the sum of the weights belonging to the samples falling into each bin. @@ -643,7 +699,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, Notes ----- - When `normed` is True, then the returned histogram is the sample + When `density` is True, then the returned histogram is the sample density, defined such that the sum over bins of the product ``bin_value * bin_area`` is 1. @@ -669,7 +725,9 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, >>> x = np.random.normal(2, 1, 100) >>> y = np.random.normal(1, 1, 100) >>> H, xedges, yedges = np.histogram2d(x, y, bins=(xedges, yedges)) - >>> H = H.T # Let each row list bins with common y range. + >>> # Histogram does not follow Cartesian convention (see Notes), + >>> # therefore transpose H for visualization purposes. + >>> H = H.T :func:`imshow <matplotlib.pyplot.imshow>` can only display square bins: @@ -696,12 +754,49 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, >>> xcenters = (xedges[:-1] + xedges[1:]) / 2 >>> ycenters = (yedges[:-1] + yedges[1:]) / 2 >>> im.set_data(xcenters, ycenters, H) - >>> ax.images.append(im) + >>> ax.add_image(im) >>> plt.show() + It is also possible to construct a 2-D histogram without specifying bin + edges: + + >>> # Generate non-symmetric test data + >>> n = 10000 + >>> x = np.linspace(1, 100, n) + >>> y = 2*np.log(x) + np.random.rand(n) - 0.5 + >>> # Compute 2d histogram. Note the order of x/y and xedges/yedges + >>> H, yedges, xedges = np.histogram2d(y, x, bins=20) + + Now we can plot the histogram using + :func:`pcolormesh <matplotlib.pyplot.pcolormesh>`, and a + :func:`hexbin <matplotlib.pyplot.hexbin>` for comparison. + + >>> # Plot histogram using pcolormesh + >>> fig, (ax1, ax2) = plt.subplots(ncols=2, sharey=True) + >>> ax1.pcolormesh(xedges, yedges, H, cmap='rainbow') + >>> ax1.plot(x, 2*np.log(x), 'k-') + >>> ax1.set_xlim(x.min(), x.max()) + >>> ax1.set_ylim(y.min(), y.max()) + >>> ax1.set_xlabel('x') + >>> ax1.set_ylabel('y') + >>> ax1.set_title('histogram2d') + >>> ax1.grid() + + >>> # Create hexbin plot for comparison + >>> ax2.hexbin(x, y, gridsize=20, cmap='rainbow') + >>> ax2.plot(x, 2*np.log(x), 'k-') + >>> ax2.set_title('hexbin') + >>> ax2.set_xlim(x.min(), x.max()) + >>> ax2.set_xlabel('x') + >>> ax2.grid() + + >>> plt.show() """ from numpy import histogramdd + if len(x) != len(y): + raise ValueError('x and y must have the same length.') + try: N = len(bins) except TypeError: @@ -710,7 +805,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, if N != 1 and N != 2: xedges = yedges = asarray(bins) bins = [xedges, yedges] - hist, edges = histogramdd([x, y], bins, range, normed, weights, density) + hist, edges = histogramdd([x, y], bins, range, density, weights) return hist, edges[0], edges[1] @@ -863,7 +958,10 @@ def tril_indices(n, k=0, m=None): [-10, -10, -10, -10]]) """ - return nonzero(tri(n, m, k=k, dtype=bool)) + tri_ = tri(n, m, k=k, dtype=bool) + + return tuple(broadcast_to(inds, tri_.shape)[tri_] + for inds in indices(tri_.shape, sparse=True)) def _trilu_indices_form_dispatcher(arr, k=None): @@ -885,9 +983,42 @@ def tril_indices_from(arr, k=0): k : int, optional Diagonal offset (see `tril` for details). + Examples + -------- + + Create a 4 by 4 array. + + >>> a = np.arange(16).reshape(4, 4) + >>> a + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]) + + Pass the array to get the indices of the lower triangular elements. + + >>> trili = np.tril_indices_from(a) + >>> trili + (array([0, 1, 1, 2, 2, 2, 3, 3, 3, 3]), array([0, 0, 1, 0, 1, 2, 0, 1, 2, 3])) + + >>> a[trili] + array([ 0, 4, 5, 8, 9, 10, 12, 13, 14, 15]) + + This is syntactic sugar for tril_indices(). + + >>> np.tril_indices(a.shape[0]) + (array([0, 1, 1, 2, 2, 2, 3, 3, 3, 3]), array([0, 0, 1, 0, 1, 2, 0, 1, 2, 3])) + + Use the `k` parameter to return the indices for the lower triangular array + up to the k-th diagonal. + + >>> trili1 = np.tril_indices_from(a, k=1) + >>> a[trili1] + array([ 0, 1, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15]) + See Also -------- - tril_indices, tril + tril_indices, tril, triu_indices_from Notes ----- @@ -979,7 +1110,10 @@ def triu_indices(n, k=0, m=None): [ 12, 13, 14, -1]]) """ - return nonzero(~tri(n, m, k=k-1, dtype=bool)) + tri_ = ~tri(n, m, k=k - 1, dtype=bool) + + return tuple(broadcast_to(inds, tri_.shape)[tri_] + for inds in indices(tri_.shape, sparse=True)) @array_function_dispatch(_trilu_indices_form_dispatcher) @@ -1001,9 +1135,43 @@ def triu_indices_from(arr, k=0): triu_indices_from : tuple, shape(2) of ndarray, shape(N) Indices for the upper-triangle of `arr`. + Examples + -------- + + Create a 4 by 4 array. + + >>> a = np.arange(16).reshape(4, 4) + >>> a + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]) + + Pass the array to get the indices of the upper triangular elements. + + >>> triui = np.triu_indices_from(a) + >>> triui + (array([0, 0, 0, 0, 1, 1, 1, 2, 2, 3]), array([0, 1, 2, 3, 1, 2, 3, 2, 3, 3])) + + >>> a[triui] + array([ 0, 1, 2, 3, 5, 6, 7, 10, 11, 15]) + + This is syntactic sugar for triu_indices(). + + >>> np.triu_indices(a.shape[0]) + (array([0, 0, 0, 0, 1, 1, 1, 2, 2, 3]), array([0, 1, 2, 3, 1, 2, 3, 2, 3, 3])) + + Use the `k` parameter to return the indices for the upper triangular array + from the k-th diagonal. + + >>> triuim1 = np.triu_indices_from(a, k=1) + >>> a[triuim1] + array([ 1, 2, 3, 6, 7, 11]) + + See Also -------- - triu_indices, triu + triu_indices, triu, tril_indices_from Notes ----- diff --git a/numpy/lib/twodim_base.pyi b/numpy/lib/twodim_base.pyi new file mode 100644 index 000000000..1b3b94bd5 --- /dev/null +++ b/numpy/lib/twodim_base.pyi @@ -0,0 +1,239 @@ +from collections.abc import Callable, Sequence +from typing import ( + Any, + overload, + TypeVar, + Union, +) + +from numpy import ( + generic, + number, + bool_, + timedelta64, + datetime64, + int_, + intp, + float64, + signedinteger, + floating, + complexfloating, + object_, + _OrderCF, +) + +from numpy._typing import ( + DTypeLike, + _DTypeLike, + ArrayLike, + _ArrayLike, + NDArray, + _SupportsArrayFunc, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeObject_co, +) + +_T = TypeVar("_T") +_SCT = TypeVar("_SCT", bound=generic) + +# The returned arrays dtype must be compatible with `np.equal` +_MaskFunc = Callable[ + [NDArray[int_], _T], + NDArray[Union[number[Any], bool_, timedelta64, datetime64, object_]], +] + +__all__: list[str] + +@overload +def fliplr(m: _ArrayLike[_SCT]) -> NDArray[_SCT]: ... +@overload +def fliplr(m: ArrayLike) -> NDArray[Any]: ... + +@overload +def flipud(m: _ArrayLike[_SCT]) -> NDArray[_SCT]: ... +@overload +def flipud(m: ArrayLike) -> NDArray[Any]: ... + +@overload +def eye( + N: int, + M: None | int = ..., + k: int = ..., + dtype: None = ..., + order: _OrderCF = ..., + *, + like: None | _SupportsArrayFunc = ..., +) -> NDArray[float64]: ... +@overload +def eye( + N: int, + M: None | int = ..., + k: int = ..., + dtype: _DTypeLike[_SCT] = ..., + order: _OrderCF = ..., + *, + like: None | _SupportsArrayFunc = ..., +) -> NDArray[_SCT]: ... +@overload +def eye( + N: int, + M: None | int = ..., + k: int = ..., + dtype: DTypeLike = ..., + order: _OrderCF = ..., + *, + like: None | _SupportsArrayFunc = ..., +) -> NDArray[Any]: ... + +@overload +def diag(v: _ArrayLike[_SCT], k: int = ...) -> NDArray[_SCT]: ... +@overload +def diag(v: ArrayLike, k: int = ...) -> NDArray[Any]: ... + +@overload +def diagflat(v: _ArrayLike[_SCT], k: int = ...) -> NDArray[_SCT]: ... +@overload +def diagflat(v: ArrayLike, k: int = ...) -> NDArray[Any]: ... + +@overload +def tri( + N: int, + M: None | int = ..., + k: int = ..., + dtype: None = ..., + *, + like: None | _SupportsArrayFunc = ... +) -> NDArray[float64]: ... +@overload +def tri( + N: int, + M: None | int = ..., + k: int = ..., + dtype: _DTypeLike[_SCT] = ..., + *, + like: None | _SupportsArrayFunc = ... +) -> NDArray[_SCT]: ... +@overload +def tri( + N: int, + M: None | int = ..., + k: int = ..., + dtype: DTypeLike = ..., + *, + like: None | _SupportsArrayFunc = ... +) -> NDArray[Any]: ... + +@overload +def tril(v: _ArrayLike[_SCT], k: int = ...) -> NDArray[_SCT]: ... +@overload +def tril(v: ArrayLike, k: int = ...) -> NDArray[Any]: ... + +@overload +def triu(v: _ArrayLike[_SCT], k: int = ...) -> NDArray[_SCT]: ... +@overload +def triu(v: ArrayLike, k: int = ...) -> NDArray[Any]: ... + +@overload +def vander( # type: ignore[misc] + x: _ArrayLikeInt_co, + N: None | int = ..., + increasing: bool = ..., +) -> NDArray[signedinteger[Any]]: ... +@overload +def vander( # type: ignore[misc] + x: _ArrayLikeFloat_co, + N: None | int = ..., + increasing: bool = ..., +) -> NDArray[floating[Any]]: ... +@overload +def vander( + x: _ArrayLikeComplex_co, + N: None | int = ..., + increasing: bool = ..., +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def vander( + x: _ArrayLikeObject_co, + N: None | int = ..., + increasing: bool = ..., +) -> NDArray[object_]: ... + +@overload +def histogram2d( # type: ignore[misc] + x: _ArrayLikeFloat_co, + y: _ArrayLikeFloat_co, + bins: int | Sequence[int] = ..., + range: None | _ArrayLikeFloat_co = ..., + density: None | bool = ..., + weights: None | _ArrayLikeFloat_co = ..., +) -> tuple[ + NDArray[float64], + NDArray[floating[Any]], + NDArray[floating[Any]], +]: ... +@overload +def histogram2d( + x: _ArrayLikeComplex_co, + y: _ArrayLikeComplex_co, + bins: int | Sequence[int] = ..., + range: None | _ArrayLikeFloat_co = ..., + density: None | bool = ..., + weights: None | _ArrayLikeFloat_co = ..., +) -> tuple[ + NDArray[float64], + NDArray[complexfloating[Any, Any]], + NDArray[complexfloating[Any, Any]], +]: ... +@overload # TODO: Sort out `bins` +def histogram2d( + x: _ArrayLikeComplex_co, + y: _ArrayLikeComplex_co, + bins: Sequence[_ArrayLikeInt_co], + range: None | _ArrayLikeFloat_co = ..., + density: None | bool = ..., + weights: None | _ArrayLikeFloat_co = ..., +) -> tuple[ + NDArray[float64], + NDArray[Any], + NDArray[Any], +]: ... + +# NOTE: we're assuming/demanding here the `mask_func` returns +# an ndarray of shape `(n, n)`; otherwise there is the possibility +# of the output tuple having more or less than 2 elements +@overload +def mask_indices( + n: int, + mask_func: _MaskFunc[int], + k: int = ..., +) -> tuple[NDArray[intp], NDArray[intp]]: ... +@overload +def mask_indices( + n: int, + mask_func: _MaskFunc[_T], + k: _T, +) -> tuple[NDArray[intp], NDArray[intp]]: ... + +def tril_indices( + n: int, + k: int = ..., + m: None | int = ..., +) -> tuple[NDArray[int_], NDArray[int_]]: ... + +def tril_indices_from( + arr: NDArray[Any], + k: int = ..., +) -> tuple[NDArray[int_], NDArray[int_]]: ... + +def triu_indices( + n: int, + k: int = ..., + m: None | int = ..., +) -> tuple[NDArray[int_], NDArray[int_]]: ... + +def triu_indices_from( + arr: NDArray[Any], + k: int = ..., +) -> tuple[NDArray[int_], NDArray[int_]]: ... diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index 2a2982ab3..3f84b80e5 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -2,17 +2,16 @@ """ import functools -import warnings __all__ = ['iscomplexobj', 'isrealobj', 'imag', 'iscomplex', 'isreal', 'nan_to_num', 'real', 'real_if_close', - 'typename', 'asfarray', 'mintypecode', 'asscalar', + 'typename', 'asfarray', 'mintypecode', 'common_type'] +from .._utils import set_module import numpy.core.numeric as _nx from numpy.core.numeric import asarray, asanyarray, isnan, zeros -from numpy.core.overrides import set_module -from numpy.core import overrides +from numpy.core import overrides, getlimits from .ufunclike import isneginf, isposinf @@ -262,6 +261,10 @@ def isreal(x): out : ndarray, bool Boolean array of same shape as `x`. + Notes + ----- + `isreal` may behave unexpectedly for string or object arrays (see examples) + See Also -------- iscomplex @@ -269,9 +272,29 @@ def isreal(x): Examples -------- - >>> np.isreal([1+1j, 1+0j, 4.5, 3, 2, 2j]) + >>> a = np.array([1+1j, 1+0j, 4.5, 3, 2, 2j], dtype=complex) + >>> np.isreal(a) array([False, True, True, True, True, False]) + The function does not work on string arrays. + + >>> a = np.array([2j, "a"], dtype="U") + >>> np.isreal(a) # Warns about non-elementwise comparison + False + + Returns True for all elements in input array of ``dtype=object`` even if + any of the elements is complex. + + >>> a = np.array([1, "2", 3+4j], dtype=object) + >>> np.isreal(a) + array([ True, True, True]) + + isreal should not be used with object arrays + + >>> a = np.array([1+2j, 2+1j], dtype=object) + >>> np.isreal(a) + array([ True, True]) + """ return imag(x) == 0 @@ -340,6 +363,19 @@ def isrealobj(x): -------- iscomplexobj, isreal + Notes + ----- + The function is only meant for arrays with numerical values but it + accepts all other objects. Since it assumes array input, the return + value of other objects may be True. + + >>> np.isrealobj('A string') + True + >>> np.isrealobj(False) + True + >>> np.isrealobj(None) + True + Examples -------- >>> np.isrealobj(1) @@ -368,14 +404,14 @@ def _nan_to_num_dispatcher(x, copy=None, nan=None, posinf=None, neginf=None): def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None): """ Replace NaN with zero and infinity with large finite numbers (default - behaviour) or with the numbers defined by the user using the `nan`, + behaviour) or with the numbers defined by the user using the `nan`, `posinf` and/or `neginf` keywords. If `x` is inexact, NaN is replaced by zero or by the user defined value in - `nan` keyword, infinity is replaced by the largest finite floating point - values representable by ``x.dtype`` or by the user defined value in - `posinf` keyword and -infinity is replaced by the most negative finite - floating point values representable by ``x.dtype`` or by the user defined + `nan` keyword, infinity is replaced by the largest finite floating point + values representable by ``x.dtype`` or by the user defined value in + `posinf` keyword and -infinity is replaced by the most negative finite + floating point values representable by ``x.dtype`` or by the user defined value in `neginf` keyword. For complex dtypes, the above is applied to each of the real and @@ -392,27 +428,27 @@ def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None): in-place (False). The in-place operation only occurs if casting to an array does not require a copy. Default is True. - + .. versionadded:: 1.13 nan : int, float, optional - Value to be used to fill NaN values. If no value is passed + Value to be used to fill NaN values. If no value is passed then NaN values will be replaced with 0.0. - + .. versionadded:: 1.17 posinf : int, float, optional - Value to be used to fill positive infinity values. If no value is + Value to be used to fill positive infinity values. If no value is passed then positive infinity values will be replaced with a very large number. - + .. versionadded:: 1.17 neginf : int, float, optional - Value to be used to fill negative infinity values. If no value is + Value to be used to fill negative infinity values. If no value is passed then negative infinity values will be replaced with a very small (or negative) number. - + .. versionadded:: 1.17 - + Returns ------- @@ -446,7 +482,7 @@ def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None): array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary -1.28000000e+002, 1.28000000e+002]) >>> np.nan_to_num(x, nan=-9999, posinf=33333333, neginf=33333333) - array([ 3.3333333e+07, 3.3333333e+07, -9.9990000e+03, + array([ 3.3333333e+07, 3.3333333e+07, -9.9990000e+03, -1.2800000e+02, 1.2800000e+02]) >>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)]) array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary @@ -492,7 +528,7 @@ def _real_if_close_dispatcher(a, tol=None): @array_function_dispatch(_real_if_close_dispatcher) def real_if_close(a, tol=100): """ - If input is complex with all imaginary parts close to zero, return + If input is complex with all imaginary parts close to zero, return real parts. "Close to zero" is defined as `tol` * (machine epsilon of the type for @@ -504,7 +540,8 @@ def real_if_close(a, tol=100): Input array. tol : float Tolerance in machine epsilons for the complex part of the elements - in the array. + in the array. If the tolerance is <=1, then the absolute tolerance + is used. Returns ------- @@ -535,51 +572,17 @@ def real_if_close(a, tol=100): """ a = asanyarray(a) - if not issubclass(a.dtype.type, _nx.complexfloating): + type_ = a.dtype.type + if not issubclass(type_, _nx.complexfloating): return a if tol > 1: - from numpy.core import getlimits - f = getlimits.finfo(a.dtype.type) + f = getlimits.finfo(type_) tol = f.eps * tol if _nx.all(_nx.absolute(a.imag) < tol): a = a.real return a -def _asscalar_dispatcher(a): - # 2018-10-10, 1.16 - warnings.warn('np.asscalar(a) is deprecated since NumPy v1.16, use ' - 'a.item() instead', DeprecationWarning, stacklevel=3) - return (a,) - - -@array_function_dispatch(_asscalar_dispatcher) -def asscalar(a): - """ - Convert an array of size 1 to its scalar equivalent. - - .. deprecated:: 1.16 - - Deprecated, use `numpy.ndarray.item()` instead. - - Parameters - ---------- - a : ndarray - Input array of size 1. - - Returns - ------- - out : scalar - Scalar representation of `a`. The output data type is the same type - returned by the input's `item` method. - - Examples - -------- - >>> np.asscalar(np.array([24])) - 24 - """ - return a.item() - #----------------------------------------------------------------------------- _namefromtype = {'S1': 'character', diff --git a/numpy/lib/type_check.pyi b/numpy/lib/type_check.pyi new file mode 100644 index 000000000..b04da21d4 --- /dev/null +++ b/numpy/lib/type_check.pyi @@ -0,0 +1,222 @@ +from collections.abc import Container, Iterable +from typing import ( + Literal as L, + Any, + overload, + TypeVar, + Protocol, +) + +from numpy import ( + dtype, + generic, + bool_, + floating, + float64, + complexfloating, + integer, +) + +from numpy._typing import ( + ArrayLike, + DTypeLike, + NBitBase, + NDArray, + _64Bit, + _SupportsDType, + _ScalarLike_co, + _ArrayLike, + _DTypeLikeComplex, +) + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_SCT = TypeVar("_SCT", bound=generic) +_NBit1 = TypeVar("_NBit1", bound=NBitBase) +_NBit2 = TypeVar("_NBit2", bound=NBitBase) + +class _SupportsReal(Protocol[_T_co]): + @property + def real(self) -> _T_co: ... + +class _SupportsImag(Protocol[_T_co]): + @property + def imag(self) -> _T_co: ... + +__all__: list[str] + +def mintypecode( + typechars: Iterable[str | ArrayLike], + typeset: Container[str] = ..., + default: str = ..., +) -> str: ... + +# `asfarray` ignores dtypes if they're not inexact + +@overload +def asfarray( + a: object, + dtype: None | type[float] = ..., +) -> NDArray[float64]: ... +@overload +def asfarray( # type: ignore[misc] + a: Any, + dtype: _DTypeLikeComplex, +) -> NDArray[complexfloating[Any, Any]]: ... +@overload +def asfarray( + a: Any, + dtype: DTypeLike, +) -> NDArray[floating[Any]]: ... + +@overload +def real(val: _SupportsReal[_T]) -> _T: ... +@overload +def real(val: ArrayLike) -> NDArray[Any]: ... + +@overload +def imag(val: _SupportsImag[_T]) -> _T: ... +@overload +def imag(val: ArrayLike) -> NDArray[Any]: ... + +@overload +def iscomplex(x: _ScalarLike_co) -> bool_: ... # type: ignore[misc] +@overload +def iscomplex(x: ArrayLike) -> NDArray[bool_]: ... + +@overload +def isreal(x: _ScalarLike_co) -> bool_: ... # type: ignore[misc] +@overload +def isreal(x: ArrayLike) -> NDArray[bool_]: ... + +def iscomplexobj(x: _SupportsDType[dtype[Any]] | ArrayLike) -> bool: ... + +def isrealobj(x: _SupportsDType[dtype[Any]] | ArrayLike) -> bool: ... + +@overload +def nan_to_num( # type: ignore[misc] + x: _SCT, + copy: bool = ..., + nan: float = ..., + posinf: None | float = ..., + neginf: None | float = ..., +) -> _SCT: ... +@overload +def nan_to_num( + x: _ScalarLike_co, + copy: bool = ..., + nan: float = ..., + posinf: None | float = ..., + neginf: None | float = ..., +) -> Any: ... +@overload +def nan_to_num( + x: _ArrayLike[_SCT], + copy: bool = ..., + nan: float = ..., + posinf: None | float = ..., + neginf: None | float = ..., +) -> NDArray[_SCT]: ... +@overload +def nan_to_num( + x: ArrayLike, + copy: bool = ..., + nan: float = ..., + posinf: None | float = ..., + neginf: None | float = ..., +) -> NDArray[Any]: ... + +# If one passes a complex array to `real_if_close`, then one is reasonably +# expected to verify the output dtype (so we can return an unsafe union here) + +@overload +def real_if_close( # type: ignore[misc] + a: _ArrayLike[complexfloating[_NBit1, _NBit1]], + tol: float = ..., +) -> NDArray[floating[_NBit1]] | NDArray[complexfloating[_NBit1, _NBit1]]: ... +@overload +def real_if_close( + a: _ArrayLike[_SCT], + tol: float = ..., +) -> NDArray[_SCT]: ... +@overload +def real_if_close( + a: ArrayLike, + tol: float = ..., +) -> NDArray[Any]: ... + +@overload +def typename(char: L['S1']) -> L['character']: ... +@overload +def typename(char: L['?']) -> L['bool']: ... +@overload +def typename(char: L['b']) -> L['signed char']: ... +@overload +def typename(char: L['B']) -> L['unsigned char']: ... +@overload +def typename(char: L['h']) -> L['short']: ... +@overload +def typename(char: L['H']) -> L['unsigned short']: ... +@overload +def typename(char: L['i']) -> L['integer']: ... +@overload +def typename(char: L['I']) -> L['unsigned integer']: ... +@overload +def typename(char: L['l']) -> L['long integer']: ... +@overload +def typename(char: L['L']) -> L['unsigned long integer']: ... +@overload +def typename(char: L['q']) -> L['long long integer']: ... +@overload +def typename(char: L['Q']) -> L['unsigned long long integer']: ... +@overload +def typename(char: L['f']) -> L['single precision']: ... +@overload +def typename(char: L['d']) -> L['double precision']: ... +@overload +def typename(char: L['g']) -> L['long precision']: ... +@overload +def typename(char: L['F']) -> L['complex single precision']: ... +@overload +def typename(char: L['D']) -> L['complex double precision']: ... +@overload +def typename(char: L['G']) -> L['complex long double precision']: ... +@overload +def typename(char: L['S']) -> L['string']: ... +@overload +def typename(char: L['U']) -> L['unicode']: ... +@overload +def typename(char: L['V']) -> L['void']: ... +@overload +def typename(char: L['O']) -> L['object']: ... + +@overload +def common_type( # type: ignore[misc] + *arrays: _SupportsDType[dtype[ + integer[Any] + ]] +) -> type[floating[_64Bit]]: ... +@overload +def common_type( # type: ignore[misc] + *arrays: _SupportsDType[dtype[ + floating[_NBit1] + ]] +) -> type[floating[_NBit1]]: ... +@overload +def common_type( # type: ignore[misc] + *arrays: _SupportsDType[dtype[ + integer[Any] | floating[_NBit1] + ]] +) -> type[floating[_NBit1 | _64Bit]]: ... +@overload +def common_type( # type: ignore[misc] + *arrays: _SupportsDType[dtype[ + floating[_NBit1] | complexfloating[_NBit2, _NBit2] + ]] +) -> type[complexfloating[_NBit1 | _NBit2, _NBit1 | _NBit2]]: ... +@overload +def common_type( + *arrays: _SupportsDType[dtype[ + integer[Any] | floating[_NBit1] | complexfloating[_NBit2, _NBit2] + ]] +) -> type[complexfloating[_64Bit | _NBit1 | _NBit2, _64Bit | _NBit1 | _NBit2]]: ... diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index 1f26a1845..05fe60c5b 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -6,72 +6,16 @@ storing results in an output array. __all__ = ['fix', 'isneginf', 'isposinf'] import numpy.core.numeric as nx -from numpy.core.overrides import ( - array_function_dispatch, ARRAY_FUNCTION_ENABLED, -) +from numpy.core.overrides import array_function_dispatch import warnings import functools -def _deprecate_out_named_y(f): - """ - Allow the out argument to be passed as the name `y` (deprecated) - - In future, this decorator should be removed. - """ - @functools.wraps(f) - def func(x, out=None, **kwargs): - if 'y' in kwargs: - if 'out' in kwargs: - raise TypeError( - "{} got multiple values for argument 'out'/'y'" - .format(f.__name__) - ) - out = kwargs.pop('y') - # NumPy 1.13.0, 2017-04-26 - warnings.warn( - "The name of the out argument to {} has changed from `y` to " - "`out`, to match other ufuncs.".format(f.__name__), - DeprecationWarning, stacklevel=3) - return f(x, out=out, **kwargs) - - return func - - -def _fix_out_named_y(f): - """ - Allow the out argument to be passed as the name `y` (deprecated) - - This decorator should only be used if _deprecate_out_named_y is used on - a corresponding dispatcher function. - """ - @functools.wraps(f) - def func(x, out=None, **kwargs): - if 'y' in kwargs: - # we already did error checking in _deprecate_out_named_y - out = kwargs.pop('y') - return f(x, out=out, **kwargs) - - return func - - -def _fix_and_maybe_deprecate_out_named_y(f): - """ - Use the appropriate decorator, depending upon if dispatching is being used. - """ - if ARRAY_FUNCTION_ENABLED: - return _fix_out_named_y(f) - else: - return _deprecate_out_named_y(f) - - -@_deprecate_out_named_y def _dispatcher(x, out=None): return (x, out) @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_and_maybe_deprecate_out_named_y def fix(x, out=None): """ Round to nearest integer towards zero. @@ -100,7 +44,7 @@ def fix(x, out=None): See Also -------- - trunc, floor, ceil + rint, trunc, floor, ceil around : Round to given number of decimals Examples @@ -125,7 +69,6 @@ def fix(x, out=None): @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_and_maybe_deprecate_out_named_y def isposinf(x, out=None): """ Test element-wise for positive infinity, return result as bool array. @@ -189,14 +132,14 @@ def isposinf(x, out=None): try: signbit = ~nx.signbit(x) except TypeError as e: - raise TypeError('This operation is not supported for complex values ' + dtype = nx.asanyarray(x).dtype + raise TypeError(f'This operation is not supported for {dtype} values ' 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) @array_function_dispatch(_dispatcher, verify=False, module='numpy') -@_fix_and_maybe_deprecate_out_named_y def isneginf(x, out=None): """ Test element-wise for negative infinity, return result as bool array. @@ -260,7 +203,8 @@ def isneginf(x, out=None): try: signbit = nx.signbit(x) except TypeError as e: - raise TypeError('This operation is not supported for complex values ' + dtype = nx.asanyarray(x).dtype + raise TypeError(f'This operation is not supported for {dtype} values ' 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) diff --git a/numpy/lib/ufunclike.pyi b/numpy/lib/ufunclike.pyi new file mode 100644 index 000000000..82537e2ac --- /dev/null +++ b/numpy/lib/ufunclike.pyi @@ -0,0 +1,66 @@ +from typing import Any, overload, TypeVar + +from numpy import floating, bool_, object_, ndarray +from numpy._typing import ( + NDArray, + _FloatLike_co, + _ArrayLikeFloat_co, + _ArrayLikeObject_co, +) + +_ArrayType = TypeVar("_ArrayType", bound=ndarray[Any, Any]) + +__all__: list[str] + +@overload +def fix( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> floating[Any]: ... +@overload +def fix( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[floating[Any]]: ... +@overload +def fix( + x: _ArrayLikeObject_co, + out: None = ..., +) -> NDArray[object_]: ... +@overload +def fix( + x: _ArrayLikeFloat_co | _ArrayLikeObject_co, + out: _ArrayType, +) -> _ArrayType: ... + +@overload +def isposinf( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> bool_: ... +@overload +def isposinf( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[bool_]: ... +@overload +def isposinf( + x: _ArrayLikeFloat_co, + out: _ArrayType, +) -> _ArrayType: ... + +@overload +def isneginf( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> bool_: ... +@overload +def isneginf( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[bool_]: ... +@overload +def isneginf( + x: _ArrayLikeFloat_co, + out: _ArrayType, +) -> _ArrayType: ... diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index d511c2a40..095c914db 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -4,18 +4,74 @@ import textwrap import types import re import warnings +import functools +import platform +from .._utils import set_module from numpy.core.numerictypes import issubclass_, issubsctype, issubdtype -from numpy.core.overrides import set_module from numpy.core import ndarray, ufunc, asarray import numpy as np __all__ = [ 'issubclass_', 'issubsctype', 'issubdtype', 'deprecate', 'deprecate_with_doc', 'get_include', 'info', 'source', 'who', - 'lookfor', 'byte_bounds', 'safe_eval' + 'lookfor', 'byte_bounds', 'safe_eval', 'show_runtime' ] + +def show_runtime(): + """ + Print information about various resources in the system + including available intrinsic support and BLAS/LAPACK library + in use + + .. versionadded:: 1.24.0 + + See Also + -------- + show_config : Show libraries in the system on which NumPy was built. + + Notes + ----- + 1. Information is derived with the help of `threadpoolctl <https://pypi.org/project/threadpoolctl/>`_ + library if available. + 2. SIMD related information is derived from ``__cpu_features__``, + ``__cpu_baseline__`` and ``__cpu_dispatch__`` + + """ + from numpy.core._multiarray_umath import ( + __cpu_features__, __cpu_baseline__, __cpu_dispatch__ + ) + from pprint import pprint + config_found = [{ + "numpy_version": np.__version__, + "python": sys.version, + "uname": platform.uname(), + }] + features_found, features_not_found = [], [] + for feature in __cpu_dispatch__: + if __cpu_features__[feature]: + features_found.append(feature) + else: + features_not_found.append(feature) + config_found.append({ + "simd_extensions": { + "baseline": __cpu_baseline__, + "found": features_found, + "not_found": features_not_found + } + }) + try: + from threadpoolctl import threadpool_info + config_found.extend(threadpool_info()) + except ImportError: + print("WARNING: `threadpoolctl` not found in system!" + " Install it by `pip install threadpoolctl`." + " Once installed, try `np.show_runtime` again" + " for more detailed build information") + pprint(config_found) + + def get_include(): """ Return the directory that contains the NumPy \\*.h header files. @@ -25,8 +81,7 @@ def get_include(): Notes ----- - When using ``distutils``, for example in ``setup.py``. - :: + When using ``distutils``, for example in ``setup.py``:: import numpy as np ... @@ -46,11 +101,6 @@ def get_include(): return d -def _set_function_name(func, name): - func.__name__ = name - return func - - class _Deprecate: """ Decorator class to deprecate old functions. @@ -78,10 +128,7 @@ class _Deprecate: message = self.message if old_name is None: - try: - old_name = func.__name__ - except AttributeError: - old_name = func.__name__ + old_name = func.__name__ if new_name is None: depdoc = "`%s` is deprecated!" % old_name else: @@ -91,12 +138,12 @@ class _Deprecate: if message is not None: depdoc += "\n" + message - def newfunc(*args,**kwds): - """`arrayrange` is deprecated, use `arange` instead!""" + @functools.wraps(func) + def newfunc(*args, **kwds): warnings.warn(depdoc, DeprecationWarning, stacklevel=2) return func(*args, **kwds) - newfunc = _set_function_name(newfunc, old_name) + newfunc.__name__ = old_name doc = func.__doc__ if doc is None: doc = depdoc @@ -118,12 +165,7 @@ class _Deprecate: depdoc = textwrap.indent(depdoc, ' ' * indent) doc = '\n\n'.join([depdoc, doc]) newfunc.__doc__ = doc - try: - d = func.__dict__ - except AttributeError: - pass - else: - newfunc.__dict__.update(d) + return newfunc @@ -193,7 +235,32 @@ def deprecate(*args, **kwargs): else: return _Deprecate(*args, **kwargs) -deprecate_with_doc = lambda msg: _Deprecate(message=msg) + +def deprecate_with_doc(msg): + """ + Deprecates a function and includes the deprecation in its docstring. + + This function is used as a decorator. It returns an object that can be + used to issue a DeprecationWarning, by passing the to-be decorated + function as argument, this adds warning to the to-be decorated function's + docstring and returns the new function object. + + See Also + -------- + deprecate : Decorate a function such that it issues a `DeprecationWarning` + + Parameters + ---------- + msg : str + Additional explanation of the deprecation. Displayed in the + docstring after the warning. + + Returns + ------- + obj : object + + """ + return _Deprecate(message=msg) #-------------------------------------------- @@ -326,8 +393,7 @@ def who(vardict=None): maxshape = 0 maxbyte = 0 totalbytes = 0 - for k in range(len(sta)): - val = sta[k] + for val in sta: if maxname < len(val[0]): maxname = len(val[0]) if maxshape < len(val[1]): @@ -344,8 +410,7 @@ def who(vardict=None): prval = "Name %s Shape %s Bytes %s Type" % (sp1*' ', sp2*' ', sp3*' ') print(prval + "\n" + "="*(len(prval)+5) + "\n") - for k in range(len(sta)): - val = sta[k] + for val in sta: print("%s %s %s %s %s %s %s" % (val[0], ' '*(sp1-len(val[0])+4), val[1], ' '*(sp2-len(val[1])+5), val[2], ' '*(sp3-len(val[2])+5), @@ -406,7 +471,7 @@ def _makenamedict(module='numpy'): return thedict, dictlist -def _info(obj, output=sys.stdout): +def _info(obj, output=None): """Provide information about ndarray obj. Parameters @@ -432,6 +497,9 @@ def _info(obj, output=sys.stdout): strides = obj.strides endian = obj.dtype.byteorder + if output is None: + output = sys.stdout + print("class: ", nm, file=output) print("shape: ", obj.shape, file=output) print("strides: ", strides, file=output) @@ -458,22 +526,24 @@ def _info(obj, output=sys.stdout): @set_module('numpy') -def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): +def info(object=None, maxwidth=76, output=None, toplevel='numpy'): """ - Get help information for a function, class, or module. + Get help information for an array, function, class, or module. Parameters ---------- object : object or str, optional - Input object or name to get information about. If `object` is a - numpy object, its docstring is given. If it is a string, available - modules are searched for matching objects. If None, information - about `info` itself is returned. + Input object or name to get information about. If `object` is + an `ndarray` instance, information about the array is printed. + If `object` is a numpy object, its docstring is given. If it is + a string, available modules are searched for matching objects. + If None, information about `info` itself is returned. maxwidth : int, optional Printing width. output : file like object, optional File like object that the output is written to, default is - ``stdout``. The object has to be opened in 'w' or 'a' mode. + ``None``, in which case ``sys.stdout`` will be used. + The object has to be opened in 'w' or 'a' mode. toplevel : str, optional Start search at this level. @@ -506,6 +576,22 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): *** Repeat reference found in numpy.fft.fftpack *** *** Total of 3 references found. *** + When the argument is an array, information about the array is printed. + + >>> a = np.array([[1 + 2j, 3, -4], [-5j, 6, 0]], dtype=np.complex64) + >>> np.info(a) + class: ndarray + shape: (2, 3) + strides: (24, 8) + itemsize: 8 + aligned: True + contiguous: True + fortran: False + data pointer: 0x562b6e0d2860 # may vary + byteorder: little + byteswap: False + type: complex64 + """ global _namedict, _dictlist # Local import to speed up numpy's import time. @@ -518,6 +604,9 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): elif hasattr(object, '_ppimport_attr'): object = object._ppimport_attr + if output is None: + output = sys.stdout + if object is None: info(info) elif isinstance(object, ndarray): @@ -587,11 +676,11 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): print(inspect.getdoc(object), file=output) methods = pydoc.allmethods(object) - if methods != []: + + public_methods = [meth for meth in methods if meth[0] != '_'] + if public_methods: print("\n\nMethods:\n", file=output) - for meth in methods: - if meth[0] == '_': - continue + for meth in public_methods: thisobj = getattr(object, meth, None) if thisobj is not None: methstr, other = pydoc.splitdoc( @@ -878,8 +967,12 @@ def _lookfor_generate_cache(module, import_modules, regenerate): finally: sys.stdout = old_stdout sys.stderr = old_stderr - # Catch SystemExit, too + except KeyboardInterrupt: + # Assume keyboard interrupt came from a user + raise except BaseException: + # Ignore also SystemExit and pytests.importorskip + # `Skipped` (these are BaseExceptions; gh-22345) continue for n, v in _getmembers(item): @@ -938,6 +1031,12 @@ def safe_eval(source): Evaluate a string containing a Python literal expression without allowing the execution of arbitrary non-literal code. + .. warning:: + + This function is identical to :py:meth:`ast.literal_eval` and + has the same security implications. It may not always be safe + to evaluate large input strings. + Parameters ---------- source : str @@ -979,7 +1078,7 @@ def safe_eval(source): return ast.literal_eval(source) -def _median_nancheck(data, result, axis, out): +def _median_nancheck(data, result, axis): """ Utility function to check median result from data for NaN values at the end and return NaN in that case. Input result can also be a MaskedArray. @@ -987,34 +1086,58 @@ def _median_nancheck(data, result, axis, out): Parameters ---------- data : array - Input data to median function + Sorted input data to median function result : Array or MaskedArray - Result of median function - axis : {int, sequence of int, None}, optional - Axis or axes along which the median was computed. - out : ndarray, optional - Output array in which to place the result. + Result of median function. + axis : int + Axis along which the median was computed. + Returns ------- - median : scalar or ndarray - Median or NaN in axes which contained NaN in the input. + result : scalar or ndarray + Median or NaN in axes which contained NaN in the input. If the input + was an array, NaN will be inserted in-place. If a scalar, either the + input itself or a scalar NaN. """ if data.size == 0: return result - data = np.moveaxis(data, axis, -1) - n = np.isnan(data[..., -1]) + n = np.isnan(data.take(-1, axis=axis)) # masked NaN values are ok if np.ma.isMaskedArray(n): n = n.filled(False) - if result.ndim == 0: - if n == True: - if out is not None: - out[...] = data.dtype.type(np.nan) - result = out - else: - result = data.dtype.type(np.nan) - elif np.count_nonzero(n.ravel()) > 0: + if np.count_nonzero(n.ravel()) > 0: + # Without given output, it is possible that the current result is a + # numpy scalar, which is not writeable. If so, just return nan. + if isinstance(result, np.generic): + return data.dtype.type(np.nan) + result[n] = np.nan return result +def _opt_info(): + """ + Returns a string contains the supported CPU features by the current build. + + The string format can be explained as follows: + - dispatched features that are supported by the running machine + end with `*`. + - dispatched features that are "not" supported by the running machine + end with `?`. + - remained features are representing the baseline. + """ + from numpy.core._multiarray_umath import ( + __cpu_features__, __cpu_baseline__, __cpu_dispatch__ + ) + + if len(__cpu_baseline__) == 0 and len(__cpu_dispatch__) == 0: + return '' + + enabled_features = ' '.join(__cpu_baseline__) + for feature in __cpu_dispatch__: + if __cpu_features__[feature]: + enabled_features += f" {feature}*" + else: + enabled_features += f" {feature}?" + + return enabled_features #----------------------------------------------------------------------------- diff --git a/numpy/lib/utils.pyi b/numpy/lib/utils.pyi new file mode 100644 index 000000000..52ca92774 --- /dev/null +++ b/numpy/lib/utils.pyi @@ -0,0 +1,91 @@ +from ast import AST +from collections.abc import Callable, Mapping, Sequence +from typing import ( + Any, + overload, + TypeVar, + Protocol, +) + +from numpy import ndarray, generic + +from numpy.core.numerictypes import ( + issubclass_ as issubclass_, + issubdtype as issubdtype, + issubsctype as issubsctype, +) + +_T_contra = TypeVar("_T_contra", contravariant=True) +_FuncType = TypeVar("_FuncType", bound=Callable[..., Any]) + +# A file-like object opened in `w` mode +class _SupportsWrite(Protocol[_T_contra]): + def write(self, s: _T_contra, /) -> Any: ... + +__all__: list[str] + +class _Deprecate: + old_name: None | str + new_name: None | str + message: None | str + def __init__( + self, + old_name: None | str = ..., + new_name: None | str = ..., + message: None | str = ..., + ) -> None: ... + # NOTE: `__call__` can in principle take arbitrary `*args` and `**kwargs`, + # even though they aren't used for anything + def __call__(self, func: _FuncType) -> _FuncType: ... + +def get_include() -> str: ... + +@overload +def deprecate( + *, + old_name: None | str = ..., + new_name: None | str = ..., + message: None | str = ..., +) -> _Deprecate: ... +@overload +def deprecate( + func: _FuncType, + /, + old_name: None | str = ..., + new_name: None | str = ..., + message: None | str = ..., +) -> _FuncType: ... + +def deprecate_with_doc(msg: None | str) -> _Deprecate: ... + +# NOTE: In practice `byte_bounds` can (potentially) take any object +# implementing the `__array_interface__` protocol. The caveat is +# that certain keys, marked as optional in the spec, must be present for +# `byte_bounds`. This concerns `"strides"` and `"data"`. +def byte_bounds(a: generic | ndarray[Any, Any]) -> tuple[int, int]: ... + +def who(vardict: None | Mapping[str, ndarray[Any, Any]] = ...) -> None: ... + +def info( + object: object = ..., + maxwidth: int = ..., + output: None | _SupportsWrite[str] = ..., + toplevel: str = ..., +) -> None: ... + +def source( + object: object, + output: None | _SupportsWrite[str] = ..., +) -> None: ... + +def lookfor( + what: str, + module: None | str | Sequence[str] = ..., + import_modules: bool = ..., + regenerate: bool = ..., + output: None | _SupportsWrite[str] =..., +) -> None: ... + +def safe_eval(source: str | AST) -> Any: ... + +def show_runtime() -> None: ... |